def lasso(lambda_,degree,x,y,z,k,B): MSE_lasso_boot = np.zeros(degree) MSE_lasso_cross = np.zeros(degree) min_error_boot = np.zeros(len(lambda_)) min_error_cross = np.zeros(len(lambda_)) lasso_heatmap_boot = np.zeros((len(lambda_),degree)) lasso_heatmap_cross = np.zeros((len(lambda_),degree)) degree_index_boot = np.zeros(len(lambda_)) degree_index_cross = np.zeros(len(lambda_)) deg = np.linspace(1,degree,degree) i = 0 for lambdas in lambda_: _,MSE_lasso_boot,bias,variance,min_error_boot[i] = bootstrap(B,x,y,z,"lasso",lambdas,degree) lasso_heatmap_boot[i] = MSE_lasso_boot degree_index_boot[i] = deg[np.argmin(MSE_lasso_boot)] MSE_lasso_cross,_,_,min_error_cross[i],_,_,_ = cross_validation(k,x,y,z,degree,"lasso",lambdas) lasso_heatmap_cross[i] = MSE_lasso_cross degree_index_cross[i] = deg[np.argmin(MSE_lasso_boot)] #plt.plot(deg,MSE_lasso_boot,label="error") #plt.plot(deg,bias,label="bias") #plt.plot(deg,variance,label="variance") i+=1 #sb.color_palette("viridis_r", as_cmap=True) """ plt.title("Bootstrap vs Cross for B = {:d} k = {:d} $\lambda$ = {:.4} n = {:d}".format(B,k,lambdas,n)) plt.plot(deg,MSE_lasso_boot,label="boot") plt.plot(deg,MSE_lasso_cross,label="cross") plt.legend() plt.savefig("./figures/lasso_boot_vs_cross{:d}.pdf".format(i),bbox_inches = 'tight',pad_inches = 0.1,dpi=1200) plt.show()""" """ heatmap = sb.heatmap(lasso_heatmap_boot,annot=True,cmap="viridis_r",yticklabels=lambda_,cbar_kws={'label': 'Mean squared error'}) heatmap.set_xlabel("Complexity") heatmap.set_ylabel("$\lambda$") heatmap.invert_yaxis() heatmap.set_title("Heatmap made from {:} bootstraps".format(B)) plt.show() heatmap = sb.heatmap(lasso_heatmap_cross,annot=True,cmap="viridis_r",yticklabels=lambda_,cbar_kws={'label': 'Mean squared error'}) heatmap.set_xlabel("Complexity") heatmap.set_ylabel("$\lambda$") heatmap.invert_yaxis() heatmap.set_title("Heatmap made from {:} folds in cross validation".format(k)) plt.show() """ """ plt.plot(deg,min_error_boot,label="bootstrap") plt.plot(deg,min_error_cross,label="cross validation") plt.legend() plt.show() """ return min_error_boot,min_error_cross,lasso_heatmap_boot,lasso_heatmap_cross,degree_index_boot,degree_index_cross
def ridge(x, y, z, k, B, lambda_, degree): scaler = StandardScaler() deg = np.linspace(1, degree, degree) #Its important to send the meshgrid into the design matrix function #how many models we'll make we have int(n/k) values per model MSE_ridge_boot = np.zeros(degree) MSE_ridge_cross = np.zeros(degree) min_error_boot = np.zeros(len(lambda_)) min_error_cross = np.zeros(len(lambda_)) ridge_heatmap_boot = np.zeros((len(lambda_), degree)) ridge_heatmap_cross = np.zeros((len(lambda_), degree)) degree_index_boot = np.zeros(len(lambda_)) degree_index_cross = np.zeros(len(lambda_)) i = 0 plt.style.use("seaborn") for lambdas in lambda_: _, MSE_ridge_boot, bias_boot, variance_boot, min_error_boot[ i] = bootstrap(B, x, y, z, "ridge", lambdas, degree) ridge_heatmap_boot[i] = MSE_ridge_boot degree_index_boot[i] = deg[np.argmin( MSE_ridge_boot )] #the smallest MSE_ridge_boot value will give a degree, which we insert into the degree_index_boot array MSE_ridge_cross, _, _, min_error_cross[i], _, _, _ = cross_validation( k, x, y, z, degree, "ridge", lambdas) ridge_heatmap_cross[i] = MSE_ridge_cross degree_index_cross[i] = deg[np.argmin(MSE_ridge_cross)] """plt.title("Bootstrap vs Cross validation $\lambda$ = {:.4} k = {:d} B = {:d} n = {:d}".format(lambdas,k,B,n)) plt.plot(deg,MSE_ridge_boot,label="error") plt.plot(deg,MSE_ridge_cross,label="Cross validation") plt.legend() plt.savefig("./figures/ridge_boot_vs_cross{:d}.pdf".format(i),bbox_inches = 'tight',pad_inches = 0.1,dpi = 1200) plt.show() plt.title("Bias Variance $\lambda$ = {:.4} B = {:d} n = {:d} ".format(lambdas,B,n)) plt.plot(deg,bias_boot,label="bias") plt.plot(deg,variance_boot,label="variance") plt.plot(deg,MSE_ridge_boot,label="error") plt.legend() plt.savefig("./figures/ridge_bias_variance{:d}.pdf".format(i),bbox_inches = 'tight',pad_inches = 0.1,dpi = 1200) plt.show() """ i += 1 return min_error_boot, min_error_cross, degree_index_boot, degree_index_cross, ridge_heatmap_boot, ridge_heatmap_cross
plt.title( "Beta coefficients with their confidence intervals calculated for n = {:d}" .format(n)) plt.xlabel(r"$\beta$ ") plt.errorbar(x_axis, beta, std_beta, fmt="o") plt.savefig("./figures/a_beta.jpg", bbox_inches='tight', pad_inches=0.1, dpi=1200) plt.show() #Then we're doing OLS with bootstrap B = 75 method = "ols" _, MSE_boot, _, _, _ = bootstrap(B, x, y, terrain_scaled, method, 0, degree) file.write("OLS with bootstrap\n") file.write("n = {:} degree = {:} bootstraps = {:}\n".format( n, deg[np.argmin(MSE_boot)], B)) file.write("MSE = {:.4}\n".format(np.min(MSE_boot))) """ plt.title("Mean squared error with bootstrap & OLS on terrain data") plt.xlabel("Complexity") plt.ylabel("MSE") plt.plot(deg,MSE_boot,label="Bootstrap OLS") plt.legend() plt.show() """ #Then we're doing OLS with k fold cross validation
""" plt.title("Cross validation with {:d} folds".format(k)) plt.plot(deg,(MSE_fold),label="MSE") plt.xlabel("Complexity") plt.ylabel("Error") plt.legend() plt.show() """ """ plt.plot(deg,R2_score,label="manual k fold R2 score") plt.legend() plt.show() """ #Comparing with Bootstrap B = 75 _, MSE_boot, _, _, _ = bootstrap(B, x, y, z, "ols", 0, degree) plt.title( "Bootstrap vs Cross validation {:} folds {:} bootstraps n = {:d}". format(k, B, n)) plt.plot(deg, MSE_fold, label="Cross validation") plt.plot(deg, MSE_boot, label="Bootstrap") plt.xlabel("Complexity") plt.ylabel("MSE") plt.legend() plt.savefig("./figures/c_boot_vs_cross.jpg", bbox_inches='tight', pad_inches=0.1) plt.show()