def main(): lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = setup_problem.load_problem(lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) # Let's plot prediction functions and compare coefficients for several fits # and the target function. pred_fns = [] x = np.sort(np.concatenate([np.arange(0,1,.001), x_train])) pred_fns.append({"name": "Target Parameter Values (i.e. Bayes Optimal)", "coefs": coefs_true, "preds": target_fn(x)}) estimator = MLPRegression(num_hidden_units=10, step_size=0.001, init_param_scale=.0005, max_num_epochs=5000) x_train_as_column_vector = x_train.reshape(x_train.shape[0],1) # fit expects a 2-dim array x_as_column_vector = x.reshape(x.shape[0],1) # fit expects a 2-dim array estimator.fit(x_train_as_column_vector, y_train) name = "MLP regression - no features" pred_fns.append({"name":name, "preds": estimator.predict(x_as_column_vector) }) X = featurize(x) estimator = MLPRegression(num_hidden_units=10, step_size=0.0005, init_param_scale=.01, max_num_epochs=500) estimator.fit(X_train, y_train) name = "MLP regression - with features" pred_fns.append({"name":name, "preds": estimator.predict(X) }) plot_utils.plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best")
def main(): lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = setup_problem.load_problem(lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) pred_fns = [] x = np.sort(np.concatenate([np.arange(0,1,.001), x_train])) X = featurize(x) l2reg = 1 estimator = RidgeRegression(l2_reg=l2reg, step_size=0.00005, max_num_epochs=2000) estimator.fit(X_train, y_train) name = "Ridge with L2Reg="+str(l2reg) pred_fns.append({"name":name, "preds": estimator.predict(X) }) l2reg = 0 estimator = RidgeRegression(l2_reg=l2reg, step_size=0.0005, max_num_epochs=500) estimator.fit(X_train, y_train) name = "Ridge with L2Reg="+str(l2reg) pred_fns.append({"name":name, "preds": estimator.predict(X) }) # Let's plot prediction functions and compare coefficients for several fits # and the target function. pred_fns.append({"name": "Target Parameter Values (i.e. Bayes Optimal)", "coefs": coefs_true, "preds": target_fn(x)}) plot_utils.plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best")
def main(): lasso_data_fname = "lasso_data.pkl" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = setup_problem.load_problem( lasso_data_fname ) # Generate features X_train = featurize(x_train) # X_val = featurize(x_val) # Let's plot prediction functions and compare coefficients for several fits # and the target function. pred_fns = [] x = np.sort(np.concatenate([np.arange(0, 1, 0.001), x_train])) pred_fns.append( { "name": "Target Parameter Values (i.e. Bayes Optimal)", "coefs": coefs_true, "preds": target_fn(x), } ) X = featurize(x) estimator = LinearRegression(step_size=0.001, max_num_epochs=1000) estimator.fit(X_train, y_train, print_every=100) name = "Linear regression" pred_fns.append({"name": name, "preds": estimator.predict(X)}) plot_utils.plot_prediction_functions( x, pred_fns, x_train, y_train, legend_loc="best" ) os.makedirs("img", exist_ok=True) plt.savefig(os.path.join("img", "linear_regression.png")) plt.show()
def main(): lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) #Visualize training data fig, ax = plt.subplots() ax.imshow(X_train) ax.set_title("Design Matrix: Color is Feature Value") ax.set_xlabel("Feature Index") ax.set_ylabel("Example Number") plt.show(block=False) # Compare our RidgeRegression to sklearn's. compare_our_ridge_with_sklearn(X_train, y_train, l2_reg = 1.5) # Do hyperparameter tuning with our ridge regression grid, results = do_grid_search_ridge(X_train, y_train, X_val, y_val) print(results) # Plot validation performance vs regularization parameter fig, ax = plt.subplots() # ax.loglog(results["param_l2reg"], results["mean_test_score"]) ax.semilogx(results["param_l2reg"], results["mean_test_score"]) ax.grid() ax.set_title("Validation Performance vs L2 Regularization") ax.set_xlabel("L2-Penalty Regularization Parameter") ax.set_ylabel("Mean Squared Error") fig.show() # Let's plot prediction functions and compare coefficients for several fits # and the target function. pred_fns = [] x = np.sort(np.concatenate([np.arange(0,1,.001), x_train])) name = "Target Parameter Values (i.e. Bayes Optimal)" pred_fns.append({"name":name, "coefs":coefs_true, "preds": target_fn(x) }) l2regs = [0, grid.best_params_['l2reg'], 1] X = featurize(x) for l2reg in l2regs: ridge_regression_estimator = RidgeRegression(l2reg=l2reg) ridge_regression_estimator.fit(X_train, y_train) name = "Ridge with L2Reg="+str(l2reg) pred_fns.append({"name":name, "coefs":ridge_regression_estimator.w_, "preds": ridge_regression_estimator.predict(X) }) f = plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best") f.show() f = compare_parameter_vectors(pred_fns) f.show() coefs_confusion_matrices(coefs_true, pred_fns[1]["coefs"], 10.**np.array([-6, -3, -1]))
def main(): lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = setup_problem.load_problem(lasso_data_fname) # Generate features X_train = featurize(x_train)#这里featurize是setup_problem.py里get_target_and_featurizer()return的函数 X_val = featurize(x_val) # Let's plot prediction functions and compare coefficients for several fits # and the target function. pred_fns = [] x = np.sort(np.concatenate([np.arange(0,1,.001), x_train])) pred_fns.append({"name": "Target Parameter Values (i.e. Bayes Optimal)", "coefs": coefs_true, "preds": target_fn(x)}) X = featurize(x) estimator = LinearRegression(step_size=0.001, max_num_epochs=1000) estimator.fit(X_train, y_train) name = "Linear regression" pred_fns.append({"name":name, "preds": estimator.predict(X) }) plot_utils.plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best")
def main(): lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem( lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) #Visualize training data fig, ax = plt.subplots() ax.imshow(X_train) ax.set_title("Design Matrix: Color is Feature Value") ax.set_xlabel("Feature Index") ax.set_ylabel("Example Number") plt.show(block=False) # Compare our RidgeRegression to sklearn's. compare_our_ridge_with_sklearn(X_train, y_train, l2_reg=1.5) # Do hyperparameter tuning with our ridge regression grid, results = do_grid_search_ridge(X_train, y_train, X_val, y_val) print(results)
def main(): lasso_data_fname = "c:/Users/jack/ml_homework_2020/homework7/code/lasso_data.pickle" #lasso_data_fname = "/Users/shunshun/github/ml_homework_2020/homework7/code/lasso_data.pickle" #lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = setup_problem.load_problem( lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) print(y_train[1]) print(X_train.shape, y_train.shape) # Let's plot prediction functions and compare coefficients for several fits # and the target function. pred_fns = [] x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train])) pred_fns.append({ "name": "Target Parameter Values (i.e. Bayes Optimal)", "coefs": coefs_true, "preds": target_fn(x) }) X = featurize(x) estimator = LinearRegression(step_size=0.001, max_num_epochs=100) estimator.fit(X_train, y_train) name = "Linear regression" pred_fns.append({"name": name, "preds": estimator.predict(X)}) plot_utils.plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best")
def main(): lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem( lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) #Visualize training data fig, ax = plt.subplots() ax.imshow(X_train) ax.set_title("Design Matrix: Color is Feature Value") ax.set_xlabel("Feature Index") ax.set_ylabel("Example Number") plt.show(block=False) # Compare our RidgeRegression to sklearn's. compare_our_ridge_with_sklearn(X_train, y_train, l2_reg=1.5) # Do hyperparameter tuning with our ridge regression grid, results = do_grid_search_ridge(X_train, y_train, X_val, y_val) print(results) # Plot validation performance vs regularization parameter fig, ax = plt.subplots() # ax.loglog(results["param_l2reg"], results["mean_test_score"]) ax.semilogx(results["param_l2reg"], results["mean_test_score"]) ax.grid() ax.set_title("Validation Performance vs L2 Regularization") ax.set_xlabel("L2-Penalty Regularization Parameter") ax.set_ylabel("Mean Squared Error") fig.show() # Let's plot prediction functions and compare coefficients for several fits # and the target function. pred_fns = [] x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train])) name = "Target Parameter Values (i.e. Bayes Optimal)" pred_fns.append({"name": name, "coefs": coefs_true, "preds": target_fn(x)}) l2regs = [0, grid.best_params_['l2reg'], 1] X = featurize(x) for l2reg in l2regs: ridge_regression_estimator = RidgeRegression(l2reg=l2reg) ridge_regression_estimator.fit(X_train, y_train) name = "Ridge with L2Reg=" + str(l2reg) pred_fns.append({ "name": name, "coefs": ridge_regression_estimator.w_, "preds": ridge_regression_estimator.predict(X) }) f = plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best") f.show() f = compare_parameter_vectors(pred_fns) f.show() ##Sample code for plotting a matrix ## Note that this is a generic code for confusion matrix ## You still have to make y_true and y_pred by thresholding as per the insturctions in the question. y_true = [1, 0, 1, 1, 0, 1] y_pred = [0, 0, 1, 1, 0, 1] eps = 1e-1 cnf_matrix = confusion_matrix(y_true, y_pred) plt.figure() plot_confusion_matrix( cnf_matrix, title="Confusion Matrix for $\epsilon = {}$".format(eps), classes=["Zero", "Non-Zero"]) plt.show()
def main(): lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem( lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) ''' #Visualize training data fig, ax = plt.subplots() ax.imshow(X_train) ax.set_title("Design Matrix: Color is Feature Value") ax.set_xlabel("Feature Index") ax.set_ylabel("Example Number") #fig.show() # Compare our RidgeRegression to sklearn's. #compare_our_ridge_with_sklearn(X_train, y_train, l2_reg = 1.5) # Do hyperparameter tuning with our ridge regression grid, results = do_grid_search_ridge(X_train, y_train, X_val, y_val) #print(results) # Plot validation performance vs regularization parameter fig, ax = plt.subplots() #ax.loglog(results["param_l2reg"], results["mean_test_score"]) ax.semilogx(results["param_l2reg"], results["mean_test_score"]) ax.grid() ax.set_title("Validation Performance vs L2 Regularization") ax.set_xlabel("L2-Penalty Regularization Parameter") ax.set_ylabel("Mean Squared Error") #fig.show() ''' #Applying coordinate descent: pred_fns = [] l1reg = 1.5 x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train])) X = featurize(x) name = "Target Parameter Values (i.e. Bayes Optimal)" pred_fns.append({"name": name, "coefs": coefs_true, "preds": target_fn(x)}) name = "Shooting algorithm with L1Reg = 1.5" #shooting_regression_estimator = random_coordinate_descent(X_train, y_train, l1reg) shooting_regression_estimator = coordinate_descent(X_train, y_train, l1reg) shooting_regression_prediction = shoot_predict( X, shooting_regression_estimator) #print(type(shooting_regression_estimator)) #print(shooting_regression_prediction.shape) pred_fns.append({ "name": name, "coefs": shooting_regression_estimator, "preds": shooting_regression_prediction }) #f = plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best") #plt.show() #f = compare_parameter_vectors(pred_fns) #plt.show() do_grid_search_lasso(X_train, y_train, X_val, y_val) '''
def main(): # Load problem lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem( lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) #Visualize training data # fig, ax = plt.subplots() # ax.imshow(X_train) # ax.set_title("Design Matrix: Color is Feature Value") # ax.set_xlabel("Feature Index") # ax.set_ylabel("Example Number") # plt.show(block=False) # Do hyperparameter tuning with our ridge regression # this is done on the training and validation set grid, results = do_grid_search_ridge(X_train, y_train, X_val, y_val) print(results) # Plot validation performance vs regularization parameter fig, ax = plt.subplots() # ax.loglog(results["param_l2reg"], results["mean_test_score"]) ax.semilogx(results["param_l2reg"], results["mean_test_score"]) ax.grid() ax.set_title("Validation Performance vs L2 Regularization") ax.set_xlabel("L2-Penalty Regularization Parameter") ax.set_ylabel("Mean Squared Error") plt.show() # Let's plot prediction functions and compare coefficients for several fits # and the target function. pred_fns = [] x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train])) name = "Target Parameter Values (i.e. Bayes Optimal)" pred_fns.append({"name": name, "coefs": coefs_true, "preds": target_fn(x)}) l2regs = [0, grid.best_params_['l2reg'], 1] X = featurize(x) for l2reg in l2regs: # for every chosen regularization constant ridge_regression_estimator = RidgeRegression(l2reg=l2reg) # fit ridge_regression_estimator.fit(X_train, y_train) name = "Ridge with L2Reg=" + str(l2reg) pred_fns.append({ "name": name, "coefs": ridge_regression_estimator.w_, "preds": ridge_regression_estimator.predict(X) }) # f = plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best") # plt.show() # f = compare_parameter_vectors(pred_fns) # plt.show() # confusion matrix for different cutoff params cutoffs = [10**(-3), 10**(-2), 10**(-1)] best = pred_fns[1] ctf_fns = [] for cutoff in cutoffs: ridge_regression_estimator = RidgeRegression() W = [w * (abs(w) > cutoff) for w in best["coefs"]] ridge_regression_estimator.w_ = W name = "Ridge with cutoff=" + str(cutoff) ctf_fns.append({ "name": name, "coefs": W, "preds": ridge_regression_estimator.predict(X) }) f = plot_prediction_functions(x, ctf_fns, x_train, y_train, legend_loc="best") plt.show()
def main(): lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) print('x_train shape: ' + str(x_train.shape)) print('y_train shape: ' + str(y_train.shape)) print('X_train shape: ' + str(X_train.shape)) lmbd_max = np.max(2 * X_train.T.dot(y_train)) print('Max lambda: {:.4f}'.format(lmbd_max)) best_score = np.finfo(np.float32).max best_estimator = None best_l1reg = 0.0 legend = [] l1reg_costs = [] l1reg_train_costs = [] # l1reg_range = [0.0001, 0.01, .1, .5, 1, 1.5, 1.75, 2, 5, 10, 20] l1reg_range = lmbd_max * np.power(.8, range(30)) x_plot = np.linspace(0, 1, 1000) f1=plt.figure(1) plt.scatter(x_train, y_train, marker='^', c='g') for l1reg in l1reg_range: lasso = LassoRegression(l1reg=l1reg, zero_start=False, random_coordinate=False) lasso.fit(X_train, y_train) # no regularization if l1reg == 0.0001: legend.append('Regular regression') plt.plot(x_plot, lasso.predict(featurize(x_plot))) score = lasso.score(X_val, y_val) l1reg_costs.append(score) score_train = lasso.score(X_train, y_train) l1reg_train_costs.append(score_train) if best_score > score: best_score = score best_estimator = lasso best_l1reg = l1reg print('l1reg: {:.4f} validation score: {:.4f} train score: {:.4f}'.format(l1reg, score, score_train)) legend.append('Best lasso regression estimation, lambda: {:.4f}'.format(best_l1reg)) plt.plot(x_plot, best_estimator.predict(featurize(x_plot)), '-r') print('length of vector w: {:} number of non-zero elements: {:}'.format(len(best_estimator.w_), np.count_nonzero(best_estimator.w_))) plt.legend(legend) plt.title('Lasso regression') plt.grid() f1.show() f2=plt.figure(2) plt.plot(l1reg_costs, '-r^', l1reg_train_costs, '-g*') plt.legend(['Test set score', 'Train set score']) plt.grid() f2.show() plt.show()
thresh = cm.max() / 2. for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): plt.text(j, i, format(cm[i, j], 'd'), horizontalalignment="center", color="white" if cm[i, j] > thresh else "black") plt.tight_layout() plt.ylabel('True label') plt.xlabel('Predicted label') lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem( lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) #Visualize training data fig, ax = plt.subplots() ax.imshow(X_train) ax.set_title("Design Matrix: Color is Feature Value") ax.set_xlabel("Feature Index") ax.set_ylabel("Example Number") plt.show(block=False) # Compare our RidgeRegression to sklearn's. compare_our_ridge_with_sklearn(X_train, y_train, l2_reg=1.5)
def main(): if len(argv) == 2: program, sol = argv else: raise RuntimeError("USAGE: python solution.py OPTION[2 or 3]") # load data and split data lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem( lasso_data_fname) # turn from 1D binary data to high dimensional featurized data X_train = featurize(x_train) X_val = featurize(x_val) if sol == "2": #### 2.1 # create array of possible L2Reg parameters l2reg_search = 10.**np.arange(-6, +1, 1) # search through l2reg_search l2reg_opt = run_2_1(X_train, y_train, X_val, y_val, l2reg_search, print_table=False, PLOT=False) #### 2.2 # x has many inputs from 0 to 1, as well as the x_train inputs, to help plotting x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train])) X = featurize(x) # pred_fns is a list of dicts with "name", "coefs" and "preds" pred_fns = [] coefs_opt = 0 #for question 2.3 # first entry: Target function pred_fns.append({ "name": "Target", "coefs": coefs_true, "preds": target_fn(x) }) l2reg_values = [0, l2reg_opt] # next entries: prediction functions for L2Reg parameters in l2reg_values for l2reg in l2reg_values: ridge = RidgeRegression(l2reg=l2reg) ridge.fit(X_train, y_train) pred_fns.append({ "name": "Ridge with L2Reg=" + str(l2reg), "coefs": ridge.w_, "preds": ridge.predict(X) }) # for question 2.3 if l2reg == l2reg_opt: coefs_opt = ridge.w_ # with pred_fns populated, plot # "PRED": prediction functions # "COEF": coefficients plots = ["PRED", "COEF"] #plots=[] run_2_2(x, x_train, y_train, pred_fns, plot=plots) #### 2.3 epsilon = [] #epsilon = [1e-6, 1e-3, 1e-2, 5e-2, 1e-1, 5e-1] for e in epsilon: run_2_3(coefs_true, coefs_opt, epsilon=e) if sol == "3": #### 3.2 - experiment with Lasso # Found that start="RR", order="cyclic", epsilon=1e-8 works MARGINALLY better #run_3_2(X_train, y_train, X_val, y_val, l1reg=1, epsilons=[1e-8, 1e-3]) #### 3.3 #### Part a: find optimal l1reg # create array of possible L1Reg parameters #l1reg_search = 10.**np.arange(-6, 2, 1) # search through l1reg_search #l1reg_opt = run_3_3_a(X_train, y_train, X_val, y_val, l1reg_search) l1reg_opt = 1.0 # found from above #### 3.3 #### Part b: plot corresponding prediction function # x has many inputs from 0 to 1, as well as the x_train inputs, to help plotting x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train])) X = featurize(x) # pred_fns is a list of dicts with "name", "coefs" and "preds" pred_fns = [] # first entry: Target function pred_fns.append({ "name": "Target", "coefs": coefs_true, "preds": target_fn(x) }) lasso = LassoRegression(l1reg=l1reg_opt) lasso.shooting_alg(X_train, y_train) pred_fns.append({ "name": "Ridge with L1Reg=" + str(l1reg_opt), "coefs": lasso.w, "preds": lasso.predict(X) }) # with pred_fns populated, plot # "PRED": prediction functions # "COEF": coefficients run_3_3_b(x, x_train, y_train, pred_fns, plot=[]) run_3_4(X_train, y_train, X_val, y_val, p=0.8)
def main(): lasso_data_fname = "lasso_data.pickle" x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem( lasso_data_fname) # Generate features X_train = featurize(x_train) X_val = featurize(x_val) # print('Original: ' + str(x_train[0]) + ' Featurized: ' + str(X_train[0])) print('Featurized shape: ' + str(X_train[0].shape)) # Visualize data f1 = plt.figure(1) plt.subplot(2, 1, 1) plt.subplots_adjust(hspace=0.5) legend = [] l2reg_range = np.concatenate( (np.linspace(0, 1, 8, endpoint=False), np.linspace(1, 5, 4))) best_score = np.finfo(np.float32).max best_estimator = None base_weights = np.zeros(X_train[0].shape[0]) best_l2reg = 0 l2reg_costs = [] x_plot = np.linspace(0, 1, 1000) for l2reg in l2reg_range: ridge_regression_estimator = MyRidge(l2reg=l2reg) ridge_regression_estimator.fit(X_train, y_train) if l2reg == 0: y_plot = ridge_regression_estimator.predict(featurize(x_plot)) base_weights = ridge_regression_estimator.w_ plt.plot(x_plot, y_plot) legend.append('l2reg {:.4}'.format(l2reg)) score = ridge_regression_estimator.score(X_val, y_val) l2reg_costs.append(score) score_train = ridge_regression_estimator.score(X_train, y_train) if best_score > score: best_score = score best_l2reg = l2reg best_estimator = ridge_regression_estimator print('l2reg: {:.2f} validation score: {:.4f} train score: {:.4f}'. format(l2reg, score, score_train)) legend.append('Best ridge estimation') plt.plot(x_plot, best_estimator.predict(featurize(x_plot)), '-r') legend.append('Bayes estimation') plt.plot(x_plot, target_fn(x_plot), '-c') legend.append('data') plt.scatter(x_train, y_train, marker='^', c='g') plt.legend(legend) plt.title('Ridge regression') plt.grid() # print('=>' + str(len(legend))) # Visualize cost vs l2reg plt.subplot(2, 1, 2) plt.title('Ridge regression cost') plt.grid() plt.plot(l2reg_range, l2reg_costs, '-rx') f1.show() # Visualize weights f2 = plt.figure(2) plt.subplots_adjust(hspace=0.5) plt.subplot(3, 1, 1) plt.grid() plt.title('Weights without regularization') plt.bar(range(X_train[0].shape[0]), base_weights) plt.subplot(3, 1, 2) plt.grid() plt.title('Best regularized weights') plt.bar(range(X_train[0].shape[0]), ridge_regression_estimator.w_) plt.subplot(3, 1, 3) plt.grid() plt.title('Bayes regularized weights') plt.bar(range(X_train[0].shape[0]), coefs_true) f2.show() print('Best performance with l2reg: {:.4f} score: {:.4f}'.format( best_l2reg, best_score)) print('length of vector w: {:} number of non-zero elements: {:}'.format( len(best_estimator.w_), np.count_nonzero(best_estimator.w_))) best_w_adjusted = np.copy(best_estimator.w_) print( 'length of vector w: {:} number of non-zero elements with tolerance 1e-6: {:}' .format(len(best_estimator.w_), np.count_nonzero(best_w_adjusted[best_w_adjusted > 1e-6]))) plt.show()