def main():
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = setup_problem.load_problem(lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
# Let's plot prediction functions and compare coefficients for several fits
# and the target function.
pred_fns = []
x = np.sort(np.concatenate([np.arange(0,1,.001), x_train]))
pred_fns.append({"name": "Target Parameter Values (i.e. Bayes Optimal)", "coefs": coefs_true, "preds": target_fn(x)})
estimator = MLPRegression(num_hidden_units=10, step_size=0.001, init_param_scale=.0005, max_num_epochs=5000)
x_train_as_column_vector = x_train.reshape(x_train.shape[0],1) # fit expects a 2-dim array
x_as_column_vector = x.reshape(x.shape[0],1) # fit expects a 2-dim array
estimator.fit(x_train_as_column_vector, y_train)
name = "MLP regression - no features"
pred_fns.append({"name":name, "preds": estimator.predict(x_as_column_vector) })
X = featurize(x)
estimator = MLPRegression(num_hidden_units=10, step_size=0.0005, init_param_scale=.01, max_num_epochs=500)
estimator.fit(X_train, y_train)
name = "MLP regression - with features"
pred_fns.append({"name":name, "preds": estimator.predict(X) })
plot_utils.plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best")
def main():
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = setup_problem.load_problem(lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
pred_fns = []
x = np.sort(np.concatenate([np.arange(0,1,.001), x_train]))
X = featurize(x)
l2reg = 1
estimator = RidgeRegression(l2_reg=l2reg, step_size=0.00005, max_num_epochs=2000)
estimator.fit(X_train, y_train)
name = "Ridge with L2Reg="+str(l2reg)
pred_fns.append({"name":name, "preds": estimator.predict(X) })
l2reg = 0
estimator = RidgeRegression(l2_reg=l2reg, step_size=0.0005, max_num_epochs=500)
estimator.fit(X_train, y_train)
name = "Ridge with L2Reg="+str(l2reg)
pred_fns.append({"name":name, "preds": estimator.predict(X) })
# Let's plot prediction functions and compare coefficients for several fits
# and the target function.
pred_fns.append({"name": "Target Parameter Values (i.e. Bayes Optimal)", "coefs": coefs_true, "preds": target_fn(x)})
plot_utils.plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best")
def main():
lasso_data_fname = "lasso_data.pkl"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = setup_problem.load_problem(
lasso_data_fname
)
# Generate features
X_train = featurize(x_train)
# X_val = featurize(x_val)
# Let's plot prediction functions and compare coefficients for several fits
# and the target function.
pred_fns = []
x = np.sort(np.concatenate([np.arange(0, 1, 0.001), x_train]))
pred_fns.append(
{
"name": "Target Parameter Values (i.e. Bayes Optimal)",
"coefs": coefs_true,
"preds": target_fn(x),
}
)
X = featurize(x)
estimator = LinearRegression(step_size=0.001, max_num_epochs=1000)
estimator.fit(X_train, y_train, print_every=100)
name = "Linear regression"
pred_fns.append({"name": name, "preds": estimator.predict(X)})
plot_utils.plot_prediction_functions(
x, pred_fns, x_train, y_train, legend_loc="best"
)
os.makedirs("img", exist_ok=True)
plt.savefig(os.path.join("img", "linear_regression.png"))
plt.show()
def main():
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
#Visualize training data
fig, ax = plt.subplots()
ax.imshow(X_train)
ax.set_title("Design Matrix: Color is Feature Value")
ax.set_xlabel("Feature Index")
ax.set_ylabel("Example Number")
plt.show(block=False)
# Compare our RidgeRegression to sklearn's.
compare_our_ridge_with_sklearn(X_train, y_train, l2_reg = 1.5)
# Do hyperparameter tuning with our ridge regression
grid, results = do_grid_search_ridge(X_train, y_train, X_val, y_val)
print(results)
# Plot validation performance vs regularization parameter
fig, ax = plt.subplots()
# ax.loglog(results["param_l2reg"], results["mean_test_score"])
ax.semilogx(results["param_l2reg"], results["mean_test_score"])
ax.grid()
ax.set_title("Validation Performance vs L2 Regularization")
ax.set_xlabel("L2-Penalty Regularization Parameter")
ax.set_ylabel("Mean Squared Error")
fig.show()
# Let's plot prediction functions and compare coefficients for several fits
# and the target function.
pred_fns = []
x = np.sort(np.concatenate([np.arange(0,1,.001), x_train]))
name = "Target Parameter Values (i.e. Bayes Optimal)"
pred_fns.append({"name":name, "coefs":coefs_true, "preds": target_fn(x) })
l2regs = [0, grid.best_params_['l2reg'], 1]
X = featurize(x)
for l2reg in l2regs:
ridge_regression_estimator = RidgeRegression(l2reg=l2reg)
ridge_regression_estimator.fit(X_train, y_train)
name = "Ridge with L2Reg="+str(l2reg)
pred_fns.append({"name":name,
"coefs":ridge_regression_estimator.w_,
"preds": ridge_regression_estimator.predict(X) })
f = plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best")
f.show()
f = compare_parameter_vectors(pred_fns)
f.show()
coefs_confusion_matrices(coefs_true, pred_fns[1]["coefs"], 10.**np.array([-6, -3, -1]))
def main():
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = setup_problem.load_problem(lasso_data_fname)
# Generate features
X_train = featurize(x_train)#这里featurize是setup_problem.py里get_target_and_featurizer（）return的函数
X_val = featurize(x_val)
# Let's plot prediction functions and compare coefficients for several fits
# and the target function.
pred_fns = []
x = np.sort(np.concatenate([np.arange(0,1,.001), x_train]))
pred_fns.append({"name": "Target Parameter Values (i.e. Bayes Optimal)", "coefs": coefs_true, "preds": target_fn(x)})
X = featurize(x)
estimator = LinearRegression(step_size=0.001, max_num_epochs=1000)
estimator.fit(X_train, y_train)
name = "Linear regression"
pred_fns.append({"name":name, "preds": estimator.predict(X) })
plot_utils.plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best")
def main():
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(
lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
#Visualize training data
fig, ax = plt.subplots()
ax.imshow(X_train)
ax.set_title("Design Matrix: Color is Feature Value")
ax.set_xlabel("Feature Index")
ax.set_ylabel("Example Number")
plt.show(block=False)
# Compare our RidgeRegression to sklearn's.
compare_our_ridge_with_sklearn(X_train, y_train, l2_reg=1.5)
# Do hyperparameter tuning with our ridge regression
grid, results = do_grid_search_ridge(X_train, y_train, X_val, y_val)
print(results)
def main():
lasso_data_fname = "c:/Users/jack/ml_homework_2020/homework7/code/lasso_data.pickle"
#lasso_data_fname = "/Users/shunshun/github/ml_homework_2020/homework7/code/lasso_data.pickle"
#lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = setup_problem.load_problem(
lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
print(y_train[1])
print(X_train.shape, y_train.shape)
# Let's plot prediction functions and compare coefficients for several fits
# and the target function.
pred_fns = []
x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train]))
pred_fns.append({
"name": "Target Parameter Values (i.e. Bayes Optimal)",
"coefs": coefs_true,
"preds": target_fn(x)
})
X = featurize(x)
estimator = LinearRegression(step_size=0.001, max_num_epochs=100)
estimator.fit(X_train, y_train)
name = "Linear regression"
pred_fns.append({"name": name, "preds": estimator.predict(X)})
plot_utils.plot_prediction_functions(x,
pred_fns,
x_train,
y_train,
legend_loc="best")
def main():
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(
lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
#Visualize training data
fig, ax = plt.subplots()
ax.imshow(X_train)
ax.set_title("Design Matrix: Color is Feature Value")
ax.set_xlabel("Feature Index")
ax.set_ylabel("Example Number")
plt.show(block=False)
# Compare our RidgeRegression to sklearn's.
compare_our_ridge_with_sklearn(X_train, y_train, l2_reg=1.5)
# Do hyperparameter tuning with our ridge regression
grid, results = do_grid_search_ridge(X_train, y_train, X_val, y_val)
print(results)
# Plot validation performance vs regularization parameter
fig, ax = plt.subplots()
# ax.loglog(results["param_l2reg"], results["mean_test_score"])
ax.semilogx(results["param_l2reg"], results["mean_test_score"])
ax.grid()
ax.set_title("Validation Performance vs L2 Regularization")
ax.set_xlabel("L2-Penalty Regularization Parameter")
ax.set_ylabel("Mean Squared Error")
fig.show()
# Let's plot prediction functions and compare coefficients for several fits
# and the target function.
pred_fns = []
x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train]))
name = "Target Parameter Values (i.e. Bayes Optimal)"
pred_fns.append({"name": name, "coefs": coefs_true, "preds": target_fn(x)})
l2regs = [0, grid.best_params_['l2reg'], 1]
X = featurize(x)
for l2reg in l2regs:
ridge_regression_estimator = RidgeRegression(l2reg=l2reg)
ridge_regression_estimator.fit(X_train, y_train)
name = "Ridge with L2Reg=" + str(l2reg)
pred_fns.append({
"name": name,
"coefs": ridge_regression_estimator.w_,
"preds": ridge_regression_estimator.predict(X)
})
f = plot_prediction_functions(x,
pred_fns,
x_train,
y_train,
legend_loc="best")
f.show()
f = compare_parameter_vectors(pred_fns)
f.show()
##Sample code for plotting a matrix
## Note that this is a generic code for confusion matrix
## You still have to make y_true and y_pred by thresholding as per the insturctions in the question.
y_true = [1, 0, 1, 1, 0, 1]
y_pred = [0, 0, 1, 1, 0, 1]
eps = 1e-1
cnf_matrix = confusion_matrix(y_true, y_pred)
plt.figure()
plot_confusion_matrix(
cnf_matrix,
title="Confusion Matrix for $\epsilon = {}$".format(eps),
classes=["Zero", "Non-Zero"])
plt.show()
def main():
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(
lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
'''
#Visualize training data
fig, ax = plt.subplots()
ax.imshow(X_train)
ax.set_title("Design Matrix: Color is Feature Value")
ax.set_xlabel("Feature Index")
ax.set_ylabel("Example Number")
#fig.show()
# Compare our RidgeRegression to sklearn's.
#compare_our_ridge_with_sklearn(X_train, y_train, l2_reg = 1.5)
# Do hyperparameter tuning with our ridge regression
grid, results = do_grid_search_ridge(X_train, y_train, X_val, y_val)
#print(results)
# Plot validation performance vs regularization parameter
fig, ax = plt.subplots()
#ax.loglog(results["param_l2reg"], results["mean_test_score"])
ax.semilogx(results["param_l2reg"], results["mean_test_score"])
ax.grid()
ax.set_title("Validation Performance vs L2 Regularization")
ax.set_xlabel("L2-Penalty Regularization Parameter")
ax.set_ylabel("Mean Squared Error")
#fig.show()
'''
#Applying coordinate descent:
pred_fns = []
l1reg = 1.5
x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train]))
X = featurize(x)
name = "Target Parameter Values (i.e. Bayes Optimal)"
pred_fns.append({"name": name, "coefs": coefs_true, "preds": target_fn(x)})
name = "Shooting algorithm with L1Reg = 1.5"
#shooting_regression_estimator = random_coordinate_descent(X_train, y_train, l1reg)
shooting_regression_estimator = coordinate_descent(X_train, y_train, l1reg)
shooting_regression_prediction = shoot_predict(
X, shooting_regression_estimator)
#print(type(shooting_regression_estimator))
#print(shooting_regression_prediction.shape)
pred_fns.append({
"name": name,
"coefs": shooting_regression_estimator,
"preds": shooting_regression_prediction
})
#f = plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best")
#plt.show()
#f = compare_parameter_vectors(pred_fns)
#plt.show()
do_grid_search_lasso(X_train, y_train, X_val, y_val)
'''
def main():
# Load problem
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(
lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
#Visualize training data
# fig, ax = plt.subplots()
# ax.imshow(X_train)
# ax.set_title("Design Matrix: Color is Feature Value")
# ax.set_xlabel("Feature Index")
# ax.set_ylabel("Example Number")
# plt.show(block=False)
# Do hyperparameter tuning with our ridge regression
# this is done on the training and validation set
grid, results = do_grid_search_ridge(X_train, y_train, X_val, y_val)
print(results)
# Plot validation performance vs regularization parameter
fig, ax = plt.subplots()
# ax.loglog(results["param_l2reg"], results["mean_test_score"])
ax.semilogx(results["param_l2reg"], results["mean_test_score"])
ax.grid()
ax.set_title("Validation Performance vs L2 Regularization")
ax.set_xlabel("L2-Penalty Regularization Parameter")
ax.set_ylabel("Mean Squared Error")
plt.show()
# Let's plot prediction functions and compare coefficients for several fits
# and the target function.
pred_fns = []
x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train]))
name = "Target Parameter Values (i.e. Bayes Optimal)"
pred_fns.append({"name": name, "coefs": coefs_true, "preds": target_fn(x)})
l2regs = [0, grid.best_params_['l2reg'], 1]
X = featurize(x)
for l2reg in l2regs: # for every chosen regularization constant
ridge_regression_estimator = RidgeRegression(l2reg=l2reg) # fit
ridge_regression_estimator.fit(X_train, y_train)
name = "Ridge with L2Reg=" + str(l2reg)
pred_fns.append({
"name": name,
"coefs": ridge_regression_estimator.w_,
"preds": ridge_regression_estimator.predict(X)
})
# f = plot_prediction_functions(x, pred_fns, x_train, y_train, legend_loc="best")
# plt.show()
# f = compare_parameter_vectors(pred_fns)
# plt.show()
# confusion matrix for different cutoff params
cutoffs = [10**(-3), 10**(-2), 10**(-1)]
best = pred_fns[1]
ctf_fns = []
for cutoff in cutoffs:
ridge_regression_estimator = RidgeRegression()
W = [w * (abs(w) > cutoff) for w in best["coefs"]]
ridge_regression_estimator.w_ = W
name = "Ridge with cutoff=" + str(cutoff)
ctf_fns.append({
"name": name,
"coefs": W,
"preds": ridge_regression_estimator.predict(X)
})
f = plot_prediction_functions(x,
ctf_fns,
x_train,
y_train,
legend_loc="best")
plt.show()
def main():
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
print('x_train shape: ' + str(x_train.shape))
print('y_train shape: ' + str(y_train.shape))
print('X_train shape: ' + str(X_train.shape))
lmbd_max = np.max(2 * X_train.T.dot(y_train))
print('Max lambda: {:.4f}'.format(lmbd_max))
best_score = np.finfo(np.float32).max
best_estimator = None
best_l1reg = 0.0
legend = []
l1reg_costs = []
l1reg_train_costs = []
# l1reg_range = [0.0001, 0.01, .1, .5, 1, 1.5, 1.75, 2, 5, 10, 20]
l1reg_range = lmbd_max * np.power(.8, range(30))
x_plot = np.linspace(0, 1, 1000)
f1=plt.figure(1)
plt.scatter(x_train, y_train, marker='^', c='g')
for l1reg in l1reg_range:
lasso = LassoRegression(l1reg=l1reg, zero_start=False, random_coordinate=False)
lasso.fit(X_train, y_train)
# no regularization
if l1reg == 0.0001:
legend.append('Regular regression')
plt.plot(x_plot, lasso.predict(featurize(x_plot)))
score = lasso.score(X_val, y_val)
l1reg_costs.append(score)
score_train = lasso.score(X_train, y_train)
l1reg_train_costs.append(score_train)
if best_score > score:
best_score = score
best_estimator = lasso
best_l1reg = l1reg
print('l1reg: {:.4f} validation score: {:.4f} train score: {:.4f}'.format(l1reg, score, score_train))
legend.append('Best lasso regression estimation, lambda: {:.4f}'.format(best_l1reg))
plt.plot(x_plot, best_estimator.predict(featurize(x_plot)), '-r')
print('length of vector w: {:} number of non-zero elements: {:}'.format(len(best_estimator.w_), np.count_nonzero(best_estimator.w_)))
plt.legend(legend)
plt.title('Lasso regression')
plt.grid()
f1.show()
f2=plt.figure(2)
plt.plot(l1reg_costs, '-r^', l1reg_train_costs, '-g*')
plt.legend(['Test set score', 'Train set score'])
plt.grid()
f2.show()
plt.show()
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j,
i,
format(cm[i, j], 'd'),
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(
lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
#Visualize training data
fig, ax = plt.subplots()
ax.imshow(X_train)
ax.set_title("Design Matrix: Color is Feature Value")
ax.set_xlabel("Feature Index")
ax.set_ylabel("Example Number")
plt.show(block=False)
# Compare our RidgeRegression to sklearn's.
compare_our_ridge_with_sklearn(X_train, y_train, l2_reg=1.5)
def main():
if len(argv) == 2:
program, sol = argv
else:
raise RuntimeError("USAGE: python solution.py OPTION[2 or 3]")
# load data and split data
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(
lasso_data_fname)
# turn from 1D binary data to high dimensional featurized data
X_train = featurize(x_train)
X_val = featurize(x_val)
if sol == "2":
#### 2.1
# create array of possible L2Reg parameters
l2reg_search = 10.**np.arange(-6, +1, 1)
# search through l2reg_search
l2reg_opt = run_2_1(X_train,
y_train,
X_val,
y_val,
l2reg_search,
print_table=False,
PLOT=False)
#### 2.2
# x has many inputs from 0 to 1, as well as the x_train inputs, to help plotting
x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train]))
X = featurize(x)
# pred_fns is a list of dicts with "name", "coefs" and "preds"
pred_fns = []
coefs_opt = 0 #for question 2.3
# first entry: Target function
pred_fns.append({
"name": "Target",
"coefs": coefs_true,
"preds": target_fn(x)
})
l2reg_values = [0, l2reg_opt]
# next entries: prediction functions for L2Reg parameters in l2reg_values
for l2reg in l2reg_values:
ridge = RidgeRegression(l2reg=l2reg)
ridge.fit(X_train, y_train)
pred_fns.append({
"name": "Ridge with L2Reg=" + str(l2reg),
"coefs": ridge.w_,
"preds": ridge.predict(X)
})
# for question 2.3
if l2reg == l2reg_opt:
coefs_opt = ridge.w_
# with pred_fns populated, plot
# "PRED": prediction functions
# "COEF": coefficients
plots = ["PRED", "COEF"]
#plots=[]
run_2_2(x, x_train, y_train, pred_fns, plot=plots)
#### 2.3
epsilon = []
#epsilon = [1e-6, 1e-3, 1e-2, 5e-2, 1e-1, 5e-1]
for e in epsilon:
run_2_3(coefs_true, coefs_opt, epsilon=e)
if sol == "3":
#### 3.2 - experiment with Lasso
# Found that start="RR", order="cyclic", epsilon=1e-8 works MARGINALLY better
#run_3_2(X_train, y_train, X_val, y_val, l1reg=1, epsilons=[1e-8, 1e-3])
#### 3.3
#### Part a: find optimal l1reg
# create array of possible L1Reg parameters
#l1reg_search = 10.**np.arange(-6, 2, 1)
# search through l1reg_search
#l1reg_opt = run_3_3_a(X_train, y_train, X_val, y_val, l1reg_search)
l1reg_opt = 1.0 # found from above
#### 3.3
#### Part b: plot corresponding prediction function
# x has many inputs from 0 to 1, as well as the x_train inputs, to help plotting
x = np.sort(np.concatenate([np.arange(0, 1, .001), x_train]))
X = featurize(x)
# pred_fns is a list of dicts with "name", "coefs" and "preds"
pred_fns = []
# first entry: Target function
pred_fns.append({
"name": "Target",
"coefs": coefs_true,
"preds": target_fn(x)
})
lasso = LassoRegression(l1reg=l1reg_opt)
lasso.shooting_alg(X_train, y_train)
pred_fns.append({
"name": "Ridge with L1Reg=" + str(l1reg_opt),
"coefs": lasso.w,
"preds": lasso.predict(X)
})
# with pred_fns populated, plot
# "PRED": prediction functions
# "COEF": coefficients
run_3_3_b(x, x_train, y_train, pred_fns, plot=[])
run_3_4(X_train, y_train, X_val, y_val, p=0.8)
def main():
lasso_data_fname = "lasso_data.pickle"
x_train, y_train, x_val, y_val, target_fn, coefs_true, featurize = load_problem(
lasso_data_fname)
# Generate features
X_train = featurize(x_train)
X_val = featurize(x_val)
# print('Original: ' + str(x_train[0]) + ' Featurized: ' + str(X_train[0]))
print('Featurized shape: ' + str(X_train[0].shape))
# Visualize data
f1 = plt.figure(1)
plt.subplot(2, 1, 1)
plt.subplots_adjust(hspace=0.5)
legend = []
l2reg_range = np.concatenate(
(np.linspace(0, 1, 8, endpoint=False), np.linspace(1, 5, 4)))
best_score = np.finfo(np.float32).max
best_estimator = None
base_weights = np.zeros(X_train[0].shape[0])
best_l2reg = 0
l2reg_costs = []
x_plot = np.linspace(0, 1, 1000)
for l2reg in l2reg_range:
ridge_regression_estimator = MyRidge(l2reg=l2reg)
ridge_regression_estimator.fit(X_train, y_train)
if l2reg == 0:
y_plot = ridge_regression_estimator.predict(featurize(x_plot))
base_weights = ridge_regression_estimator.w_
plt.plot(x_plot, y_plot)
legend.append('l2reg {:.4}'.format(l2reg))
score = ridge_regression_estimator.score(X_val, y_val)
l2reg_costs.append(score)
score_train = ridge_regression_estimator.score(X_train, y_train)
if best_score > score:
best_score = score
best_l2reg = l2reg
best_estimator = ridge_regression_estimator
print('l2reg: {:.2f} validation score: {:.4f} train score: {:.4f}'.
format(l2reg, score, score_train))
legend.append('Best ridge estimation')
plt.plot(x_plot, best_estimator.predict(featurize(x_plot)), '-r')
legend.append('Bayes estimation')
plt.plot(x_plot, target_fn(x_plot), '-c')
legend.append('data')
plt.scatter(x_train, y_train, marker='^', c='g')
plt.legend(legend)
plt.title('Ridge regression')
plt.grid()
# print('=>' + str(len(legend)))
# Visualize cost vs l2reg
plt.subplot(2, 1, 2)
plt.title('Ridge regression cost')
plt.grid()
plt.plot(l2reg_range, l2reg_costs, '-rx')
f1.show()
# Visualize weights
f2 = plt.figure(2)
plt.subplots_adjust(hspace=0.5)
plt.subplot(3, 1, 1)
plt.grid()
plt.title('Weights without regularization')
plt.bar(range(X_train[0].shape[0]), base_weights)
plt.subplot(3, 1, 2)
plt.grid()
plt.title('Best regularized weights')
plt.bar(range(X_train[0].shape[0]), ridge_regression_estimator.w_)
plt.subplot(3, 1, 3)
plt.grid()
plt.title('Bayes regularized weights')
plt.bar(range(X_train[0].shape[0]), coefs_true)
f2.show()
print('Best performance with l2reg: {:.4f} score: {:.4f}'.format(
best_l2reg, best_score))
print('length of vector w: {:} number of non-zero elements: {:}'.format(
len(best_estimator.w_), np.count_nonzero(best_estimator.w_)))
best_w_adjusted = np.copy(best_estimator.w_)
print(
'length of vector w: {:} number of non-zero elements with tolerance 1e-6: {:}'
.format(len(best_estimator.w_),
np.count_nonzero(best_w_adjusted[best_w_adjusted > 1e-6])))
plt.show()
