def test_validation_curve_poly(self):
highest_degree = 8
X_poly = map_poly_features(self.X, highest_degree)
X_poly, mu, sigma = feature_normalization(X_poly)
Xval_poly = map_poly_features(self.Xval, highest_degree)
Xval_poly = (Xval_poly - mu) / sigma
lamdas = np.array([0,0.001,0.003,0.01,0.03,0.1,0.3,1,3,10])
error_train, error_val = validation_curve(X_poly, self.y, Xval_poly, self.yval, lamdas)
plt.xlabel('lamda')
plt.ylabel("Error")
plt.plot( lamdas, error_train, label='Train')
plt.plot( lamdas, error_val, label='Validation')
plt.legend()
plt.show()
def test_validation_curve_poly(self):
highest_degree = 8
X_poly = map_poly_features(self.X, highest_degree)
X_poly, mu, sigma = feature_normalization(X_poly)
Xval_poly = map_poly_features(self.Xval, highest_degree)
Xval_poly = (Xval_poly - mu) / sigma
lamdas = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10])
error_train, error_val = validation_curve(X_poly, self.y, Xval_poly,
self.yval, lamdas)
plt.xlabel('lamda')
plt.ylabel("Error")
plt.plot(lamdas, error_train, label='Train')
plt.plot(lamdas, error_val, label='Validation')
plt.legend()
plt.show()
def set_validation_score_and_curve(classifier, x_train, y_train, x_cv, y_cv,
x_test, y_test, parameter, parameter_values,
classifier_class):
data_x, data_y = np.concatenate((x_train, x_cv)), np.concatenate(
(y_train, y_cv))
train_indices = np.full((x_train.shape[0], ), -1, dtype=int)
cv_indices = np.full((x_cv.shape[0], ), 0, dtype=int)
ps = PredefinedSplit(np.append(train_indices, cv_indices))
estimators_svm, train_scores_svm, valid_scores_svm, fit_times, score_times = validation_curve(
classifier,
data_x,
data_y.ravel(),
parameter,
parameter_values,
cv=ps,
n_jobs=-1)
classifier_list = []
for i in range(estimators_svm.shape[0]):
classifier = estimators_svm[i]
train_score = train_scores_svm[i]
valid_score = valid_scores_svm[i]
fit_time = fit_times[i]
score_time = score_times[i]
c: Classifier = eval(classifier_class)
c.update_params(train_score=train_score,
valid_score=valid_score,
fit_time=fit_time,
score_time=score_time)
c.accuracy(x_test, y_test, "Test set Accuracy")
c.confusion_matrix(x_cv, y_cv, "CV confusion matrix")
c.confusion_matrix(x_test, y_test, "Test confusion matrix")
c.save_classifier()
classifier_list.append(c)
plot_data(classifier_list)
reg = 1.0
error_train,error_val = utils.averaged_learning_curve(XX_poly,y,XX_poly_val,yval,reg)
plot_utils.plot_learning_curve(error_train,error_val,reg)
plt.savefig('fig11.pdf')
#######################################################################
## =========== Part 6: Selecting Lambda ===============#
#######################################################################
# now implement the validation_curve function in utils.py
# this function helps in determining the best lambda using a
# a validation set
# The script will now run your function and plot the figure in fig12.pdf
reg_vec, error_train, error_val = utils.validation_curve(XX_poly,y,XX_poly_val,yval)
plot_utils.plot_lambda_selection(reg_vec,error_train,error_val)
plt.savefig('fig12.pdf')
<<<<<<< HEAD
#Problem 3.2.A6
# lambda = 1.0 gives the best model (from fig12.pdf)
reg = 1.0
reglinear_reg3 = RegularizedLinearReg_SquaredLoss()
theta_opt = reglinear_reg3.train(XX_poly,y,reg,num_iters=1000)
error_test = reglinear_reg3.loss(theta_opt,XX_poly_test,ytest,0.0)
print 'Theta at lambda = 1.0 (problem3.2.A6) is ', theta_opt
print 'Error at lambda = 1.0 (problem3.2.A6) is ', error_test
=======
>>>>>>> 89dd6a53aa0ff700b713b57c5d8d001424557b1d
X_train_norm.T]).T
XX_test_norm = np.vstack([np.ones((X_test_norm.shape[0], )), X_test_norm.T]).T
XX_val_norm = np.vstack([np.ones((X_val_norm.shape[0], )), X_val_norm.T]).T
# lambda = 0
reglinear_reg1 = RegularizedLinearReg_SquaredLoss()
theta_opt0 = reglinear_reg1.train(XX_train_norm,
y_train,
reg=0.0,
num_iters=1000)
print 'Theta at lambda = 0 is ', theta_opt0
print 'Test error of the best linear model with lambda = 0 is: ' + str(
reglinear_reg1.loss(theta_opt0, XX_test_norm, y_test, 0))
# linear
reg_vec, error_train, error_val = utils.validation_curve(
XX_train_norm, y_train, XX_val_norm, y_val)
plot_utils.plot_lambda_selection(reg_vec, error_train, error_val)
plt.savefig('liner' + '.png')
plt.show()
reg_best = reg_vec[error_val.tolist().index(min(error_val))]
theta_opt1 = reglinear_reg1.train(XX_train_norm,
y_train,
reg=reg_best,
num_iters=10000)
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~linear model~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'the best lambda is: ', reg_best
print 'When lambda=' + str(
reg_best) + ', test error of the linear model is: ' + str(
reglinear_reg1.loss(theta_opt1, XX_test_norm, y_test, 0))
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'