Example #1
0
 def test_validation_curve_poly(self):
     highest_degree = 8
     X_poly = map_poly_features(self.X, highest_degree)
     X_poly, mu, sigma = feature_normalization(X_poly)
     Xval_poly = map_poly_features(self.Xval, highest_degree)
     Xval_poly = (Xval_poly - mu) / sigma
     lamdas = np.array([0,0.001,0.003,0.01,0.03,0.1,0.3,1,3,10])
     
     error_train, error_val = validation_curve(X_poly, self.y, Xval_poly, self.yval, lamdas)
     plt.xlabel('lamda')
     plt.ylabel("Error")
     plt.plot( lamdas, error_train, label='Train')
     plt.plot( lamdas, error_val, label='Validation')
     plt.legend()
     plt.show()
Example #2
0
    def test_validation_curve_poly(self):
        highest_degree = 8
        X_poly = map_poly_features(self.X, highest_degree)
        X_poly, mu, sigma = feature_normalization(X_poly)
        Xval_poly = map_poly_features(self.Xval, highest_degree)
        Xval_poly = (Xval_poly - mu) / sigma
        lamdas = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10])

        error_train, error_val = validation_curve(X_poly, self.y, Xval_poly,
                                                  self.yval, lamdas)
        plt.xlabel('lamda')
        plt.ylabel("Error")
        plt.plot(lamdas, error_train, label='Train')
        plt.plot(lamdas, error_val, label='Validation')
        plt.legend()
        plt.show()
Example #3
0
def set_validation_score_and_curve(classifier, x_train, y_train, x_cv, y_cv,
                                   x_test, y_test, parameter, parameter_values,
                                   classifier_class):
    data_x, data_y = np.concatenate((x_train, x_cv)), np.concatenate(
        (y_train, y_cv))

    train_indices = np.full((x_train.shape[0], ), -1, dtype=int)
    cv_indices = np.full((x_cv.shape[0], ), 0, dtype=int)
    ps = PredefinedSplit(np.append(train_indices, cv_indices))

    estimators_svm, train_scores_svm, valid_scores_svm, fit_times, score_times = validation_curve(
        classifier,
        data_x,
        data_y.ravel(),
        parameter,
        parameter_values,
        cv=ps,
        n_jobs=-1)

    classifier_list = []
    for i in range(estimators_svm.shape[0]):
        classifier = estimators_svm[i]
        train_score = train_scores_svm[i]
        valid_score = valid_scores_svm[i]
        fit_time = fit_times[i]
        score_time = score_times[i]

        c: Classifier = eval(classifier_class)
        c.update_params(train_score=train_score,
                        valid_score=valid_score,
                        fit_time=fit_time,
                        score_time=score_time)

        c.accuracy(x_test, y_test, "Test set Accuracy")
        c.confusion_matrix(x_cv, y_cv, "CV confusion matrix")
        c.confusion_matrix(x_test, y_test, "Test confusion matrix")
        c.save_classifier()

        classifier_list.append(c)

    plot_data(classifier_list)
Example #4
0
reg = 1.0
error_train,error_val = utils.averaged_learning_curve(XX_poly,y,XX_poly_val,yval,reg)
plot_utils.plot_learning_curve(error_train,error_val,reg)
plt.savefig('fig11.pdf')


#######################################################################
## =========== Part 6: Selecting Lambda                ===============#
#######################################################################

# now implement the validation_curve function in utils.py
# this function helps in determining the best lambda using a
# a validation set
# The script will now run your function and plot the figure in fig12.pdf

reg_vec, error_train, error_val = utils.validation_curve(XX_poly,y,XX_poly_val,yval)
plot_utils.plot_lambda_selection(reg_vec,error_train,error_val)
plt.savefig('fig12.pdf')
<<<<<<< HEAD

#Problem 3.2.A6
# lambda = 1.0 gives the best model (from fig12.pdf)
reg = 1.0
reglinear_reg3 = RegularizedLinearReg_SquaredLoss()
theta_opt = reglinear_reg3.train(XX_poly,y,reg,num_iters=1000)
error_test = reglinear_reg3.loss(theta_opt,XX_poly_test,ytest,0.0)
print 'Theta at lambda = 1.0 (problem3.2.A6) is ', theta_opt
print 'Error at lambda = 1.0 (problem3.2.A6) is ', error_test
=======
>>>>>>> 89dd6a53aa0ff700b713b57c5d8d001424557b1d
Example #5
0
                           X_train_norm.T]).T
XX_test_norm = np.vstack([np.ones((X_test_norm.shape[0], )), X_test_norm.T]).T
XX_val_norm = np.vstack([np.ones((X_val_norm.shape[0], )), X_val_norm.T]).T

# lambda = 0
reglinear_reg1 = RegularizedLinearReg_SquaredLoss()
theta_opt0 = reglinear_reg1.train(XX_train_norm,
                                  y_train,
                                  reg=0.0,
                                  num_iters=1000)
print 'Theta at lambda = 0 is ', theta_opt0
print 'Test error of the best linear model with lambda = 0 is: ' + str(
    reglinear_reg1.loss(theta_opt0, XX_test_norm, y_test, 0))

# linear
reg_vec, error_train, error_val = utils.validation_curve(
    XX_train_norm, y_train, XX_val_norm, y_val)
plot_utils.plot_lambda_selection(reg_vec, error_train, error_val)
plt.savefig('liner' + '.png')
plt.show()

reg_best = reg_vec[error_val.tolist().index(min(error_val))]
theta_opt1 = reglinear_reg1.train(XX_train_norm,
                                  y_train,
                                  reg=reg_best,
                                  num_iters=10000)
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~linear model~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'the best lambda is: ', reg_best
print 'When lambda=' + str(
    reg_best) + ', test error of the linear model is: ' + str(
        reglinear_reg1.loss(theta_opt1, XX_test_norm, y_test, 0))
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'