## =========== Part 7: Learning Curve for Polynomial Regression =============
# Now, you will get to experiment with polynomial regression with multiple
# values of Lambda. The code below runs polynomial regression with
# Lambda = 0. You should try running the code with different values of
# Lambda to see how the fit and learning curve change.
#
Lambda = 0
theta = trainLinearReg(X_poly, y, Lambda, method='BFGS', maxiter=10)
# Plot training data and fit
plt.figure()
plt.scatter(X, y, marker='x', s=10, edgecolor='r', lw=1.5)
plotFit(min(X), max(X), mu, sigma, theta, p)
plt.xlabel('Change in water level (x)') # Set the y-axis label
plt.ylabel('Water flowing out of the dam (y)') # Set the x-axis label
# plt.plot(X, np.column_stack((np.ones(m), X)).dot(theta), marker='_', lw=2.0)
plt.title('Polynomial Regression Fit (Lambda = %f)' % Lambda)
error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval, Lambda)
plt.plot(range(m), error_train, label='Train')
plt.plot(range(m), error_val, label='Cross Validation')
plt.title('Polynomial Regression Learning Curve (Lambda = %f)' % Lambda)
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.xlim(0, 13)
plt.ylim(0, 150)
plt.legend()
## =========== Part 7: Learning Curve for Polynomial Regression =============
# Now, you will get to experiment with polynomial regression with multiple
# values of lambda. The code below runs polynomial regression with
# lambda = 0. You should try running the code with different values of
# lambda to see how the fit and learning curve change.
#
lamda = 0
theta = trainLinearReg(X_poly, y, lamda)
xmin = X[:, 1:].min()
xmax = X[:, 1:].max()
# Plot training data and fit
plt.figure(3)
plt.subplot(311)
plotData(X[:, 1:], y)
plotFit(xmin, xmax, mu, sigma, theta, p)
plt.text(xmin, 28, 'Polynomial Regression Fit (lambda = %.2f)' % lamda)
error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval, lamda)
plt.figure(4)
plt.subplot(121)
plt.plot(np.arange(1, m + 1), error_train, np.arange(1, m + 1), error_val)
plt.legend(['Train', 'Cross Validation'], loc=0, ncol=2)
plt.title('Polynomial Learing Curve (lambda = %.2f)' % lamda)
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.grid(True)
print('Polynomial Regression (lambda = %f)' % lamda)
print('# Training Examples\tTrain Error\tCross Validation Error\n')
X_poly_val = X_poly_val / sigma
X_poly_val = np.hstack((np.ones((X_poly_val.shape[0], 1)), X_poly_val))
print('\nNormalized Training Example 1:')
print(X_poly[1, 1:].reshape((p, 1)), '\n')
# =========== Part 7: Learning Curve for Polynomial Regression =============
# Now, you will get to experiment with polynomial regression with multiple
# values of lambda. The code below runs polynomial regression with
# lambda = 0. You should try running the code with different values of
# lambda to see how the fit and learning curve change.
lamda = 0
theta = trainLinearReg(X_poly, y, lamda)
plt.scatter(X, y, marker='x', c='r', s=60)
plotFit(np.min(X), np.max(X), mu, sigma, theta, p)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.title('Polynomial Regression Fit (lambda = %f)' % lamda)
plt.show()
error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval, lamda)
plt.plot(range(1, m + 1), error_train, label='Train')
plt.plot(range(1, m + 1), error_val, label='Cross Validation')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.legend()
plt.show()
print('Polynomial Regression (lambda = %f)' % lamda)
print('# Training Examples\tTrain Error\tCross Validation Error')
## =========== Part 7: Learning Curve for Polynomial Regression =============
# Now, you will get to experiment with polynomial regression with multiple
# values of Lambda. The code below runs polynomial regression with
# Lambda = 0. You should try running the code with different values of
# Lambda to see how the fit and learning curve change.
#
Lambda = 0
theta = trainLinearReg(X_poly, y, Lambda, method='BFGS', maxiter=10)
# Plot training data and fit
plt.figure()
plt.scatter(X, y, marker='x', s=10, edgecolor='r', lw=1.5)
plotFit(min(X), max(X), mu, sigma, theta, p)
plt.xlabel('Change in water level (x)') # Set the y-axis label
plt.ylabel('Water flowing out of the dam (y)') # Set the x-axis label
# plt.plot(X, np.column_stack((np.ones(m), X)).dot(theta), marker='_', lw=2.0)
plt.title('Polynomial Regression Fit (Lambda = %f)' % Lambda)
error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval, Lambda)
plt.plot(range(m), error_train, label='Train')
plt.plot(range(m), error_val, label='Cross Validation')
plt.title('Polynomial Regression Learning Curve (Lambda = %f)' % Lambda)
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.xlim(0, 13)
plt.ylim(0, 150)
plt.legend()
def ex5():
## Machine Learning Online Class
# Exercise 5 | Regularized Linear Regression and Bias-Variance
#
# Instructions
# ------------
#
# This file contains code that helps you get started on the
# exercise. You will need to complete the following functions:
#
# linearRegCostFunction.m
# learningCurve.m
# validationCurve.m
#
# For this exercise, you will not need to change any code in this file,
# or any other files other than those mentioned above.
#
## Initialization
#clear ; close all; clc
## =========== Part 1: Loading and Visualizing Data =============
# We start the exercise by first loading and visualizing the dataset.
# The following code will load the dataset into your environment and plot
# the data.
#
# Load Training Data
print('Loading and Visualizing Data ...')
# Load from ex5data1:
# You will have X, y, Xval, yval, Xtest, ytest in your environment
mat = scipy.io.loadmat('ex5data1.mat')
X = mat['X']
y = mat['y'].ravel()
Xval = mat['Xval']
yval = mat['yval'].ravel()
Xtest = mat['Xtest']
ytest = mat['ytest'].ravel()
# m = Number of examples
m = X.shape[0]
# Plot training data
plt.plot(X, y, marker='x', linestyle='None', ms=10, lw=1.5)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.savefig('figure1.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 2: Regularized Linear Regression Cost =============
# You should now implement the cost function for regularized linear
# regression.
#
theta = np.array([1, 1])
J, _ = linearRegCostFunction(np.concatenate([np.ones((m, 1)), X], axis=1),
y, theta, 1)
print(
'Cost at theta = [1 ; 1]: %f \n(this value should be about 303.993192)'
% J)
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 3: Regularized Linear Regression Gradient =============
# You should now implement the gradient for regularized linear
# regression.
#
theta = np.array([1, 1])
J, grad = linearRegCostFunction(
np.concatenate([np.ones((m, 1)), X], axis=1), y, theta, 1)
print(
'Gradient at theta = [1 ; 1]: [%f; %f] \n(this value should be about [-15.303016; 598.250744])'
% (grad[0], grad[1]))
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 4: Train Linear Regression =============
# Once you have implemented the cost and gradient correctly, the
# trainLinearReg function will use your cost function to train
# regularized linear regression.
#
# Write Up Note: The data is non-linear, so this will not give a great
# fit.
#
fig = plt.figure()
# Train linear regression with lambda = 0
lambda_value = 0
theta = trainLinearReg(np.concatenate([np.ones((m, 1)), X], axis=1), y,
lambda_value)
# Plot fit over the data
plt.plot(X, y, marker='x', linestyle='None', ms=10, lw=1.5)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.plot(X,
np.dot(np.concatenate([np.ones((m, 1)), X], axis=1), theta),
'--',
lw=2)
plt.savefig('figure2.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
fig = plt.figure()
lambda_value = 0
error_train, error_val = learningCurve(
np.concatenate([np.ones((m, 1)), X], axis=1), y,
np.concatenate([np.ones((yval.size, 1)), Xval], axis=1), yval,
lambda_value)
plt.plot(np.arange(1, m + 1), error_train, np.arange(1, m + 1), error_val)
plt.title('Learning curve for linear regression')
plt.legend(['Train', 'Cross Validation'])
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.axis([0, 13, 0, 150])
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
print(' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i]))
plt.savefig('figure3.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 6: Feature Mapping for Polynomial Regression =============
# One solution to this is to use polynomial regression. You should now
# complete polyFeatures to map each example into its powers
#
p = 8
# Map X onto Polynomial Features and Normalize
X_poly = polyFeatures(X, p)
X_poly, mu, sigma = featureNormalize(X_poly) # Normalize
X_poly = np.concatenate([np.ones((m, 1)), X_poly], axis=1) # Add Ones
# Map X_poly_test and normalize (using mu and sigma)
X_poly_test = polyFeatures(Xtest, p)
X_poly_test -= mu
X_poly_test /= sigma
X_poly_test = np.concatenate(
[np.ones((X_poly_test.shape[0], 1)), X_poly_test], axis=1) # Add Ones
# Map X_poly_val and normalize (using mu and sigma)
X_poly_val = polyFeatures(Xval, p)
X_poly_val -= mu
X_poly_val /= sigma
X_poly_val = np.concatenate(
[np.ones((X_poly_val.shape[0], 1)), X_poly_val], axis=1) # Add Ones
print('Normalized Training Example 1:')
print(formatter(' %f \n', X_poly[0, :]))
print('\nProgram paused. Press enter to continue.')
#pause;
## =========== Part 7: Learning Curve for Polynomial Regression =============
# Now, you will get to experiment with polynomial regression with multiple
# values of lambda. The code below runs polynomial regression with
# lambda = 0. You should try running the code with different values of
# lambda to see how the fit and learning curve change.
#
fig = plt.figure()
lambda_value = 0
theta = trainLinearReg(X_poly, y, lambda_value)
# Plot training data and fit
plt.plot(X, y, marker='x', ms=10, lw=1.5)
plotFit(np.min(X), np.max(X), mu, sigma, theta, p)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.title('Polynomial Regression Fit (lambda = %f)' % lambda_value)
plt.figure()
error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval,
lambda_value)
plt.plot(np.arange(1, 1 + m), error_train, np.arange(1, 1 + m), error_val)
plt.title('Polynomial Regression Learning Curve (lambda = %f)' %
lambda_value)
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.axis([0, 13, 0, 100])
plt.legend(['Train', 'Cross Validation'])
print('Polynomial Regression (lambda = %f)\n' % lambda_value)
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
print(' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i]))
plt.savefig('figure4.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 8: Validation for Selecting Lambda =============
# You will now implement validationCurve to test various values of
# lambda on a validation set. You will then use this to select the
# "best" lambda value.
#
fig = plt.figure()
lambda_vec, error_train, error_val = validationCurve(
X_poly, y, X_poly_val, yval)
plt.plot(lambda_vec, error_train, lambda_vec, error_val)
plt.legend(['Train', 'Cross Validation'])
plt.xlabel('lambda')
plt.ylabel('Error')
print('lambda\t\tTrain Error\tValidation Error')
for i in range(lambda_vec.size):
print(' %f\t%f\t%f' % (lambda_vec[i], error_train[i], error_val[i]))
plt.savefig('figure5.png')
print('Program paused. Press enter to continue.')
# X2test_pol, Xtest_pol_mean, Xtest_pol_std = feature_normalize(Xtest1_pol)
# X2val_pol, Xval_pol_mean, Xval_pol_std = feature_normalize(Xval1_pol)
X2test_pol = (Xtest1_pol - X_pol_mean) / X_pol_std
X2val_pol = (Xval1_pol - X_pol_mean) / X_pol_std
X_pol = np.hstack((np.ones((X2_pol.shape[0])).reshape(X2_pol.shape[0],
1), X2_pol))
Xtest_pol = np.hstack((np.ones(
(X2test_pol.shape[0])).reshape(X2test_pol.shape[0], 1), X2test_pol))
Xval_pol = np.hstack((np.ones(
(X2val_pol.shape[0])).reshape(X2val_pol.shape[0], 1), X2val_pol))
"""线性回归最优化学习参数theta"""
theta_pol_init = np.zeros((9, 1))
lmda = 3
theta_pol = line_trainLinearReg(theta_pol_init, X_pol, y, lmda)
x, yfit = plotFit(min(X1), max(X1), X_pol_mean, X_pol_std, theta_pol, 8)
'''用线性函数拟合样本'''
plt.figure(2)
plt.plot(x, yfit, 'b-', lw=1) # 绘制直线图
plt.scatter(X1, y, c='red', marker='x', s=20)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.show()
'''绘制误差曲线，观察是否过拟合欠拟合'''
J_train_pol, J_val_pol = learningCurve(X_pol, Xval_pol, y, yval,
theta_pol_init, lmda)
plt.figure(3)
plt.plot(range(m), J_train_pol, range(m), J_val_pol)
plt.title('Learning curve for polynomial regression')
plt.xlabel('m')
plt.ylabel('Error')
% Now, you will get to experiment with polynomial regression with multiple
% values of lambda. The code below runs polynomial regression with
% lambda = 0. You should try running the code with different values of
% lambda to see how the fit and learning curve change.
%
'''
_lambda = 0
theta = trainLinearReg(X_poly, y, _lambda)
# Plot training data and fit
plt.figure(1)
plt.plot(X, y, 'rx')
plotFit(np.min(X), np.max(X), mu, sigma, theta, p)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.title ('Polynomial Regression Fit')
plt.figure(2)
[error_train, error_val] = learningCurve(X_poly, y, X_poly_val, yval, _lambda)
_x = np.arange(0,m)
plt.plot(_x, error_train, _x, error_val)
plt.title('Polynomial Regression Learning Curve ')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.axis([0,13,0,100])
plt.legend('Train', 'Cross Validation')
""" =========== Part 7: Learning Curve for Polynomial Regression =============
Now, you will get to experiment with polynomial regression with multiple
values of lambda. The code below runs polynomial regression with
lambda = 0. You should try running the code with different values of
lambda to see how the fit and learning curve change.
"""
reg_lambda = 3
theta = trainLinearReg(X_poly, y, reg_lambda)
# Plot training data and fit
plt.figure(3) #new window
training_data_plot, = plt.plot(X, y, color='red', marker='+', linestyle="None", markersize=30, label="Training data")
fit_plot = plotFit(np.min(X), np.max(X), mu, sigma, theta, p, label="Learned function")
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.title('Polynomial Regression Fit (lambda = {})'.format(reg_lambda))
plt.legend(handles=[training_data_plot, fit_plot])
plt.axis([-80, 80, -60, 40])
plt.draw()
plt.show(block=False)
pause()
plt.figure(4) #new window
error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval, reg_lambda)
train_plot, = plt.plot(np.arange(1, m), error_train[1:m], linestyle='solid', color='blue',label="Train")
val_plot, = plt.plot(np.arange(1, m), error_val[1:m], linestyle='solid', color='red', label='Cross Validation')
plt.title('Polynomial Regression Learning Curve (lambda = {})'.format(reg_lambda))
