def output(partId):
# Random Test Cases
X = np.vstack([
np.ones(10),
np.sin(np.arange(1, 15, 1.5)),
np.cos(np.arange(1, 15, 1.5))
]).T
y = np.sin(np.arange(1, 31, 3))
Xval = np.vstack([
np.ones(10),
np.sin(np.arange(0, 14, 1.5)),
np.cos(np.arange(0, 14, 1.5))
]).T
yval = np.sin(np.arange(1, 11))
if partId == '1':
J, _ = linearRegCostFunction(X, y, np.array([0.1, 0.2, 0.3]), 0.5)
out = formatter('%0.5f ', J)
elif partId == '2':
J, grad = linearRegCostFunction(X, y, np.array([0.1, 0.2, 0.3]), 0.5)
out = formatter('%0.5f ', grad)
elif partId == '3':
error_train, error_val = learningCurve(X, y, Xval, yval, 1)
out = formatter(
'%0.5f ', np.concatenate([error_train.ravel(),
error_val.ravel()]))
elif partId == '4':
X_poly = polyFeatures(X[1, :].T, 8)
out = formatter('%0.5f ', X_poly)
elif partId == '5':
lambda_vec, error_train, error_val = validationCurve(X, y, Xval, yval)
out = formatter(
'%0.5f ',
np.concatenate(
[lambda_vec.ravel(),
error_train.ravel(),
error_val.ravel()]))
return out
def output(partId):
# Random Test Cases
X = column_stack([ones(10), sin(arange(1,15,1.5)), cos(arange(1,15,1.5))])
y = sin(arange(1,30,3))
Xval = column_stack([ones(10), sin(arange(0,14,1.5)), cos(arange(0,14,1.5))])
yval = sin(arange(1,11))
if partId == '1':
J, _ = linearRegCostFunction(X, y, hstack([0.1, 0.2, 0.3]), 0.5)
return sprintf('%0.5f ', J)
elif partId == '2':
J, grad = linearRegCostFunction(X, y, hstack([0.1, 0.2, 0.3]), 0.5)
return sprintf('%0.5f ', grad)
elif partId == '3':
error_train, error_val = \
learningCurve(X, y, Xval, yval, 1);
return sprintf('%0.5f ', vstack([error_train, error_val]))
elif partId == '4':
X_poly = polyFeatures(X[1,:], 8)
return sprintf('%0.5f ', X_poly)
elif partId == '5':
lambda_vec, error_train, error_val = \
validationCurve(X, y, Xval, yval)
return sprintf('%0.5f ', \
vstack([lambda_vec, error_train, error_val]))
# Plot fit over the data
plt.plot(X, np.hstack((np.ones((m, 1)), X)).dot(theta).T, '-', linewidth=2)
input('Program paused. Press enter to continue.')
# =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
lmbda = 0
error_train, error_val = learningCurve(
np.hstack((np.ones((m, 1)), X)), y,
np.hstack((np.ones((Xval.shape[0], 1)), Xval)), yval, lmbda)
fig, ax = plt.subplots()
ax.plot(np.arange(m), error_train, label='Train')
ax.plot(np.arange(m), error_val, label='Cross Validation')
ax.set_title('Learning curve for linear regression')
ax.set_xlabel('Number of training examples')
ax.set_ylabel('Error')
ax.axis([0, 13, 0, 150])
ax.legend(numpoints=1)
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
print(' \t%d\t\t%f\t%f' % (i + 1, error_train[i], error_val[i]))
theta = trainLinearReg(np.hstack((np.ones((m, 1)), X)), y, lmbda)
# Plot fit over the data
plt.plot(X, np.hstack((np.ones((m, 1)), X)).dot(theta).T, '-', linewidth=2)
input('Program paused. Press enter to continue.')
# =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
lmbda = 0
error_train, error_val = learningCurve(np.hstack((np.ones((m, 1)), X)), y,
np.hstack((np.ones((Xval.shape[0], 1)), Xval)), yval, lmbda)
fig, ax = plt.subplots()
ax.plot(np.arange(m), error_train, label='Train')
ax.plot(np.arange(m), error_val, label='Cross Validation')
ax.set_title('Learning curve for linear regression')
ax.set_xlabel('Number of training examples')
ax.set_ylabel('Error')
ax.axis([0, 13, 0, 150])
ax.legend(numpoints=1)
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
print(' \t%d\t\t%f\t%f' % (i + 1, error_train[i], error_val[i]))
input('Program paused. Press enter to continue.')
lamda = 0
theta = trainLinearReg(X_ones, y, lamda)
plt.scatter(X, y, marker='x', c='r', s=60)
plt.plot(X, X_ones.dot(theta))
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.show()
# =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
lamda = 0
error_train, error_val = learningCurve(X_ones, y, Xval_ones, yval, lamda)
plt.plot(range(1, m + 1), error_train, label='Train')
plt.plot(range(1, m + 1), error_val, label='Cross Validation')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.legend()
plt.show()
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
print(' \t%d\t\t%f\t%f' % (i + 1, error_train[i], error_val[i]))
# =========== Part 6: Feature Mapping for Polynomial Regression =============
# One solution to this is to use polynomial regression. You should now
# complete polyFeatures to map each example into its powers
#display
plt.close()
plt.plot(X, y, 'rx', markersize=10, linewidth=1.5)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.hold(True)
plt.plot(X, np.dot(np.column_stack((np.ones((m,1)), X)), theta), '--', linewidth=2)
plt.show()
raw_input('Program paused. Press enter to continue.\n')
plt.hold(False)
## =========== Part 5: Learning Curve for Linear Regression =============
lambda_val = 0
error_train, error_val = lc.learningCurve(np.column_stack((np.ones((m,1)), X)), y, np.column_stack((np.ones((Xval.shape[0], 1)), Xval)), yval, lambda_val)
# resets plot
plt.close()
p1, p2 = plt.plot(range(m), error_train, range(m), error_val)
plt.title('Learning curve for linear regression')
plt.legend((p1, p2), ('Train', 'Cross Validation'), numpoints=1, handlelength=0.5)
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.show()
plt.ylabel('Water flowing out of the dam (y)') # Set the y-axis label
plt.xlabel('Change in water level (x)') # Set the x-axis label
plt.plot(X, np.column_stack((np.ones(m), X)).dot(theta), '--', lw=2.0)
raw_input("Program paused. Press Enter to continue...")
## =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
Lambda = 0
error_train, error_val = learningCurve(np.column_stack((np.ones(m), X)), y,
np.column_stack((np.ones(Xval.shape[0]), Xval)), yval, Lambda)
plt.figure()
plt.plot(range(m), error_train, color='b', lw=0.5, label='Train')
plt.plot(range(m), error_val, color='r', lw=0.5, label='Cross Validation')
plt.title('Learning curve for linear regression')
plt.legend()
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.xlim(0, 13)
plt.ylim(0, 150)
plt.legend(loc='upper right', shadow=True, fontsize='x-large', numpoints=1)
print 'Training Examples\tTrain Error\tCross Validation Error'
for i in range(m):
print ' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i])
degree = 1 # degree of the polynomial
theta = initialize(degree)
# checking code by printing values of cost, derivatives and plotting
# the linear regression
# we expect 303.951 for cost
# then [ -15.30, 598.250 ] for gradient
print "Cost for initial theta = [1,1]: ", \
costFunction.computeCost(theta,X,y,lam)
print "Gradient for initial theta = [1,1]: ", \
costFunction.computeDeriv(theta,X,y,lam)
theta = regression(theta,X,y,lam)
plotData.plotTheta(X,y,theta)
# Computing the learning curve for linear regression
(error_train,error_cv) = learningCurve(theta,X,y,Xcv,ycv,lam)
plotData.plotError(error_train,error_cv)
# Unregularized polynomial regression for a degree 8 polynomial
degree = 8
theta = initialize(degree)
X_poly = polyFeatures(X,degree)
(X_norm,mu,sigma) = featureNormalization(X_poly)
theta = regression(theta,X_norm,y,lam)
#plotData.plotNorm(X,y,theta,mu,sigma)
# Learning curve for a degree 8 polynomial
Xcv_norm = polyFeatures(Xcv,degree)
for ii in range(degree):
Xcv_norm[:,ii] -= mu[ii]
Xcv_norm[:,ii] /= sigma[ii]
(error_train,error_cv) = learningCurve(theta,X_norm,y,Xcv_norm,ycv,lam)
plt.plot(X, pred, '--r', lw=2.0)
plt.grid()
plt.show()
# %% =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
Lambda = 0
X_stack = np.column_stack((np.ones(m), X))
Xval_stack = np.column_stack((np.ones(Xval.shape[0]), Xval))
error_train, error_val = learningCurve(X_stack, y, Xval_stack, yval, Lambda)
# display
plt.figure()
plt.plot(range(m), error_train, color='b', lw=2, label='Train')
plt.plot(range(m), error_val, color='r', lw=2, label='Cross Validation')
plt.title('Learning curve for linear regression')
plt.grid()
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.xlim(0, 13)
plt.ylim(0, 150)
plt.legend(loc='upper right', shadow=True, fontsize='x-large', numpoints=1)
plt.show()
print('Training Examples\tTrain Error\tCross Validation Error')
'''用线性函数拟合样本'''
n = X.shape[1]
m = X.shape[0]
# fmin输出的theta是(n,)要正确运算，需reshpe 成(n,1)
theta = line_trainLinearReg(theta_init, X, y, lmda)
theta = theta.reshape(n, 1)
a = [-50, 40]
b = [theta[0] - 50 * theta[1], theta[0] + 40 * theta[1]]
plt.figure(0)
plt.plot(a, b, 'b-', lw=1) # 绘制直线图
plt.scatter(X1, y, c='red', marker='x', s=20)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.show()
'''绘制误差曲线，观察是否过拟合欠拟合'''
J_train, J_val = learningCurve(X, Xval, y, yval, theta_init, lmda)
plt.figure(1)
plt.plot(range(m), J_train, range(m), J_val)
plt.title('Learning curve for linear regression')
plt.xlabel('m')
plt.ylabel('Error')
plt.show()
"""应用到训练集、测试集、交叉验证集"""
'''增加特征数'''
X1_pol = polyFeatures(X1, 8)
Xtest1_pol = polyFeatures(Xtest, 8)
Xval1_pol = polyFeatures(Xval1, 8)
"""特征归一化"""
X2_pol, X_pol_mean, X_pol_std = feature_normalize(X1_pol)
# X2test_pol, Xtest_pol_mean, Xtest_pol_std = feature_normalize(Xtest1_pol)
# X2val_pol, Xval_pol_mean, Xval_pol_std = feature_normalize(Xval1_pol)
plt.ylabel('Water flowing out of the dam (y)')
plt.show()
input('Program paused. Press enter to continue.\n')
# =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- Figure 3 in ex5.pdf
lambda_par = 0
error_train, error_val = \
learningCurve(np.concatenate((np.ones(m).reshape(m, 1), X), axis=1), y,
np.concatenate(
(np.ones(Xval.shape[0]).reshape(Xval.shape[0], 1), Xval), axis=1), yval,
lambda_par)
plt.plot(range(1, m+1), error_train, range(1, m+1), error_val)
plt.title('Learning curve for linear regression')
plt.legend(['Train', 'Cross Validation'])
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.axis([0, 13, 0, 150])
plt.show()
print('# Training Examples\tTrain Error\tCross Validation Error\n')
for i in range(m):
print(' \t{}\t\t{:.6f}\t{:.6f}\n'.format(
i, float(error_train[i]), float(error_val[i])))
theta = trainLinearReg(_X, y, _lambda)
# Plot fit over the data
plt.plot(X, np.dot(_X, theta), '--', linewidth=2)
input('Program paused. Press enter to continue.\n')
# # =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
_lambda = 0
error_train, error_val = learningCurve(_X, y, _Xval, yval, _lambda)
plt.figure()
plt.plot(np.arange(m), error_train, np.arange(m), error_val)
plt.title('Learning curve for linear regression')
plt.legend(['Train', 'Cross Validation'])
plt.xlabel('Number of training examples')
plt.ylabel('Error')
print('# Training Examples\tTrain Error\tCross Validation Error\n')
for i in range(m):
print(' \t%d\t\t%f\t%f\n' % (i, error_train[i], error_val[i]))
input('Program paused. Press enter to continue.\n')
# # =========== Part 6: Feature Mapping for Polynomial Regression =============
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
y_new = ones_X.dot(theta)
plt.plot(X, y_new, '--')
plt.show()
'''
%% =========== Part 5: Learning Curve for Linear Regression =============
% Next, you should implement the learningCurve function.
%
% Write Up Note: Since the model is underfitting the data, we expect to
% see a graph with "high bias" -- slide 8 in ML-advice.pdf
%
'''
lambda_ = 0
error_train, error_val = learningCurve(ones_X, y, ones_Xval, yval, lambda_)
plt.plot(range(0,m), error_train, range(0,m), error_val)
plt.title('Learning curve for linear regression')
#plt.legend('Train', 'Cross Validation')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.axis([1,13,1,150])
print('# Training Examples\tTrain Error\tCross Validation Error\n')
for i in range(0,m):
print(error_train[i], error_val[i])
plt.show()
'''
%% =========== Part 6: Feature Mapping for Polynomial Regression =============
# Train linear regression with lambda = 0
xlambda = 0
theta = tLR.trainLinearReg(np.c_[(np.ones([m,1]),X)],y,xlambda)
# Plot fit over the data
plt = plot.plotNormal(X,y)
y_pred = np.dot(np.c_[(np.ones([m, 1]),X)],theta)
plt.plot(X, y_pred, color='b')
plt.show()
# =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
# Write Up Note: Since the model is underfitting the data, we expect to see a graph with "high bias" -- Figure 3 in ex5.pdf
xlambda = 0
error_train,error_val = lC.learningCurve(np.c_[(np.ones([m,1]),X)],y,np.c_[np.ones([Xval.shape[0],1]),Xval],yval,xlambda)
# plot the learning curve
plt = plot.plotLC(np.arange(1,m+1), error_train, np.arange(1,m+1), error_val)
plt.show()
# print the contrast between training error and CV error
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(0,m):
print(' \t%d\t\t%.4f\t%.4f'%(i,error_train[i],error_val[i]))
input('Program paused. Press enter to continue.\n')
## =========== Part 6: Feature Mapping for Polynomial Regression =============
# One solution to this is to use polynomial regression. You should now
# complete polyFeatures to map each example into its powers
""" =========== Part 5: Learning Curve for Linear Regression =============
Next, you should implement the learningCurve function.
Write Up Note: Since the model is underfitting the data, we expect to
see a graph with "high bias" -- Figure 3 in ex5.pdf
"""
reg_lambda = 0
new__training_input = np.c_[np.ones((m, 1)), X]
new__validation_input = np.c_[np.ones((Xval.shape[0], 1)), Xval]
error_train, error_val = learningCurve(new__training_input, y, new__validation_input, yval, reg_lambda)
plt.figure(2) #new window
train_plot, = plt.plot(np.arange(1, m), error_train[1:m], linestyle='solid', color='blue',label="Train")
val_plot, = plt.plot(np.arange(1, m), error_val[1:m], linestyle='solid', color='red', label='Cross Validation')
plt.title('Learning curve for linear regression')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.legend(handles=[train_plot, val_plot])
plt.axis([0, 13, 0, 150])
plt.draw()
plt.show(block = False)
print('# Training Examples\tTrain Error\tCross Validation Error\n');
for i in range(m):
print(' \t{}\t\t{}\t{}\n'.format(i, error_train[i], error_val[i]))
fig.show()
print 'Program paused. Press enter to continue.'
raw_input()
## =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
lambda_ = 0.
error_train, error_val = \
learningCurve(addOnes(X), y, addOnes(Xval), yval, lambda_)
fig = figure()
plot(arange(m)+1, error_train, arange(m)+1, error_val);
title('Learning curve for linear regression')
legend(('Train', 'Cross Validation'))
xlabel('Number of training examples')
ylabel('Error')
axis([0, 13, 0, 150])
fig.show()
print '# Training Examples\tTrain Error\tCross Validation Error'
for i in range(m):
print ' \t%d\t\t%f\t%f' % (i+1, error_train[i], error_val[i])
print 'Program paused. Press enter to continue.'
plt.plot(X, y, 'rx')
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
y_new = ones_X.dot(theta)
plt.plot(X, y_new, '--')
plt.show()
'''
%% =========== Part 5: Learning Curve for Linear Regression =============
% Next, you should implement the learningCurve function.
%
% Write Up Note: Since the model is underfitting the data, we expect to
% see a graph with "high bias" -- slide 8 in ML-advice.pdf
%
'''
lambda_ = 0
error_train, error_val = learningCurve(ones_X, y, ones_Xval, yval, lambda_)
plt.plot(range(0, m), error_train, range(0, m), error_val)
plt.title('Learning curve for linear regression')
#plt.legend('Train', 'Cross Validation')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.axis([1, 13, 1, 150])
print('# Training Examples\tTrain Error\tCross Validation Error\n')
for i in range(0, m):
print(error_train[i], error_val[i])
plt.show()
'''
%% =========== Part 6: Feature Mapping for Polynomial Regression =============
% One solution to this is to use polynomial regression. You should now
Xpoly, mu, sigma = featureNormalize(Xpoly) Xpoly = np.hstack((ones, Xpoly)) mtest = len(ytest) ones = np.ones((mtest,1)) Xtest = np.hstack((ones, Xtest)) X_poly_test = polyFeatures(Xtest,p) X_poly_test -= mu X_poly_test /= sigma X_poly_test = np.hstack((ones, X_poly_test)) mval = len(yval ) ones = np.ones((mval,1)) Xval = np.hstack((ones, Xval )) X_poly_val = polyFeatures(Xval,p) X_poly_val -= mu X_poly_val /= sigma X_poly_val = np.hstack((ones, X_poly_val )) lam = 0 thetaOpt = trainLinearReg(Xpoly,y,lam) polyfit(np.min(Xpoly),np.max(Xpoly),mu, sigma, thetaOpt, p) lam =0 learningCurve(Xpoly, y, X_poly_val, yval, lam) plt.show()
from learningCurve import learningCurve
import matplotlib.pyplot as plt
import scipy.io as spio
mat = spio.loadmat('ex5data1.mat', squeeze_me=True)
X = mat['X']
y = mat['y']
Xtest = mat['Xtest']
ytest = mat['ytest']
Xval = mat['Xval']
yval = mat['yval']
X_ones = np.c_[np.ones(X.shape[0]), X]
Xval_ones = np.c_[np.ones(Xval.shape[0]), Xval]
erro_treino, erro_validacao = learningCurve(X_ones, y, Xval_ones, yval, 0)
plt.plot(np.arange(1, 13), erro_treino, label='Train', color='purple')
plt.plot(np.arange(1, 13),
erro_validacao,
label='Cross Validation',
color='green')
plt.title('Learning curve for linear regression')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.legend()
plt.show()
def ex5():
## Machine Learning Online Class
# Exercise 5 | Regularized Linear Regression and Bias-Variance
#
# Instructions
# ------------
#
# This file contains code that helps you get started on the
# exercise. You will need to complete the following functions:
#
# linearRegCostFunction.m
# learningCurve.m
# validationCurve.m
#
# For this exercise, you will not need to change any code in this file,
# or any other files other than those mentioned above.
#
## Initialization
#clear ; close all; clc
## =========== Part 1: Loading and Visualizing Data =============
# We start the exercise by first loading and visualizing the dataset.
# The following code will load the dataset into your environment and plot
# the data.
#
# Load Training Data
print('Loading and Visualizing Data ...')
# Load from ex5data1:
# You will have X, y, Xval, yval, Xtest, ytest in your environment
mat = scipy.io.loadmat('ex5data1.mat')
X = mat['X']
y = mat['y'].ravel()
Xval = mat['Xval']
yval = mat['yval'].ravel()
Xtest = mat['Xtest']
ytest = mat['ytest'].ravel()
# m = Number of examples
m = X.shape[0]
# Plot training data
plt.plot(X, y, marker='x', linestyle='None', ms=10, lw=1.5)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.savefig('figure1.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 2: Regularized Linear Regression Cost =============
# You should now implement the cost function for regularized linear
# regression.
#
theta = np.array([1, 1])
J, _ = linearRegCostFunction(np.concatenate([np.ones((m, 1)), X], axis=1),
y, theta, 1)
print(
'Cost at theta = [1 ; 1]: %f \n(this value should be about 303.993192)'
% J)
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 3: Regularized Linear Regression Gradient =============
# You should now implement the gradient for regularized linear
# regression.
#
theta = np.array([1, 1])
J, grad = linearRegCostFunction(
np.concatenate([np.ones((m, 1)), X], axis=1), y, theta, 1)
print(
'Gradient at theta = [1 ; 1]: [%f; %f] \n(this value should be about [-15.303016; 598.250744])'
% (grad[0], grad[1]))
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 4: Train Linear Regression =============
# Once you have implemented the cost and gradient correctly, the
# trainLinearReg function will use your cost function to train
# regularized linear regression.
#
# Write Up Note: The data is non-linear, so this will not give a great
# fit.
#
fig = plt.figure()
# Train linear regression with lambda = 0
lambda_value = 0
theta = trainLinearReg(np.concatenate([np.ones((m, 1)), X], axis=1), y,
lambda_value)
# Plot fit over the data
plt.plot(X, y, marker='x', linestyle='None', ms=10, lw=1.5)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.plot(X,
np.dot(np.concatenate([np.ones((m, 1)), X], axis=1), theta),
'--',
lw=2)
plt.savefig('figure2.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
fig = plt.figure()
lambda_value = 0
error_train, error_val = learningCurve(
np.concatenate([np.ones((m, 1)), X], axis=1), y,
np.concatenate([np.ones((yval.size, 1)), Xval], axis=1), yval,
lambda_value)
plt.plot(np.arange(1, m + 1), error_train, np.arange(1, m + 1), error_val)
plt.title('Learning curve for linear regression')
plt.legend(['Train', 'Cross Validation'])
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.axis([0, 13, 0, 150])
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
print(' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i]))
plt.savefig('figure3.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 6: Feature Mapping for Polynomial Regression =============
# One solution to this is to use polynomial regression. You should now
# complete polyFeatures to map each example into its powers
#
p = 8
# Map X onto Polynomial Features and Normalize
X_poly = polyFeatures(X, p)
X_poly, mu, sigma = featureNormalize(X_poly) # Normalize
X_poly = np.concatenate([np.ones((m, 1)), X_poly], axis=1) # Add Ones
# Map X_poly_test and normalize (using mu and sigma)
X_poly_test = polyFeatures(Xtest, p)
X_poly_test -= mu
X_poly_test /= sigma
X_poly_test = np.concatenate(
[np.ones((X_poly_test.shape[0], 1)), X_poly_test], axis=1) # Add Ones
# Map X_poly_val and normalize (using mu and sigma)
X_poly_val = polyFeatures(Xval, p)
X_poly_val -= mu
X_poly_val /= sigma
X_poly_val = np.concatenate(
[np.ones((X_poly_val.shape[0], 1)), X_poly_val], axis=1) # Add Ones
print('Normalized Training Example 1:')
print(formatter(' %f \n', X_poly[0, :]))
print('\nProgram paused. Press enter to continue.')
#pause;
## =========== Part 7: Learning Curve for Polynomial Regression =============
# Now, you will get to experiment with polynomial regression with multiple
# values of lambda. The code below runs polynomial regression with
# lambda = 0. You should try running the code with different values of
# lambda to see how the fit and learning curve change.
#
fig = plt.figure()
lambda_value = 0
theta = trainLinearReg(X_poly, y, lambda_value)
# Plot training data and fit
plt.plot(X, y, marker='x', ms=10, lw=1.5)
plotFit(np.min(X), np.max(X), mu, sigma, theta, p)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.title('Polynomial Regression Fit (lambda = %f)' % lambda_value)
plt.figure()
error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval,
lambda_value)
plt.plot(np.arange(1, 1 + m), error_train, np.arange(1, 1 + m), error_val)
plt.title('Polynomial Regression Learning Curve (lambda = %f)' %
lambda_value)
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.axis([0, 13, 0, 100])
plt.legend(['Train', 'Cross Validation'])
print('Polynomial Regression (lambda = %f)\n' % lambda_value)
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
print(' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i]))
plt.savefig('figure4.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 8: Validation for Selecting Lambda =============
# You will now implement validationCurve to test various values of
# lambda on a validation set. You will then use this to select the
# "best" lambda value.
#
fig = plt.figure()
lambda_vec, error_train, error_val = validationCurve(
X_poly, y, X_poly_val, yval)
plt.plot(lambda_vec, error_train, lambda_vec, error_val)
plt.legend(['Train', 'Cross Validation'])
plt.xlabel('lambda')
plt.ylabel('Error')
print('lambda\t\tTrain Error\tValidation Error')
for i in range(lambda_vec.size):
print(' %f\t%f\t%f' % (lambda_vec[i], error_train[i], error_val[i]))
plt.savefig('figure5.png')
print('Program paused. Press enter to continue.')
'Gradient at theta = [1 ; 1] - (this value should be about [-15.303016; 598.250744])\n',
grad)
# =========== Part 4: Train Linear Regression =============
_lambda = 0
result = trainLinearReg(np.column_stack((np.ones((m, 1)), X)), y, _lambda)
plt.plot(X, y, marker='x', linestyle='None')
plt.ylabel('Water flowing out of the dam (y)')
plt.xlabel('Change in water level (x)')
plt.xticks(np.arange(-50, 50, 10.0))
plt.plot(X, np.dot(np.column_stack((np.ones((m, 1)), X)), result.x))
plt.show()
# =========== Part 5: Learning Curve for Linear Regression =============
_lambda = 0
error_train, error_val = learningCurve(np.column_stack((np.ones((m,1)), X)), y, \
np.column_stack((np.ones((Xval.shape[0],1)), Xval)), yval, _lambda)
plt.plot(range(0, m), error_train, label="Training Error")
plt.plot(range(0, m), error_val, label="Validation Error")
plt.legend()
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.show()
print('Training Examples\tTrain Error\tCross Validation Error\n')
for i in range(0, m):
print('{:d}\t\t{:f}\t{:f}\n'.format(i + 1, float(error_train[i]),
float(error_val[i])))
# =========== Part 6: Feature Mapping for Polynomial Regression =============
p = 8
x_poly = polyFeatures(X, p)
x_poly, mu, sigma = featureNormalize(x_poly)
fit = trainLinearReg(X_train, y_train, 0)
regr = LinearRegression(fit_intercept=False)
regr.fit(X_train, y_train.ravel())
plt.plot(np.linspace(-50,40), (fit.x[0]+ (fit.x[1]*np.linspace(-50,40))), label='Scipy optimize')
plt.plot(np.linspace(-50,40), (regr.coef_[0]+ (regr.coef_[1]*np.linspace(-50,40))), label='Scikit-learn')
plt.scatter(X_train[:,1], y_train, s=50, c='r', marker='x', linewidths=1)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.legend(loc=4)
plt.show()
t_error, v_error = learningCurve(X_train, y_train, Xval, yval, 0)
plt.plot(np.arange(1,13), t_error, label='Training error')
plt.plot(np.arange(1,13), v_error, label='Validation error')
plt.title('Learning curve for linear regression')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.legend();
poly = PolynomialFeatures(degree=8)
X_train_poly = poly.fit_transform(X_train[:,1].reshape(-1,1))
regr2 = LinearRegression()
pred = np.hstack((np.ones((m, 1)), X)).dot(theta)
plt.figure()
plt.plot(X, y, linestyle='', marker='x', color='r')
plt.plot(X, pred, linestyle='--', marker='', color='b')
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
#plt.show()
# ===================== 2. Bias-variance ==============================
# ===================== 2.1 Learning curves ===========================
l = 0.0
error_train, error_val = learningCurve(np.hstack((np.ones((m, 1)), X)), y,
np.hstack((np.ones((m_val, 1)), X_val)),
y_val, l)
plt.figure()
plt.plot(range(1, m + 1), error_train, color='b', label='Train')
plt.plot(range(1, m + 1), error_val, color='r', label='Cross Validation')
plt.legend(loc='upper right')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
#plt.show()
print('# Training Examples / Train Error / Cross Validation Error')
for i in range(m):
print(' {0:<19} {1:<13.8f} {2:<.8f}'.format(i + 1, error_train[i],
error_val[i]))
plt.plot(X, np.c_[np.ones(m), X] @ theta.T, '--', lw=2.0)
plt.show(block=False)
input('Program paused. Press Enter to continue...')
# =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
lambda_ = 0
error_train, error_val = learningCurve(np.c_[np.ones(m), X], y,
np.c_[np.ones(np.size(Xval, 0)),
Xval], yval, lambda_)
plt.figure()
plt.plot(range(m), error_train, color='b', lw=0.5)
plt.plot(range(m), error_val, color='g', lw=0.5)
plt.title('Learning curve for linear regression')
plt.legend(['Train', 'Cross Validation'], loc='upper right')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.xlim(0, 13)
plt.ylim(0, 150)
plt.show(block=False)
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
lamda = 0
theta = trainLinearReg(X, y, lamda)
# Plot fit over the data
pred = X @ theta
plt.plot(X[:, 1:], pred, 'b--')
# plt.show()
## =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- Figure 3 in ex5.pdf
#
error_train, error_val = learningCurve(X, y, np.insert(Xval, 0, 1, axis=1),
yval, 0)
plt.figure(2)
plt.plot(np.arange(1, m + 1), error_train, np.arange(1, m + 1), error_val)
plt.title('Learning curve for linear regression')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.xlim((0, 13))
plt.ylim((0, 150))
print('# Training Examples\tTrain Error\tCross Validation Error\n')
for i in range(m):
print(' \t%d\t\t%f\t%f\n' % (i, error_train[i], error_val[i]))
# plt.show()
print('Часть 4. Кривые обучения для регуляризованной линейной регрессии')
# Формирование проверочных данных
X_val = data['Xval']
y_val = data['yval']
m_val = X_val.shape[0]
# Нормализация и добавление единичного признака
X_val_norm = np.divide(X_val - repmat(mu, X_val.shape[0], 1),
repmat(sigma, X_val.shape[0], 1))
X_val_norm = np.concatenate((np.ones((m_val, 1)), X_val_norm), axis=1)
# Вычисление ошибок на обучающем и проверочном множествах
error_train, error_val = learningCurve(X_norm, y, X_val_norm, y_val, alpha,
iterations, lam)
# Визуализация кривых обучения
plt.figure()
plt.plot(range(1, m + 1), error_train, 'r-', label='Ошибка обучения')
plt.plot(range(1, m + 1), error_val, 'b-', label='Ошибка проверки')
plt.legend(loc='upper right', shadow=True, fontsize=12, numpoints=1)
plt.xlabel('Число тренировочных примеров')
plt.ylabel('Ошибка')
plt.grid()
plt.show()
input('Программа остановлена. Нажмите Enter для продолжения ... \n')
# ==== Часть 5. Создание свойств для полиномиальной регрессии ====
plt.ylabel('Water flowing out of the dam (y)') # Set the y-axis label
plt.xlabel('Change in water level (x)') # Set the x-axis label
plt.plot(X, np.column_stack((np.ones(m), X)).dot(theta), '--', lw=2.0)
input("Program paused. Press Enter to continue...")
## =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
Lambda = 0
error_train, error_val = learningCurve(
np.column_stack((np.ones(m), X)), y,
np.column_stack((np.ones(Xval.shape[0]), Xval)), yval, Lambda)
plt.figure()
plt.plot(range(m), error_train, color='b', lw=0.5, label='Train')
plt.plot(range(m), error_val, color='r', lw=0.5, label='Cross Validation')
plt.title('Learning curve for linear regression')
plt.legend()
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.xlim(0, 13)
plt.ylim(0, 150)
plt.legend(loc='upper right', shadow=True, fontsize='x-large', numpoints=1)
print('Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
# Plot fit over the data
plt.plot(X, np.dot(_X, theta), '--', linewidth=2)
input('Program paused. Press enter to continue.\n')
## =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
_lambda = 0
error_train, error_val = learningCurve(_X, y, _Xval, yval, _lambda)
plt.figure()
plt.plot(np.arange(m), error_train, np.arange(m), error_val)
plt.title('Learning curve for linear regression')
plt.legend(['Train', 'Cross Validation'])
plt.xlabel('Number of training examples')
plt.ylabel('Error')
print('# Training Examples\tTrain Error\tCross Validation Error\n')
for i in range(m):
print(' \t%d\t\t%f\t%f\n'%(i, error_train[i], error_val[i]))
input('Program paused. Press enter to continue.\n')
## =========== Part 6: Feature Mapping for Polynomial Regression =============
