def polynomialDegreeCurve(X, y, Xval, yval, reg_lambda):
"""Error cruve in function of degree of polynimal d
"""
dimensions = np.arange(1, 80).reshape(-1, 1)
# You need to return these variables correctly.
error_train = np.zeros((len(dimensions), 1))
error_val = np.zeros((len(dimensions), 1))
m_train_set = X.shape[0]
m_val_set = Xval.shape[0]
for i in range(len(dimensions)):
dimension = dimensions[i]
X_poly = polyFeatures(X, dimension)
X_poly, mu, sigma = featureNormalize(X_poly) # Normalize
X_poly = np.c_[np.ones((m_train_set, 1)), X_poly]
X_poly_val = polyFeatures(Xval, dimension)
X_poly_val = X_poly_val - mu
X_poly_val = X_poly_val / sigma
X_poly_val = np.c_[np.ones((m_val_set, 1)), X_poly_val]
theta = trainLinearReg(X_poly, y, reg_lambda)
error_train[i], tmp = linearRegCostFunction(X_poly, y, theta, 0)
error_val[i], tmp = linearRegCostFunction(X_poly_val, yval, theta, 0)
return dimensions, error_train, error_val
def polynomialDegreeCurve(X, y, Xval, yval, reg_lambda):
"""Error cruve in function of degree of polynimal d
"""
dimensions = np.arange(1, 80).reshape(-1, 1)
# You need to return these variables correctly.
error_train = np.zeros((len(dimensions), 1))
error_val = np.zeros((len(dimensions), 1))
m_train_set = X.shape[0]
m_val_set = Xval.shape[0]
for i in range(len(dimensions)):
dimension = dimensions[i]
X_poly = polyFeatures(X, dimension)
X_poly, mu, sigma = featureNormalize(X_poly) # Normalize
X_poly = np.c_[np.ones((m_train_set, 1)), X_poly]
X_poly_val = polyFeatures(Xval, dimension)
X_poly_val = X_poly_val - mu
X_poly_val = X_poly_val / sigma
X_poly_val = np.c_[np.ones((m_val_set, 1)), X_poly_val]
theta = trainLinearReg(X_poly, y, reg_lambda)
error_train[i], tmp = linearRegCostFunction(X_poly, y, theta, 0)
error_val[i], tmp = linearRegCostFunction(X_poly_val, yval, theta, 0)
return dimensions, error_train, error_val
def plotFit(min_x, max_x, mu, sigma, theta, p):
'''
PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial
fit with power p and feature normalization (mu, sigma).
'''
import numpy as np
from polyFeatures import polyFeatures
import matplotlib.pyplot as plt
# We plot a range slightly bigger than the min and max values to get
# an idea of how the fit will vary outside the range of the data points
x = np.arange(min_x - 15, max_x + 25, 0.05)
# Map the X values
X_poly = polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly / sigma
# Add ones
X_poly = np.hstack((np.ones((X_poly.shape[0], 1)), X_poly))
# Plot
plt.plot(x, X_poly.dot(theta), '--', linewidth=2)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
def plotNorm(X,y,theta,mu,sigma):
"""Plots the data and best fit curve for polynomial features."""
p.scatter(X[:,0],y[:,0],marker='x')
# finding the range of x-values
xstart = np.amin(X[:,0])
xstop = np.amax(X[:,0])
step = (xstop-xstart)/100.
# extending the range of the x values
xstart -= step*10
xstop += step*10
xplot = np.arange(xstart,xstop,step)
# reshaping into a [100,1] array, since polyFeatures assumes this shape
xplot = np.reshape(xplot,[xplot.size,1])
degree = theta.size-1
xpoly = polyFeatures(xplot,degree)
for ii in range(degree):
xpoly[:,ii] -= mu[ii]
xpoly[:,ii] /= sigma[ii]
# endfor
# adding the bias unit for the dot product with theta
ndims = xpoly.shape
X = np.ones([ndims[0],ndims[1]+1])
X[:,1:] = xpoly
yplot = np.dot(theta,np.transpose(X))
p.plot(xplot,yplot,'r')
p.show()
def plotFit(min_x, max_x, mu, sigma, theta, p):
'''
%PLOTFIT Plots a learned polynomial regression fit over an existing figure.
%Also works with linear regression.
% PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial
% fit with power p and feature normalization (mu, sigma).
% Hold on to the current figure
hold on;
% We plot a range slightly bigger than the min and max values to get
% an idea of how the fit will vary outside the range of the data points
'''
x = np.arange(min_x - 15.,max_x + 25., 0.05)
#% Map the X values
x = x.reshape(-1,1)
X_poly = polyFeatures(x, p)
X_poly = X_poly- mu
X_poly = X_poly/sigma
#% Add ones
ones = np.ones((X_poly.shape[0], 1))
X_poly = np.hstack((ones, X_poly))
theta = theta.reshape(-1,1)
y = X_poly.dot(theta) #2512x9 9x1
#% Plot
plt.plot(x,y , '-')
def plotFit(min_x, max_x, mu, sigma, theta, p):
'''
%PLOTFIT Plots a learned polynomial regression fit over an existing figure.
%Also works with linear regression.
% PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial
% fit with power p and feature normalization (mu, sigma).
% Hold on to the current figure
hold on;
% We plot a range slightly bigger than the min and max values to get
% an idea of how the fit will vary outside the range of the data points
'''
x = np.arange(min_x - 15., max_x + 25., 0.05)
#% Map the X values
x = x.reshape(-1, 1)
X_poly = polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly / sigma
#% Add ones
ones = np.ones((X_poly.shape[0], 1))
X_poly = np.hstack((ones, X_poly))
theta = theta.reshape(-1, 1)
y = X_poly.dot(theta) #2512x9 9x1
#% Plot
plt.plot(x, y, '-')
def plotFit(min_x, max_x, mu, sigma, theta, p):
x = np.array(np.arange(min_x - 15, max_x + 25, 0.05))
X_poly = polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly / sigma
X_poly = np.column_stack((np.ones((x.shape[0], 1)), X_poly))
plt.plot(x, np.dot(X_poly, theta), '--', linewidth=2)
def plotFit(min_x, max_x, mu, sigma, theta, p):
#PLOTFIT Plots a learned polynomial regression fit over an existing figure.
#Also works with linear regression.
# PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial
# fit with power p and feature normalization (mu, sigma).
# Hold on to the current figure
plt.hold(True)
# We plot a range slightly bigger than the min and max values to get
# an idea of how the fit will vary outside the range of the data points
x = np.array(np.arange(min_x - 15, max_x + 25, 0.05)) # 1D vector
# Map the X values
X_poly = pf.polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly/sigma
# Add ones
X_poly = np.column_stack((np.ones((x.shape[0],1)), X_poly))
# Plot
plt.plot(x, np.dot(X_poly, theta), '--', linewidth=2)
# Hold off to the current figure
plt.hold(False)
def polyfit(min_x, max_x, mu, sigma, theta, p):
x = np.arange(min_x - 50, max_x + 50, 0.05).reshape(-1, 1)
x = np.hstack((np.ones((x.shape[0], 1)), x))
X_poly = polyFeatures(x, p)
X_poly -= mu
X_poly /= sigma
X_poly = np.hstack((np.ones((x.shape[0], 1)), X_poly))
plt.plot(x[:, 1], np.dot(X_poly, theta), '--', lw=2)
return
def plotFit(min_x, max_x, mu, sigma, theta, p):
theta = theta.reshape((theta.size, 1))
x = np.arange(min_x - 15, max_x + 25, 0.05)
x = x.reshape(x.size, 1)
X_poly = polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly / sigma
X_poly = np.hstack((np.ones((X_poly.shape[0], 1)), X_poly))
plt.plot(x, X_poly.dot(theta), linestyle='--')
def plotFit(min_x, max_x, mu, sigma, theta, p):
x = np.arange(min_x - 10, max_x + 10, 0.05)
x = x.reshape(x.size, 1)
X_poly = polyFeatures(x, p)
X_poly -= mu
X_poly /= sigma
X_poly = np.column_stack((np.ones(x.size), X_poly))
Y_fit = X_poly.dot(theta)
return x, Y_fit
def plotFit(min_x, max_x, mu, sigma, theta, degree):
x = np.arange(min_x - 60, max_x + 65, 0.05)
x = x.reshape(x.size, 1)
# Map X
X_poly = polyFeatures(x, degree)
X_poly = bsxfun(sub_op, X_poly, mu)
X_poly = bsxfun(div_op, X_poly, sigma)
# Add ones
X_poly = np.hstack((np.ones((x.shape[0], 1)), X_poly))
# plot
plt.plot(x.flatten(), X_poly.dot(theta), '--')
def plotFit(X, y, mu, sigma, theta, p, xlambda):
plotNormal(X, y)
plt.title('Polynomial Regression Fit (lambda = %f)' % xlambda)
x = np.arange((np.min(X) - 15), (np.max(X) + 25), 0.05)
l = len(x)
x = x.reshape(l, 1)
# Map the X values
X_poly = polyFeatures.polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly / sigma
# Add ones
X_poly = np.c_[(np.ones([x.shape[0], 1]), X_poly)]
# Plot
plt.plot(x, np.dot(X_poly, theta), color='b')
return plt
def plotFit(min_x, max_x, mu, sigma, theta, p):
"""plots the learned polynomial fit with power p
and feature normalization (mu, sigma).
"""
# We plot a range slightly bigger than the min and max values to get
# an idea of how the fit will vary outside the range of the data points
x = np.arange(min_x - 15, max_x + 25, 0.05).T
# Map the X values
X_poly = polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly / sigma
# Add ones
X_poly = np.column_stack((np.ones(x.shape[0]), X_poly))
# Plot
plt.plot(x, X_poly.dot(theta), '--', lw=2)
def plotFit(min_x, max_x, mu, sigma, theta, p):
# We plot a range slightly bigger than the min and max values to get
# an idea of how the fit will vary outside the range of the data points
x = np.arange(min_x - 15, max_x + 25, 0.05).reshape(-1,1)
# Map the X values
X_poly = polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly / sigma
# Add ones
m = X_poly.shape[0]
X_poly = np.vstack((np.ones(m), X_poly.T)).T
# Plot
theta = theta.reshape(-1,1)
plt.plot(x, np.dot(X_poly, theta), '--', linewidth=2)
plt.axis([-70, 70, -50, 50])
def plotFit(min_x, max_x, mu, sigma, theta, p):
"""plots the learned polynomial fit with power p
and feature normalization (mu, sigma).
"""
# We plot a range slightly bigger than the min and max values to get
# an idea of how the fit will vary outside the range of the data points
x = np.arange(min_x - 15, max_x + 25, 0.05).reshape(-1, 1)
# Map the X values
X_poly = polyFeatures(x, p)
X_poly -= mu
X_poly /= sigma
# Add ones
X_poly = np.c_[np.ones(np.size(X_poly, 0)), X_poly]
# Plot
plt.plot(x, X_poly @ theta.T, '--', lw=2)
def plotFit(min_x, max_x, mu, sigma, theta, p):
"""plots the learned polynomial fit with power p
and feature normalization (mu, sigma).
"""
# We plot a range slightly bigger than the min and max values to get
# an idea of how the fit will vary outside the range of the data points
x = np.arange(min_x - 15, max_x + 25, 0.05).T
# Map the X values
X_poly = polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly / sigma
# Add ones
X_poly = np.column_stack((np.ones(x.shape[0]), X_poly))
# Plot
plt.plot(x, X_poly.dot(theta), '--', lw=2)
def plotFit(min_x, max_x, mu, sigma, theta, p):
"""
PLOTFIT Plots a learned polynomial regression fit over an existing figure.
Also works with linear regression.
PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial
fit with power p and feature normalization (mu, sigma).
"""
# We plot a range slightly bigger than the min and max values to get
# an idea of how the fit will vary outside the range of the data points
x = np.matrix(np.arange(min_x - 15, max_x + 25, 0.05)).T
# Map the X values
X_poly = polyFeatures(x, p)
X_poly = X_poly - mu
X_poly /= sigma
# Add ones
X_poly = np.matrix(np.hstack((np.ones((X_poly.shape[0], 1)), X_poly)))
# Plot
plt.plot(x, X_poly.dot(theta).T, '-', linewidth=2)
def plotFit(min_x, max_x, mu, sigma, theta, p):
"""
Plots a learned polynomial regression fit over an existing figure.
Parameters
----------
min_x : float
Minimum value of features.
max_x : float
Maximum value of features.
mu : ndarray, shape (n_features - 1,)
Mean value of features, without the intercept term.
sigma : ndarray, shape (n_features - 1,)
Standard deviation of features, without the intercept term.
theta : ndarray, shape (n_features,)
Linear regression parameter.
p : int
Power of polynomial fit.
"""
x = np.arange(min_x - 15, max_x + 25, 0.05)
X_poly = polyFeatures(x, p)
X_poly, dummy_mu, dummy_sigma = featureNormalize(X_poly, mu, sigma)
X_poly = np.hstack((np.ones((X_poly.shape[0], 1)), X_poly))
plt.plot(x, X_poly.dot(theta), linestyle='--', marker='', color='b')
def plotFit(min_x, max_x, mu, sigma, theta, p, label):
"""Plots a learned polynomial regression fit over an existing figure.
Also works with linear regression.
PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial
fit with power p and feature normalization (mu, sigma).
"""
# We plot a range slightly bigger than the min and max values to get
# an idea of how the fit will vary outside the range of the data points
x = np.arange(min_x - 15, max_x + 25, 0.05).reshape(-1, 1)
# Map the X values
X_poly = polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly / sigma
# Add ones
X_poly = np.c_[np.ones((x.shape[0], 1)), X_poly]
curve, = plt.plot(x, X_poly.dot(theta), color='blue', label=label)
return curve
def plotFit(min_x, max_x, mu, sigma, theta, p, label):
"""Plots a learned polynomial regression fit over an existing figure.
Also works with linear regression.
PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial
fit with power p and feature normalization (mu, sigma).
"""
# We plot a range slightly bigger than the min and max values to get
# an idea of how the fit will vary outside the range of the data points
x = np.arange(min_x - 15, max_x + 25, 0.05).reshape(-1, 1)
# Map the X values
X_poly = polyFeatures(x, p)
X_poly = X_poly - mu
X_poly = X_poly / sigma
# Add ones
X_poly = np.c_[np.ones((x.shape[0], 1)), X_poly]
curve, = plt.plot(x, X_poly.dot(theta), color='blue', label=label)
return curve
def plotFit(min_x, max_x, mu, sigma, theta, p):
#PLOTFIT Plots a learned polynomial regression fit over an existing figure.
#Also works with linear regression.
# PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial
# fit with power p and feature normalization (mu, sigma).
# Hold on to the current figure
#hold on;
# We plot a range slightly bigger than the min and max values to get
# an idea of how the fit will vary outside the range of the data points
x = np.arange(min_x - 15, max_x + 25, 0.05)[None].T
# Map the X values
X_poly = polyFeatures(x, p)
X_poly -= mu
X_poly /= sigma
# Add ones
X_poly = np.concatenate([np.ones((x.shape[0], 1)), X_poly], axis=1)
# Plot
plt.plot(x, np.dot(X_poly, theta), '--', lw=2)
def output(partId):
# Random Test Cases
X = np.vstack([
np.ones(10),
np.sin(np.arange(1, 15, 1.5)),
np.cos(np.arange(1, 15, 1.5))
]).T
y = np.sin(np.arange(1, 31, 3))
Xval = np.vstack([
np.ones(10),
np.sin(np.arange(0, 14, 1.5)),
np.cos(np.arange(0, 14, 1.5))
]).T
yval = np.sin(np.arange(1, 11))
if partId == '1':
J, _ = linearRegCostFunction(X, y, np.array([0.1, 0.2, 0.3]), 0.5)
out = formatter('%0.5f ', J)
elif partId == '2':
J, grad = linearRegCostFunction(X, y, np.array([0.1, 0.2, 0.3]), 0.5)
out = formatter('%0.5f ', grad)
elif partId == '3':
error_train, error_val = learningCurve(X, y, Xval, yval, 1)
out = formatter(
'%0.5f ', np.concatenate([error_train.ravel(),
error_val.ravel()]))
elif partId == '4':
X_poly = polyFeatures(X[1, :].T, 8)
out = formatter('%0.5f ', X_poly)
elif partId == '5':
lambda_vec, error_train, error_val = validationCurve(X, y, Xval, yval)
out = formatter(
'%0.5f ',
np.concatenate(
[lambda_vec.ravel(),
error_train.ravel(),
error_val.ravel()]))
return out
def output(partId):
# Random Test Cases
X = column_stack([ones(10), sin(arange(1,15,1.5)), cos(arange(1,15,1.5))])
y = sin(arange(1,30,3))
Xval = column_stack([ones(10), sin(arange(0,14,1.5)), cos(arange(0,14,1.5))])
yval = sin(arange(1,11))
if partId == '1':
J, _ = linearRegCostFunction(X, y, hstack([0.1, 0.2, 0.3]), 0.5)
return sprintf('%0.5f ', J)
elif partId == '2':
J, grad = linearRegCostFunction(X, y, hstack([0.1, 0.2, 0.3]), 0.5)
return sprintf('%0.5f ', grad)
elif partId == '3':
error_train, error_val = \
learningCurve(X, y, Xval, yval, 1);
return sprintf('%0.5f ', vstack([error_train, error_val]))
elif partId == '4':
X_poly = polyFeatures(X[1,:], 8)
return sprintf('%0.5f ', X_poly)
elif partId == '5':
lambda_vec, error_train, error_val = \
validationCurve(X, y, Xval, yval)
return sprintf('%0.5f ', \
vstack([lambda_vec, error_train, error_val]))
print('Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
print(' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i]))
input("Program paused. Press Enter to continue...")
## =========== Part 6: Feature Mapping for Polynomial Regression =============
# One solution to this is to use polynomial regression. You should now
# complete polyFeatures to map each example into its powers
#
p = 8
# Map X onto Polynomial Features and Normalize
X_poly = polyFeatures(X, p)
X_poly, mu, sigma = featureNormalize(X_poly) # Normalize
X_poly = np.column_stack((np.ones(m), X_poly)) # Add Ones
# Map X_poly_test and normalize (using mu and sigma)
X_poly_test = polyFeatures(Xtest, p)
X_poly_test = X_poly_test - mu
X_poly_test = X_poly_test / sigma
X_poly_test = np.column_stack(
(np.ones(X_poly_test.shape[0]), X_poly_test)) # Add Ones
# Map X_poly_val and normalize (using mu and sigma)
X_poly_val = polyFeatures(Xval, p)
X_poly_val = X_poly_val - mu
X_poly_val = X_poly_val / sigma
X_poly_val = np.column_stack(
plt.legend(loc='upper right', shadow=True, fontsize=12, numpoints=1)
plt.xlabel('Число тренировочных примеров')
plt.ylabel('Ошибка')
plt.grid()
plt.show()
input('Программа остановлена. Нажмите Enter для продолжения ... \n')
# ==== Часть 5. Создание свойств для полиномиальной регрессии ====
print('Часть 5. Создание свойств для полиномиальной регрессии')
# Задание степени полинома
p = 8
X_poly = polyFeatures(X, p)
X_norm_poly, mu_poly, sigma_poly = featureNormalize(X_poly)
X_norm_poly = np.concatenate((np.ones((m, 1)), X_norm_poly), axis=1)
X_val_poly = polyFeatures(X_val, p)
X_val_norm_poly = np.divide(
X_val_poly - repmat(mu_poly, X_val_poly.shape[0], 1),
repmat(sigma_poly, X_val_poly.shape[0], 1))
X_val_norm_poly = np.concatenate((np.ones((m_val, 1)), X_val_norm_poly),
axis=1)
input('Программа остановлена. Нажмите Enter для продолжения ... \n')
# ========= Часть 6. Обучение полиномиальной регрессии ===========
print('Часть 6. Обучение полиномиальной регрессии')
print 'Training Examples\tTrain Error\tCross Validation Error'
for i in range(m):
print ' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i])
raw_input("Program paused. Press Enter to continue...")
## =========== Part 6: Feature Mapping for Polynomial Regression =============
# One solution to this is to use polynomial regression. You should now
# complete polyFeatures to map each example into its powers
#
p = 8
# Map X onto Polynomial Features and Normalize
X_poly = polyFeatures(X, p)
X_poly, mu, sigma = featureNormalize(X_poly) # Normalize
X_poly = np.column_stack((np.ones(m), X_poly)) # Add Ones
# Map X_poly_test and normalize (using mu and sigma)
X_poly_test = polyFeatures(Xtest, p)
X_poly_test = X_poly_test - mu
X_poly_test = X_poly_test / sigma
X_poly_test = np.column_stack((np.ones(X_poly_test.shape[0]), X_poly_test)) # Add Ones
# Map X_poly_val and normalize (using mu and sigma)
X_poly_val = polyFeatures(Xval, p)
X_poly_val = X_poly_val - mu
X_poly_val = X_poly_val / sigma
X_poly_val = np.column_stack((np.ones(X_poly_test.shape[0]), X_poly_val)) # Add Ones
plt.show()
#Xval_1 = np.hstack((np.ones((21,1)),Xval))
error_train, error_val = learningCurve.lc(X1, y, Xval, yval, greek_lambda)
plt.plot(range(12), error_train, label="Train")
plt.plot(range(12), error_val, label="Cross Validation", color="r")
plt.title("Learning Curve for Linear Regression")
plt.xlabel("Number of training examples")
plt.ylabel("Error")
plt.legend()
plt.show()
# Map X onto Polynomial features and normalize
p = 8
X_poly = polyFeatures.polyFeatures(X, p)
sc_X = StandardScaler()
X_poly = sc_X.fit_transform(X_poly)
X_poly = np.hstack((np.ones((X_poly.shape[0], 1)), X_poly))
# Map Xtest onto polynomial features and normalize
X_poly_test = polyFeatures.polyFeatures(Xtest, p)
X_poly_test = sc_X.transform(X_poly_test)
X_poly_test = np.hstack((np.ones((X_poly_test.shape[0], 1)), X_poly_test))
# Map Xval onto polynomial features and normalize
X_poly_val = polyFeatures.polyFeatures(Xval, p)
X_poly_val = sc_X.transform(X_poly_val)
X_poly_val = np.hstack((np.ones((X_poly_val.shape[0], 1)), X_poly_val))
# then [ -15.30, 598.250 ] for gradient
print "Cost for initial theta = [1,1]: ", \
costFunction.computeCost(theta,X,y,lam)
print "Gradient for initial theta = [1,1]: ", \
costFunction.computeDeriv(theta,X,y,lam)
theta = regression(theta,X,y,lam)
plotData.plotTheta(X,y,theta)
# Computing the learning curve for linear regression
(error_train,error_cv) = learningCurve(theta,X,y,Xcv,ycv,lam)
plotData.plotError(error_train,error_cv)
# Unregularized polynomial regression for a degree 8 polynomial
degree = 8
theta = initialize(degree)
X_poly = polyFeatures(X,degree)
(X_norm,mu,sigma) = featureNormalization(X_poly)
theta = regression(theta,X_norm,y,lam)
#plotData.plotNorm(X,y,theta,mu,sigma)
# Learning curve for a degree 8 polynomial
Xcv_norm = polyFeatures(Xcv,degree)
for ii in range(degree):
Xcv_norm[:,ii] -= mu[ii]
Xcv_norm[:,ii] /= sigma[ii]
(error_train,error_cv) = learningCurve(theta,X_norm,y,Xcv_norm,ycv,lam)
plotData.plotError(error_train,error_cv)
# Computing the validation curve for different regularization values
degree = 8
theta = initialize(degree)
(error_train,error_cv,lam) = validationCurve(theta,X_norm,y,Xcv_norm,ycv)
plt.plot(a, b, 'b-', lw=1) # 绘制直线图
plt.scatter(X1, y, c='red', marker='x', s=20)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.show()
'''绘制误差曲线，观察是否过拟合欠拟合'''
J_train, J_val = learningCurve(X, Xval, y, yval, theta_init, lmda)
plt.figure(1)
plt.plot(range(m), J_train, range(m), J_val)
plt.title('Learning curve for linear regression')
plt.xlabel('m')
plt.ylabel('Error')
plt.show()
"""应用到训练集、测试集、交叉验证集"""
'''增加特征数'''
X1_pol = polyFeatures(X1, 8)
Xtest1_pol = polyFeatures(Xtest, 8)
Xval1_pol = polyFeatures(Xval1, 8)
"""特征归一化"""
X2_pol, X_pol_mean, X_pol_std = feature_normalize(X1_pol)
# X2test_pol, Xtest_pol_mean, Xtest_pol_std = feature_normalize(Xtest1_pol)
# X2val_pol, Xval_pol_mean, Xval_pol_std = feature_normalize(Xval1_pol)
X2test_pol = (Xtest1_pol - X_pol_mean) / X_pol_std
X2val_pol = (Xval1_pol - X_pol_mean) / X_pol_std
X_pol = np.hstack((np.ones((X2_pol.shape[0])).reshape(X2_pol.shape[0],
1), X2_pol))
Xtest_pol = np.hstack((np.ones(
(X2test_pol.shape[0])).reshape(X2test_pol.shape[0], 1), X2test_pol))
Xval_pol = np.hstack((np.ones(
(X2val_pol.shape[0])).reshape(X2val_pol.shape[0], 1), X2val_pol))
def ex5():
## Machine Learning Online Class
# Exercise 5 | Regularized Linear Regression and Bias-Variance
#
# Instructions
# ------------
#
# This file contains code that helps you get started on the
# exercise. You will need to complete the following functions:
#
# linearRegCostFunction.m
# learningCurve.m
# validationCurve.m
#
# For this exercise, you will not need to change any code in this file,
# or any other files other than those mentioned above.
#
## Initialization
#clear ; close all; clc
## =========== Part 1: Loading and Visualizing Data =============
# We start the exercise by first loading and visualizing the dataset.
# The following code will load the dataset into your environment and plot
# the data.
#
# Load Training Data
print('Loading and Visualizing Data ...')
# Load from ex5data1:
# You will have X, y, Xval, yval, Xtest, ytest in your environment
mat = scipy.io.loadmat('ex5data1.mat')
X = mat['X']
y = mat['y'].ravel()
Xval = mat['Xval']
yval = mat['yval'].ravel()
Xtest = mat['Xtest']
ytest = mat['ytest'].ravel()
# m = Number of examples
m = X.shape[0]
# Plot training data
plt.plot(X, y, marker='x', linestyle='None', ms=10, lw=1.5)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.savefig('figure1.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 2: Regularized Linear Regression Cost =============
# You should now implement the cost function for regularized linear
# regression.
#
theta = np.array([1, 1])
J, _ = linearRegCostFunction(np.concatenate([np.ones((m, 1)), X], axis=1),
y, theta, 1)
print(
'Cost at theta = [1 ; 1]: %f \n(this value should be about 303.993192)'
% J)
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 3: Regularized Linear Regression Gradient =============
# You should now implement the gradient for regularized linear
# regression.
#
theta = np.array([1, 1])
J, grad = linearRegCostFunction(
np.concatenate([np.ones((m, 1)), X], axis=1), y, theta, 1)
print(
'Gradient at theta = [1 ; 1]: [%f; %f] \n(this value should be about [-15.303016; 598.250744])'
% (grad[0], grad[1]))
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 4: Train Linear Regression =============
# Once you have implemented the cost and gradient correctly, the
# trainLinearReg function will use your cost function to train
# regularized linear regression.
#
# Write Up Note: The data is non-linear, so this will not give a great
# fit.
#
fig = plt.figure()
# Train linear regression with lambda = 0
lambda_value = 0
theta = trainLinearReg(np.concatenate([np.ones((m, 1)), X], axis=1), y,
lambda_value)
# Plot fit over the data
plt.plot(X, y, marker='x', linestyle='None', ms=10, lw=1.5)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.plot(X,
np.dot(np.concatenate([np.ones((m, 1)), X], axis=1), theta),
'--',
lw=2)
plt.savefig('figure2.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 5: Learning Curve for Linear Regression =============
# Next, you should implement the learningCurve function.
#
# Write Up Note: Since the model is underfitting the data, we expect to
# see a graph with "high bias" -- slide 8 in ML-advice.pdf
#
fig = plt.figure()
lambda_value = 0
error_train, error_val = learningCurve(
np.concatenate([np.ones((m, 1)), X], axis=1), y,
np.concatenate([np.ones((yval.size, 1)), Xval], axis=1), yval,
lambda_value)
plt.plot(np.arange(1, m + 1), error_train, np.arange(1, m + 1), error_val)
plt.title('Learning curve for linear regression')
plt.legend(['Train', 'Cross Validation'])
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.axis([0, 13, 0, 150])
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
print(' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i]))
plt.savefig('figure3.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 6: Feature Mapping for Polynomial Regression =============
# One solution to this is to use polynomial regression. You should now
# complete polyFeatures to map each example into its powers
#
p = 8
# Map X onto Polynomial Features and Normalize
X_poly = polyFeatures(X, p)
X_poly, mu, sigma = featureNormalize(X_poly) # Normalize
X_poly = np.concatenate([np.ones((m, 1)), X_poly], axis=1) # Add Ones
# Map X_poly_test and normalize (using mu and sigma)
X_poly_test = polyFeatures(Xtest, p)
X_poly_test -= mu
X_poly_test /= sigma
X_poly_test = np.concatenate(
[np.ones((X_poly_test.shape[0], 1)), X_poly_test], axis=1) # Add Ones
# Map X_poly_val and normalize (using mu and sigma)
X_poly_val = polyFeatures(Xval, p)
X_poly_val -= mu
X_poly_val /= sigma
X_poly_val = np.concatenate(
[np.ones((X_poly_val.shape[0], 1)), X_poly_val], axis=1) # Add Ones
print('Normalized Training Example 1:')
print(formatter(' %f \n', X_poly[0, :]))
print('\nProgram paused. Press enter to continue.')
#pause;
## =========== Part 7: Learning Curve for Polynomial Regression =============
# Now, you will get to experiment with polynomial regression with multiple
# values of lambda. The code below runs polynomial regression with
# lambda = 0. You should try running the code with different values of
# lambda to see how the fit and learning curve change.
#
fig = plt.figure()
lambda_value = 0
theta = trainLinearReg(X_poly, y, lambda_value)
# Plot training data and fit
plt.plot(X, y, marker='x', ms=10, lw=1.5)
plotFit(np.min(X), np.max(X), mu, sigma, theta, p)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.title('Polynomial Regression Fit (lambda = %f)' % lambda_value)
plt.figure()
error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval,
lambda_value)
plt.plot(np.arange(1, 1 + m), error_train, np.arange(1, 1 + m), error_val)
plt.title('Polynomial Regression Learning Curve (lambda = %f)' %
lambda_value)
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.axis([0, 13, 0, 100])
plt.legend(['Train', 'Cross Validation'])
print('Polynomial Regression (lambda = %f)\n' % lambda_value)
print('# Training Examples\tTrain Error\tCross Validation Error')
for i in range(m):
print(' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i]))
plt.savefig('figure4.png')
print('Program paused. Press enter to continue.')
#pause;
## =========== Part 8: Validation for Selecting Lambda =============
# You will now implement validationCurve to test various values of
# lambda on a validation set. You will then use this to select the
# "best" lambda value.
#
fig = plt.figure()
lambda_vec, error_train, error_val = validationCurve(
X_poly, y, X_poly_val, yval)
plt.plot(lambda_vec, error_train, lambda_vec, error_val)
plt.legend(['Train', 'Cross Validation'])
plt.xlabel('lambda')
plt.ylabel('Error')
print('lambda\t\tTrain Error\tValidation Error')
for i in range(lambda_vec.size):
print(' %f\t%f\t%f' % (lambda_vec[i], error_train[i], error_val[i]))
plt.savefig('figure5.png')
print('Program paused. Press enter to continue.')
