def polynomialDegreeCurve(X, y, Xval, yval, reg_lambda): """Error cruve in function of degree of polynimal d """ dimensions = np.arange(1, 80).reshape(-1, 1) # You need to return these variables correctly. error_train = np.zeros((len(dimensions), 1)) error_val = np.zeros((len(dimensions), 1)) m_train_set = X.shape[0] m_val_set = Xval.shape[0] for i in range(len(dimensions)): dimension = dimensions[i] X_poly = polyFeatures(X, dimension) X_poly, mu, sigma = featureNormalize(X_poly) # Normalize X_poly = np.c_[np.ones((m_train_set, 1)), X_poly] X_poly_val = polyFeatures(Xval, dimension) X_poly_val = X_poly_val - mu X_poly_val = X_poly_val / sigma X_poly_val = np.c_[np.ones((m_val_set, 1)), X_poly_val] theta = trainLinearReg(X_poly, y, reg_lambda) error_train[i], tmp = linearRegCostFunction(X_poly, y, theta, 0) error_val[i], tmp = linearRegCostFunction(X_poly_val, yval, theta, 0) return dimensions, error_train, error_val
def plotFit(min_x, max_x, mu, sigma, theta, p): ''' PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial fit with power p and feature normalization (mu, sigma). ''' import numpy as np from polyFeatures import polyFeatures import matplotlib.pyplot as plt # We plot a range slightly bigger than the min and max values to get # an idea of how the fit will vary outside the range of the data points x = np.arange(min_x - 15, max_x + 25, 0.05) # Map the X values X_poly = polyFeatures(x, p) X_poly = X_poly - mu X_poly = X_poly / sigma # Add ones X_poly = np.hstack((np.ones((X_poly.shape[0], 1)), X_poly)) # Plot plt.plot(x, X_poly.dot(theta), '--', linewidth=2) plt.xlabel('Change in water level (x)') plt.ylabel('Water flowing out of the dam (y)')
def plotNorm(X,y,theta,mu,sigma): """Plots the data and best fit curve for polynomial features.""" p.scatter(X[:,0],y[:,0],marker='x') # finding the range of x-values xstart = np.amin(X[:,0]) xstop = np.amax(X[:,0]) step = (xstop-xstart)/100. # extending the range of the x values xstart -= step*10 xstop += step*10 xplot = np.arange(xstart,xstop,step) # reshaping into a [100,1] array, since polyFeatures assumes this shape xplot = np.reshape(xplot,[xplot.size,1]) degree = theta.size-1 xpoly = polyFeatures(xplot,degree) for ii in range(degree): xpoly[:,ii] -= mu[ii] xpoly[:,ii] /= sigma[ii] # endfor # adding the bias unit for the dot product with theta ndims = xpoly.shape X = np.ones([ndims[0],ndims[1]+1]) X[:,1:] = xpoly yplot = np.dot(theta,np.transpose(X)) p.plot(xplot,yplot,'r') p.show()
def plotFit(min_x, max_x, mu, sigma, theta, p): ''' %PLOTFIT Plots a learned polynomial regression fit over an existing figure. %Also works with linear regression. % PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial % fit with power p and feature normalization (mu, sigma). % Hold on to the current figure hold on; % We plot a range slightly bigger than the min and max values to get % an idea of how the fit will vary outside the range of the data points ''' x = np.arange(min_x - 15.,max_x + 25., 0.05) #% Map the X values x = x.reshape(-1,1) X_poly = polyFeatures(x, p) X_poly = X_poly- mu X_poly = X_poly/sigma #% Add ones ones = np.ones((X_poly.shape[0], 1)) X_poly = np.hstack((ones, X_poly)) theta = theta.reshape(-1,1) y = X_poly.dot(theta) #2512x9 9x1 #% Plot plt.plot(x,y , '-')
def plotFit(min_x, max_x, mu, sigma, theta, p): ''' %PLOTFIT Plots a learned polynomial regression fit over an existing figure. %Also works with linear regression. % PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial % fit with power p and feature normalization (mu, sigma). % Hold on to the current figure hold on; % We plot a range slightly bigger than the min and max values to get % an idea of how the fit will vary outside the range of the data points ''' x = np.arange(min_x - 15., max_x + 25., 0.05) #% Map the X values x = x.reshape(-1, 1) X_poly = polyFeatures(x, p) X_poly = X_poly - mu X_poly = X_poly / sigma #% Add ones ones = np.ones((X_poly.shape[0], 1)) X_poly = np.hstack((ones, X_poly)) theta = theta.reshape(-1, 1) y = X_poly.dot(theta) #2512x9 9x1 #% Plot plt.plot(x, y, '-')
def plotFit(min_x, max_x, mu, sigma, theta, p): x = np.array(np.arange(min_x - 15, max_x + 25, 0.05)) X_poly = polyFeatures(x, p) X_poly = X_poly - mu X_poly = X_poly / sigma X_poly = np.column_stack((np.ones((x.shape[0], 1)), X_poly)) plt.plot(x, np.dot(X_poly, theta), '--', linewidth=2)
def plotFit(min_x, max_x, mu, sigma, theta, p): #PLOTFIT Plots a learned polynomial regression fit over an existing figure. #Also works with linear regression. # PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial # fit with power p and feature normalization (mu, sigma). # Hold on to the current figure plt.hold(True) # We plot a range slightly bigger than the min and max values to get # an idea of how the fit will vary outside the range of the data points x = np.array(np.arange(min_x - 15, max_x + 25, 0.05)) # 1D vector # Map the X values X_poly = pf.polyFeatures(x, p) X_poly = X_poly - mu X_poly = X_poly/sigma # Add ones X_poly = np.column_stack((np.ones((x.shape[0],1)), X_poly)) # Plot plt.plot(x, np.dot(X_poly, theta), '--', linewidth=2) # Hold off to the current figure plt.hold(False)
def polyfit(min_x, max_x, mu, sigma, theta, p): x = np.arange(min_x - 50, max_x + 50, 0.05).reshape(-1, 1) x = np.hstack((np.ones((x.shape[0], 1)), x)) X_poly = polyFeatures(x, p) X_poly -= mu X_poly /= sigma X_poly = np.hstack((np.ones((x.shape[0], 1)), X_poly)) plt.plot(x[:, 1], np.dot(X_poly, theta), '--', lw=2) return
def plotFit(min_x, max_x, mu, sigma, theta, p): theta = theta.reshape((theta.size, 1)) x = np.arange(min_x - 15, max_x + 25, 0.05) x = x.reshape(x.size, 1) X_poly = polyFeatures(x, p) X_poly = X_poly - mu X_poly = X_poly / sigma X_poly = np.hstack((np.ones((X_poly.shape[0], 1)), X_poly)) plt.plot(x, X_poly.dot(theta), linestyle='--')
def plotFit(min_x, max_x, mu, sigma, theta, p): x = np.arange(min_x - 10, max_x + 10, 0.05) x = x.reshape(x.size, 1) X_poly = polyFeatures(x, p) X_poly -= mu X_poly /= sigma X_poly = np.column_stack((np.ones(x.size), X_poly)) Y_fit = X_poly.dot(theta) return x, Y_fit
def plotFit(min_x, max_x, mu, sigma, theta, degree): x = np.arange(min_x - 60, max_x + 65, 0.05) x = x.reshape(x.size, 1) # Map X X_poly = polyFeatures(x, degree) X_poly = bsxfun(sub_op, X_poly, mu) X_poly = bsxfun(div_op, X_poly, sigma) # Add ones X_poly = np.hstack((np.ones((x.shape[0], 1)), X_poly)) # plot plt.plot(x.flatten(), X_poly.dot(theta), '--')
def plotFit(X, y, mu, sigma, theta, p, xlambda): plotNormal(X, y) plt.title('Polynomial Regression Fit (lambda = %f)' % xlambda) x = np.arange((np.min(X) - 15), (np.max(X) + 25), 0.05) l = len(x) x = x.reshape(l, 1) # Map the X values X_poly = polyFeatures.polyFeatures(x, p) X_poly = X_poly - mu X_poly = X_poly / sigma # Add ones X_poly = np.c_[(np.ones([x.shape[0], 1]), X_poly)] # Plot plt.plot(x, np.dot(X_poly, theta), color='b') return plt
def plotFit(min_x, max_x, mu, sigma, theta, p): """plots the learned polynomial fit with power p and feature normalization (mu, sigma). """ # We plot a range slightly bigger than the min and max values to get # an idea of how the fit will vary outside the range of the data points x = np.arange(min_x - 15, max_x + 25, 0.05).T # Map the X values X_poly = polyFeatures(x, p) X_poly = X_poly - mu X_poly = X_poly / sigma # Add ones X_poly = np.column_stack((np.ones(x.shape[0]), X_poly)) # Plot plt.plot(x, X_poly.dot(theta), '--', lw=2)
def plotFit(min_x, max_x, mu, sigma, theta, p): # We plot a range slightly bigger than the min and max values to get # an idea of how the fit will vary outside the range of the data points x = np.arange(min_x - 15, max_x + 25, 0.05).reshape(-1,1) # Map the X values X_poly = polyFeatures(x, p) X_poly = X_poly - mu X_poly = X_poly / sigma # Add ones m = X_poly.shape[0] X_poly = np.vstack((np.ones(m), X_poly.T)).T # Plot theta = theta.reshape(-1,1) plt.plot(x, np.dot(X_poly, theta), '--', linewidth=2) plt.axis([-70, 70, -50, 50])
def plotFit(min_x, max_x, mu, sigma, theta, p): """plots the learned polynomial fit with power p and feature normalization (mu, sigma). """ # We plot a range slightly bigger than the min and max values to get # an idea of how the fit will vary outside the range of the data points x = np.arange(min_x - 15, max_x + 25, 0.05).reshape(-1, 1) # Map the X values X_poly = polyFeatures(x, p) X_poly -= mu X_poly /= sigma # Add ones X_poly = np.c_[np.ones(np.size(X_poly, 0)), X_poly] # Plot plt.plot(x, X_poly @ theta.T, '--', lw=2)
def plotFit(min_x, max_x, mu, sigma, theta, p): """ PLOTFIT Plots a learned polynomial regression fit over an existing figure. Also works with linear regression. PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial fit with power p and feature normalization (mu, sigma). """ # We plot a range slightly bigger than the min and max values to get # an idea of how the fit will vary outside the range of the data points x = np.matrix(np.arange(min_x - 15, max_x + 25, 0.05)).T # Map the X values X_poly = polyFeatures(x, p) X_poly = X_poly - mu X_poly /= sigma # Add ones X_poly = np.matrix(np.hstack((np.ones((X_poly.shape[0], 1)), X_poly))) # Plot plt.plot(x, X_poly.dot(theta).T, '-', linewidth=2)
def plotFit(min_x, max_x, mu, sigma, theta, p): """ Plots a learned polynomial regression fit over an existing figure. Parameters ---------- min_x : float Minimum value of features. max_x : float Maximum value of features. mu : ndarray, shape (n_features - 1,) Mean value of features, without the intercept term. sigma : ndarray, shape (n_features - 1,) Standard deviation of features, without the intercept term. theta : ndarray, shape (n_features,) Linear regression parameter. p : int Power of polynomial fit. """ x = np.arange(min_x - 15, max_x + 25, 0.05) X_poly = polyFeatures(x, p) X_poly, dummy_mu, dummy_sigma = featureNormalize(X_poly, mu, sigma) X_poly = np.hstack((np.ones((X_poly.shape[0], 1)), X_poly)) plt.plot(x, X_poly.dot(theta), linestyle='--', marker='', color='b')
def plotFit(min_x, max_x, mu, sigma, theta, p, label): """Plots a learned polynomial regression fit over an existing figure. Also works with linear regression. PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial fit with power p and feature normalization (mu, sigma). """ # We plot a range slightly bigger than the min and max values to get # an idea of how the fit will vary outside the range of the data points x = np.arange(min_x - 15, max_x + 25, 0.05).reshape(-1, 1) # Map the X values X_poly = polyFeatures(x, p) X_poly = X_poly - mu X_poly = X_poly / sigma # Add ones X_poly = np.c_[np.ones((x.shape[0], 1)), X_poly] curve, = plt.plot(x, X_poly.dot(theta), color='blue', label=label) return curve
def plotFit(min_x, max_x, mu, sigma, theta, p): #PLOTFIT Plots a learned polynomial regression fit over an existing figure. #Also works with linear regression. # PLOTFIT(min_x, max_x, mu, sigma, theta, p) plots the learned polynomial # fit with power p and feature normalization (mu, sigma). # Hold on to the current figure #hold on; # We plot a range slightly bigger than the min and max values to get # an idea of how the fit will vary outside the range of the data points x = np.arange(min_x - 15, max_x + 25, 0.05)[None].T # Map the X values X_poly = polyFeatures(x, p) X_poly -= mu X_poly /= sigma # Add ones X_poly = np.concatenate([np.ones((x.shape[0], 1)), X_poly], axis=1) # Plot plt.plot(x, np.dot(X_poly, theta), '--', lw=2)
def output(partId): # Random Test Cases X = np.vstack([ np.ones(10), np.sin(np.arange(1, 15, 1.5)), np.cos(np.arange(1, 15, 1.5)) ]).T y = np.sin(np.arange(1, 31, 3)) Xval = np.vstack([ np.ones(10), np.sin(np.arange(0, 14, 1.5)), np.cos(np.arange(0, 14, 1.5)) ]).T yval = np.sin(np.arange(1, 11)) if partId == '1': J, _ = linearRegCostFunction(X, y, np.array([0.1, 0.2, 0.3]), 0.5) out = formatter('%0.5f ', J) elif partId == '2': J, grad = linearRegCostFunction(X, y, np.array([0.1, 0.2, 0.3]), 0.5) out = formatter('%0.5f ', grad) elif partId == '3': error_train, error_val = learningCurve(X, y, Xval, yval, 1) out = formatter( '%0.5f ', np.concatenate([error_train.ravel(), error_val.ravel()])) elif partId == '4': X_poly = polyFeatures(X[1, :].T, 8) out = formatter('%0.5f ', X_poly) elif partId == '5': lambda_vec, error_train, error_val = validationCurve(X, y, Xval, yval) out = formatter( '%0.5f ', np.concatenate( [lambda_vec.ravel(), error_train.ravel(), error_val.ravel()])) return out
def output(partId): # Random Test Cases X = column_stack([ones(10), sin(arange(1,15,1.5)), cos(arange(1,15,1.5))]) y = sin(arange(1,30,3)) Xval = column_stack([ones(10), sin(arange(0,14,1.5)), cos(arange(0,14,1.5))]) yval = sin(arange(1,11)) if partId == '1': J, _ = linearRegCostFunction(X, y, hstack([0.1, 0.2, 0.3]), 0.5) return sprintf('%0.5f ', J) elif partId == '2': J, grad = linearRegCostFunction(X, y, hstack([0.1, 0.2, 0.3]), 0.5) return sprintf('%0.5f ', grad) elif partId == '3': error_train, error_val = \ learningCurve(X, y, Xval, yval, 1); return sprintf('%0.5f ', vstack([error_train, error_val])) elif partId == '4': X_poly = polyFeatures(X[1,:], 8) return sprintf('%0.5f ', X_poly) elif partId == '5': lambda_vec, error_train, error_val = \ validationCurve(X, y, Xval, yval) return sprintf('%0.5f ', \ vstack([lambda_vec, error_train, error_val]))
print('Training Examples\tTrain Error\tCross Validation Error') for i in range(m): print(' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i])) input("Program paused. Press Enter to continue...") ## =========== Part 6: Feature Mapping for Polynomial Regression ============= # One solution to this is to use polynomial regression. You should now # complete polyFeatures to map each example into its powers # p = 8 # Map X onto Polynomial Features and Normalize X_poly = polyFeatures(X, p) X_poly, mu, sigma = featureNormalize(X_poly) # Normalize X_poly = np.column_stack((np.ones(m), X_poly)) # Add Ones # Map X_poly_test and normalize (using mu and sigma) X_poly_test = polyFeatures(Xtest, p) X_poly_test = X_poly_test - mu X_poly_test = X_poly_test / sigma X_poly_test = np.column_stack( (np.ones(X_poly_test.shape[0]), X_poly_test)) # Add Ones # Map X_poly_val and normalize (using mu and sigma) X_poly_val = polyFeatures(Xval, p) X_poly_val = X_poly_val - mu X_poly_val = X_poly_val / sigma X_poly_val = np.column_stack(
plt.legend(loc='upper right', shadow=True, fontsize=12, numpoints=1) plt.xlabel('Число тренировочных примеров') plt.ylabel('Ошибка') plt.grid() plt.show() input('Программа остановлена. Нажмите Enter для продолжения ... \n') # ==== Часть 5. Создание свойств для полиномиальной регрессии ==== print('Часть 5. Создание свойств для полиномиальной регрессии') # Задание степени полинома p = 8 X_poly = polyFeatures(X, p) X_norm_poly, mu_poly, sigma_poly = featureNormalize(X_poly) X_norm_poly = np.concatenate((np.ones((m, 1)), X_norm_poly), axis=1) X_val_poly = polyFeatures(X_val, p) X_val_norm_poly = np.divide( X_val_poly - repmat(mu_poly, X_val_poly.shape[0], 1), repmat(sigma_poly, X_val_poly.shape[0], 1)) X_val_norm_poly = np.concatenate((np.ones((m_val, 1)), X_val_norm_poly), axis=1) input('Программа остановлена. Нажмите Enter для продолжения ... \n') # ========= Часть 6. Обучение полиномиальной регрессии =========== print('Часть 6. Обучение полиномиальной регрессии')
print 'Training Examples\tTrain Error\tCross Validation Error' for i in range(m): print ' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i]) raw_input("Program paused. Press Enter to continue...") ## =========== Part 6: Feature Mapping for Polynomial Regression ============= # One solution to this is to use polynomial regression. You should now # complete polyFeatures to map each example into its powers # p = 8 # Map X onto Polynomial Features and Normalize X_poly = polyFeatures(X, p) X_poly, mu, sigma = featureNormalize(X_poly) # Normalize X_poly = np.column_stack((np.ones(m), X_poly)) # Add Ones # Map X_poly_test and normalize (using mu and sigma) X_poly_test = polyFeatures(Xtest, p) X_poly_test = X_poly_test - mu X_poly_test = X_poly_test / sigma X_poly_test = np.column_stack((np.ones(X_poly_test.shape[0]), X_poly_test)) # Add Ones # Map X_poly_val and normalize (using mu and sigma) X_poly_val = polyFeatures(Xval, p) X_poly_val = X_poly_val - mu X_poly_val = X_poly_val / sigma X_poly_val = np.column_stack((np.ones(X_poly_test.shape[0]), X_poly_val)) # Add Ones
plt.show() #Xval_1 = np.hstack((np.ones((21,1)),Xval)) error_train, error_val = learningCurve.lc(X1, y, Xval, yval, greek_lambda) plt.plot(range(12), error_train, label="Train") plt.plot(range(12), error_val, label="Cross Validation", color="r") plt.title("Learning Curve for Linear Regression") plt.xlabel("Number of training examples") plt.ylabel("Error") plt.legend() plt.show() # Map X onto Polynomial features and normalize p = 8 X_poly = polyFeatures.polyFeatures(X, p) sc_X = StandardScaler() X_poly = sc_X.fit_transform(X_poly) X_poly = np.hstack((np.ones((X_poly.shape[0], 1)), X_poly)) # Map Xtest onto polynomial features and normalize X_poly_test = polyFeatures.polyFeatures(Xtest, p) X_poly_test = sc_X.transform(X_poly_test) X_poly_test = np.hstack((np.ones((X_poly_test.shape[0], 1)), X_poly_test)) # Map Xval onto polynomial features and normalize X_poly_val = polyFeatures.polyFeatures(Xval, p) X_poly_val = sc_X.transform(X_poly_val) X_poly_val = np.hstack((np.ones((X_poly_val.shape[0], 1)), X_poly_val))
# then [ -15.30, 598.250 ] for gradient print "Cost for initial theta = [1,1]: ", \ costFunction.computeCost(theta,X,y,lam) print "Gradient for initial theta = [1,1]: ", \ costFunction.computeDeriv(theta,X,y,lam) theta = regression(theta,X,y,lam) plotData.plotTheta(X,y,theta) # Computing the learning curve for linear regression (error_train,error_cv) = learningCurve(theta,X,y,Xcv,ycv,lam) plotData.plotError(error_train,error_cv) # Unregularized polynomial regression for a degree 8 polynomial degree = 8 theta = initialize(degree) X_poly = polyFeatures(X,degree) (X_norm,mu,sigma) = featureNormalization(X_poly) theta = regression(theta,X_norm,y,lam) #plotData.plotNorm(X,y,theta,mu,sigma) # Learning curve for a degree 8 polynomial Xcv_norm = polyFeatures(Xcv,degree) for ii in range(degree): Xcv_norm[:,ii] -= mu[ii] Xcv_norm[:,ii] /= sigma[ii] (error_train,error_cv) = learningCurve(theta,X_norm,y,Xcv_norm,ycv,lam) plotData.plotError(error_train,error_cv) # Computing the validation curve for different regularization values degree = 8 theta = initialize(degree) (error_train,error_cv,lam) = validationCurve(theta,X_norm,y,Xcv_norm,ycv)
plt.plot(a, b, 'b-', lw=1) # 绘制直线图 plt.scatter(X1, y, c='red', marker='x', s=20) plt.xlabel('Change in water level (x)') plt.ylabel('Water flowing out of the dam (y)') plt.show() '''绘制误差曲线,观察是否过拟合欠拟合''' J_train, J_val = learningCurve(X, Xval, y, yval, theta_init, lmda) plt.figure(1) plt.plot(range(m), J_train, range(m), J_val) plt.title('Learning curve for linear regression') plt.xlabel('m') plt.ylabel('Error') plt.show() """应用到训练集、测试集、交叉验证集""" '''增加特征数''' X1_pol = polyFeatures(X1, 8) Xtest1_pol = polyFeatures(Xtest, 8) Xval1_pol = polyFeatures(Xval1, 8) """特征归一化""" X2_pol, X_pol_mean, X_pol_std = feature_normalize(X1_pol) # X2test_pol, Xtest_pol_mean, Xtest_pol_std = feature_normalize(Xtest1_pol) # X2val_pol, Xval_pol_mean, Xval_pol_std = feature_normalize(Xval1_pol) X2test_pol = (Xtest1_pol - X_pol_mean) / X_pol_std X2val_pol = (Xval1_pol - X_pol_mean) / X_pol_std X_pol = np.hstack((np.ones((X2_pol.shape[0])).reshape(X2_pol.shape[0], 1), X2_pol)) Xtest_pol = np.hstack((np.ones( (X2test_pol.shape[0])).reshape(X2test_pol.shape[0], 1), X2test_pol)) Xval_pol = np.hstack((np.ones( (X2val_pol.shape[0])).reshape(X2val_pol.shape[0], 1), X2val_pol))
def ex5(): ## Machine Learning Online Class # Exercise 5 | Regularized Linear Regression and Bias-Variance # # Instructions # ------------ # # This file contains code that helps you get started on the # exercise. You will need to complete the following functions: # # linearRegCostFunction.m # learningCurve.m # validationCurve.m # # For this exercise, you will not need to change any code in this file, # or any other files other than those mentioned above. # ## Initialization #clear ; close all; clc ## =========== Part 1: Loading and Visualizing Data ============= # We start the exercise by first loading and visualizing the dataset. # The following code will load the dataset into your environment and plot # the data. # # Load Training Data print('Loading and Visualizing Data ...') # Load from ex5data1: # You will have X, y, Xval, yval, Xtest, ytest in your environment mat = scipy.io.loadmat('ex5data1.mat') X = mat['X'] y = mat['y'].ravel() Xval = mat['Xval'] yval = mat['yval'].ravel() Xtest = mat['Xtest'] ytest = mat['ytest'].ravel() # m = Number of examples m = X.shape[0] # Plot training data plt.plot(X, y, marker='x', linestyle='None', ms=10, lw=1.5) plt.xlabel('Change in water level (x)') plt.ylabel('Water flowing out of the dam (y)') plt.savefig('figure1.png') print('Program paused. Press enter to continue.') #pause; ## =========== Part 2: Regularized Linear Regression Cost ============= # You should now implement the cost function for regularized linear # regression. # theta = np.array([1, 1]) J, _ = linearRegCostFunction(np.concatenate([np.ones((m, 1)), X], axis=1), y, theta, 1) print( 'Cost at theta = [1 ; 1]: %f \n(this value should be about 303.993192)' % J) print('Program paused. Press enter to continue.') #pause; ## =========== Part 3: Regularized Linear Regression Gradient ============= # You should now implement the gradient for regularized linear # regression. # theta = np.array([1, 1]) J, grad = linearRegCostFunction( np.concatenate([np.ones((m, 1)), X], axis=1), y, theta, 1) print( 'Gradient at theta = [1 ; 1]: [%f; %f] \n(this value should be about [-15.303016; 598.250744])' % (grad[0], grad[1])) print('Program paused. Press enter to continue.') #pause; ## =========== Part 4: Train Linear Regression ============= # Once you have implemented the cost and gradient correctly, the # trainLinearReg function will use your cost function to train # regularized linear regression. # # Write Up Note: The data is non-linear, so this will not give a great # fit. # fig = plt.figure() # Train linear regression with lambda = 0 lambda_value = 0 theta = trainLinearReg(np.concatenate([np.ones((m, 1)), X], axis=1), y, lambda_value) # Plot fit over the data plt.plot(X, y, marker='x', linestyle='None', ms=10, lw=1.5) plt.xlabel('Change in water level (x)') plt.ylabel('Water flowing out of the dam (y)') plt.plot(X, np.dot(np.concatenate([np.ones((m, 1)), X], axis=1), theta), '--', lw=2) plt.savefig('figure2.png') print('Program paused. Press enter to continue.') #pause; ## =========== Part 5: Learning Curve for Linear Regression ============= # Next, you should implement the learningCurve function. # # Write Up Note: Since the model is underfitting the data, we expect to # see a graph with "high bias" -- slide 8 in ML-advice.pdf # fig = plt.figure() lambda_value = 0 error_train, error_val = learningCurve( np.concatenate([np.ones((m, 1)), X], axis=1), y, np.concatenate([np.ones((yval.size, 1)), Xval], axis=1), yval, lambda_value) plt.plot(np.arange(1, m + 1), error_train, np.arange(1, m + 1), error_val) plt.title('Learning curve for linear regression') plt.legend(['Train', 'Cross Validation']) plt.xlabel('Number of training examples') plt.ylabel('Error') plt.axis([0, 13, 0, 150]) print('# Training Examples\tTrain Error\tCross Validation Error') for i in range(m): print(' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i])) plt.savefig('figure3.png') print('Program paused. Press enter to continue.') #pause; ## =========== Part 6: Feature Mapping for Polynomial Regression ============= # One solution to this is to use polynomial regression. You should now # complete polyFeatures to map each example into its powers # p = 8 # Map X onto Polynomial Features and Normalize X_poly = polyFeatures(X, p) X_poly, mu, sigma = featureNormalize(X_poly) # Normalize X_poly = np.concatenate([np.ones((m, 1)), X_poly], axis=1) # Add Ones # Map X_poly_test and normalize (using mu and sigma) X_poly_test = polyFeatures(Xtest, p) X_poly_test -= mu X_poly_test /= sigma X_poly_test = np.concatenate( [np.ones((X_poly_test.shape[0], 1)), X_poly_test], axis=1) # Add Ones # Map X_poly_val and normalize (using mu and sigma) X_poly_val = polyFeatures(Xval, p) X_poly_val -= mu X_poly_val /= sigma X_poly_val = np.concatenate( [np.ones((X_poly_val.shape[0], 1)), X_poly_val], axis=1) # Add Ones print('Normalized Training Example 1:') print(formatter(' %f \n', X_poly[0, :])) print('\nProgram paused. Press enter to continue.') #pause; ## =========== Part 7: Learning Curve for Polynomial Regression ============= # Now, you will get to experiment with polynomial regression with multiple # values of lambda. The code below runs polynomial regression with # lambda = 0. You should try running the code with different values of # lambda to see how the fit and learning curve change. # fig = plt.figure() lambda_value = 0 theta = trainLinearReg(X_poly, y, lambda_value) # Plot training data and fit plt.plot(X, y, marker='x', ms=10, lw=1.5) plotFit(np.min(X), np.max(X), mu, sigma, theta, p) plt.xlabel('Change in water level (x)') plt.ylabel('Water flowing out of the dam (y)') plt.title('Polynomial Regression Fit (lambda = %f)' % lambda_value) plt.figure() error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval, lambda_value) plt.plot(np.arange(1, 1 + m), error_train, np.arange(1, 1 + m), error_val) plt.title('Polynomial Regression Learning Curve (lambda = %f)' % lambda_value) plt.xlabel('Number of training examples') plt.ylabel('Error') plt.axis([0, 13, 0, 100]) plt.legend(['Train', 'Cross Validation']) print('Polynomial Regression (lambda = %f)\n' % lambda_value) print('# Training Examples\tTrain Error\tCross Validation Error') for i in range(m): print(' \t%d\t\t%f\t%f' % (i, error_train[i], error_val[i])) plt.savefig('figure4.png') print('Program paused. Press enter to continue.') #pause; ## =========== Part 8: Validation for Selecting Lambda ============= # You will now implement validationCurve to test various values of # lambda on a validation set. You will then use this to select the # "best" lambda value. # fig = plt.figure() lambda_vec, error_train, error_val = validationCurve( X_poly, y, X_poly_val, yval) plt.plot(lambda_vec, error_train, lambda_vec, error_val) plt.legend(['Train', 'Cross Validation']) plt.xlabel('lambda') plt.ylabel('Error') print('lambda\t\tTrain Error\tValidation Error') for i in range(lambda_vec.size): print(' %f\t%f\t%f' % (lambda_vec[i], error_train[i], error_val[i])) plt.savefig('figure5.png') print('Program paused. Press enter to continue.')