def validationCurve(X, y, Xval, yval): ''' [lambda_vec, error_train, error_val] = VALIDATIONCURVE(X, y, Xval, yval) returns the train and validation errors (in error_train, error_val) for different values of lambda. You are given the training set (X, y) and validation set (Xval, yval). ''' import numpy as np from trainLinearReg import trainLinearReg from linearRegCostFunction import linearRegCostFunction # Selected values of lambda (you should not change this) lambda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10] # You need to return these variables correctly. error_train = np.zeros((len(lambda_vec), 1)) error_val = np.zeros((len(lambda_vec), 1)) for i in range(len(lambda_vec)): lambda_par = lambda_vec[i] # Compute train / val errors when training linear # regression with regularization parameter lambda theta = trainLinearReg(X,y,lambda_par) error_train[i], _ = linearRegCostFunction(X, y, theta, 0) error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0) return lambda_vec, error_train, error_val
def polynomialDegreeCurve(X, y, Xval, yval, reg_lambda): """Error cruve in function of degree of polynimal d """ dimensions = np.arange(1, 80).reshape(-1, 1) # You need to return these variables correctly. error_train = np.zeros((len(dimensions), 1)) error_val = np.zeros((len(dimensions), 1)) m_train_set = X.shape[0] m_val_set = Xval.shape[0] for i in range(len(dimensions)): dimension = dimensions[i] X_poly = polyFeatures(X, dimension) X_poly, mu, sigma = featureNormalize(X_poly) # Normalize X_poly = np.c_[np.ones((m_train_set, 1)), X_poly] X_poly_val = polyFeatures(Xval, dimension) X_poly_val = X_poly_val - mu X_poly_val = X_poly_val / sigma X_poly_val = np.c_[np.ones((m_val_set, 1)), X_poly_val] theta = trainLinearReg(X_poly, y, reg_lambda) error_train[i], tmp = linearRegCostFunction(X_poly, y, theta, 0) error_val[i], tmp = linearRegCostFunction(X_poly_val, yval, theta, 0) return dimensions, error_train, error_val
def learningCurve(X, y, Xval, yval, lambda_par): ''' LEARNINGCURVE(X, y, Xval, yval, lambda) returns the train and cross validation set errors for a learning curve. In particular, it returns two vectors of the same length - error_train and error_val. Then, error_train(i) contains the training error for i examples (and similarly for error_val(i)). ''' import numpy as np from trainLinearReg import trainLinearReg from linearRegCostFunction import linearRegCostFunction # In this function, you will compute the train and test errors for # dataset sizes from 1 up to m. In practice, when working with larger # datasets, you might want to do this in larger intervals. # Number of training examples m = X.shape[0] # You need to return these values correctly error_train = np.zeros((m, 1)) error_val = np.zeros((m, 1)) # Loop over the examples with the following: for i in range(1, m + 1): # Use the trainLinearReg to find the theta parameters theta = trainLinearReg(X[0:i, :], y[0:i, :], lambda_par) # Compute train/cross validation errors J_train, _ = linearRegCostFunction(X[0:i, :], y[0:i, :], theta, 0) J_CV, _ = linearRegCostFunction(Xval, yval, theta, 0) # store the results error_train[i - 1] = J_train error_val[i - 1] = J_CV return error_train, error_val
def validationCurve(X, y, Xval, yval): """Generates the train and validation errors needed to plot a validation curve that we can use to select lambda VALIDATIONCURVE(X, y, Xval, yval) returns the train and validation errors (in error_train, error_val) for different values of lambda. You are given the training set (X, y) and validation set (Xval, yval). """ # Selected values of lambda (you should not change this) lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]).reshape(-1, 1) # You need to return these variables correctly. error_train = np.zeros((len(lambda_vec), 1)) error_val = np.zeros((len(lambda_vec), 1)) for i in range(len(lambda_vec)): curr_lambda = lambda_vec[i] theta = trainLinearReg(X, y, curr_lambda) error_train[i], tmp = linearRegCostFunction(X, y, theta, 0) error_val[i], tmp = linearRegCostFunction(Xval, yval, theta, 0) return lambda_vec, error_train, error_val
def learningCurve(X, y, Xval, yval, Lambda): """returns the train and cross validation set errors for a learning curve. In particular, it returns two vectors of the same length - error_train and error_val. Then, error_train(i) contains the training error for i examples (and similarly for error_val(i)). In this function, you will compute the train and test errors for dataset sizes from 1 up to m. In practice, when working with larger datasets, you might want to do this in larger intervals. """ # Number of training examples m, _ = X.shape # You need to return these values correctly error_train = np.zeros(m) error_val = np.zeros(m) # ====================== YOUR CODE HERE ====================== # Instructions: Fill in this function to return training errors in # error_train and the cross validation errors in error_val. # i.e., error_train(i) and # error_val(i) should give you the errors # obtained after training on i examples. # # Note: You should evaluate the training error on the first i training # examples (i.e., X(1:i, :) and y(1:i)). # # For the cross-validation error, you should instead evaluate on # the _entire_ cross validation set (Xval and yval). # # Note: If you are using your cost function (linearRegCostFunction) # to compute the training and cross validation error, you should # call the function with the lambda argument set to 0. # Do note that you will still need to use lambda when running # the training to obtain the theta parameters. # # Hint: You can loop over the examples with the following: # # for i = 1:m # # Compute train/cross validation errors using training examples # # X(1:i, :) and y(1:i), storing the result in # # error_train(i) and error_val(i) # .... # # end # # ---------------------- Sample Solution ---------------------- for i in xrange(1, m + 1): theta = trainLinearReg(X[:i], y[:i], Lambda) error_train[i - 1] = linearRegCostFunction(X[:i], y[:i], theta, 0)[0] error_val[i - 1] = linearRegCostFunction(Xval, yval, theta, 0)[0] # ------------------------------------------------------------------------- # ========================================================================= return error_train, error_val
def learningCurve(X, y, Xval, yval, Lambda): """returns the train and cross validation set errors for a learning curve. In particular, it returns two vectors of the same length - error_train and error_val. Then, error_train(i) contains the training error for i examples (and similarly for error_val(i)). In this function, you will compute the train and test errors for dataset sizes from 1 up to m. In practice, when working with larger datasets, you might want to do this in larger intervals. """ m, _ = X.shape error_train = np.zeros(m) error_val = np.zeros(m) for i in range(1, m): theta = trainLinearReg(X[:i], y[:i], 0) error_train[i], _ = linearRegCostFunction(X[:i], y[:i], theta, 0) error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0) # Note: You should evaluate the training error on the first i training # examples (i.e., X(1:i, :) and y(1:i)). # For the cross-validation error, you should instead evaluate on # the _entire_ cross validation set (Xval and yval). # Note: If you are using your cost function (linearRegCostFunction) # to compute the training and cross validation error, you should # call the function with the lambda argument set to 0. # Do note that you will still need to use lambda when running # the training to obtain the theta parameters. return error_train, error_val
def trainLinearReg(X, y, reg_lambda): """TRAINLINEARREG Trains linear regression given a dataset (X, y) and a regularization parameter lambda [theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using the dataset (X, y) and regularization parameter lambda. Returns the trained parameters theta. """ #Initialize Theta initial_theta = np.zeros((X.shape[1], 1)) # Create "short hand" for the cost function to be minimized costFunction = lambda theta : linearRegCostFunction(X, y, theta, reg_lambda, returnOnlyCost = True) gradFunc = lambda theta : linearRegCostFunction(X, y, theta, reg_lambda, returnOnlyGrad = True, flattenResult = True) #should finish learning (reach local minima) in 4 iterations max_iter = 200 # Run fmincg to obtain the optimal theta Result = op.minimize(fun = costFunction, x0 = initial_theta, method = 'TNC', jac = gradFunc, options={'maxiter' : max_iter, 'disp': True}) optimal_theta = Result.x return optimal_theta
def validationCurve(theta_init, X, X_val, y, y_val): lmdaa = np.array([0., 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]) error_train = np.zeros(lmdaa.size) error_val = np.zeros(lmdaa.size) for i in range(lmdaa.size): lmd = lmdaa[i] theta = line_trainLinearReg(theta_init, X, y, lmd) error_train[i],_ ,_ = linearRegCostFunction(X, y, theta, lmd) error_val[i],_ ,_ = linearRegCostFunction(X_val, y_val, theta, lmd) return error_train, error_val
def trainLinearReg(X, y, lamda): initial_theta = np.zeros((X.shape[1], 1)) cost_function = lambda theta: linearRegCostFunction(X, y, theta, lamda)[0] grad_function = lambda theta: linearRegCostFunction(X, y, theta, lamda)[1] theta = fmin_cg(cost_function, initial_theta, fprime=grad_function, maxiter=50, disp=False) return theta
def trainLinearReg(X, y, lambda_): initial_theta = np.zeros((X.shape[1])) theta = opt.fmin_cg( f=lambda x: linearRegCostFunction(x, X, y, lambda_)[0], x0=initial_theta, fprime=lambda x: linearRegCostFunction(x, X, y, lambda_)[1], maxiter=200, disp=0) return theta
def learningCurve(X, y, Xval, yval, lambda_): m = len(y) error_train = np.zeros((X.shape[0], 1)).ravel() error_val = np.zeros((X.shape[0], 1)).ravel() theta = np.zeros((X.shape[1], 1)) for i in range(m): theta = trainLinearReg(X[:i+1, :], y[:i+1], lambda_) error_train[i] = linearRegCostFunction(theta, X[:i+1, :], y[:i+1], 0)[0] error_val[i] = linearRegCostFunction(theta, Xval, yval, 0)[0] return error_train, error_val
def learningCurve(x, y, xval, yval, _lambda): m, n = x.shape error_train = np.zeros((m, 1)) error_val = np.zeros((m, 1)) for i in range(0, m): x1 = x[0:i + 1, :] y1 = y[0:i + 1, :] result = trainLinearReg(x1, y1, _lambda) theta = result.x error_train[i] = linearRegCostFunction(theta, x1, y1, _lambda) error_val[i] = linearRegCostFunction(theta, xval, yval, _lambda) return error_train, error_val
def validationCurve(X, y, Xval, yval): #VALIDATIONCURVE Generate the train and validation errors needed to #plot a validation curve that we can use to select lambda # [lambda_vec, error_train, error_val] = ... # VALIDATIONCURVE(X, y, Xval, yval) returns the train # and validation errors (in error_train, error_val) # for different values of lambda. You are given the training set (X, # y) and validation set (Xval, yval). # # Selected values of lambda (you should not change this) lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]) # You need to return these variables correctly. error_train = np.zeros((len(lambda_vec), 1)) error_val = np.zeros((len(lambda_vec), 1)) # ====================== YOUR CODE HERE ====================== # Instructions: Fill in this function to return training errors in # error_train and the validation errors in error_val. The # vector lambda_vec contains the different lambda parameters # to use for each calculation of the errors, i.e, # error_train(i), and error_val(i) should give # you the errors obtained after training with # lambda = lambda_vec(i) # # Note: You can loop over lambda_vec with the following: # # for i = 1:length(lambda_vec) # lambda = lambda_vec(i); # # Compute train / val errors when training linear # # regression with regularization parameter lambda # # You should store the result in error_train(i) # # and error_val(i) # .... # # end # # for i in xrange(len(lambda_vec)): lambda_val = lambda_vec[i] # learn theta parameters with current lambda value theta = tlr.trainLinearReg(X, y, lambda_val) # fill in error_train[i] and error_val[i] # note that for error computation, we set lambda = 0 in the last argument error_train[i] = lrcf.linearRegCostFunction(X, y, theta, 0) error_val[i] = lrcf.linearRegCostFunction(Xval, yval, theta, 0) return lambda_vec, error_train, error_val
def validationCurve(X, y, Xval, yval): lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]) error_train = np.zeros(lambda_vec.size) error_val = np.zeros(lambda_vec.size) for i in range(lambda_vec.size): lamda = lambda_vec[i] theta = trainLinearReg(X, y, lamda) error_train[i], _ = linearRegCostFunction(X, y, theta, 0) error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0) return lambda_vec, error_train, error_val
def learningCurve(X, y, Xval, yval, reg): m = y.size error_train = np.zeros((m, 1)) error_val = np.zeros((m, 1)) for i in np.arange(m): res = trainLinearReg(X[:i + 1], y[:i + 1], reg) error_train[i] = linearRegCostFunction(res.x, X[:i + 1], y[:i + 1], reg) error_val[i] = linearRegCostFunction(res.x, Xval, yval, reg) return (error_train, error_val)
def learningCurve(X, y, Xval, yval, lamda): m = y.size error_train = np.zeros(m) error_val = np.zeros(m) for i in range(m): Xtrain = X[:i + 1, :] ytrain = y[:i + 1, :] theta = trainLinearReg(Xtrain, ytrain, lamda) error_train[i], _ = linearRegCostFunction(Xtrain, ytrain, theta, 0) error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0) return error_train, error_val
def validationCurve(X, y, Xval, yval): """returns the train and validation errors (in error_train, error_val) for different values of lambda. You are given the training set (X, y) and validation set (Xval, yval). """ # Selected values of lambda (you should not change this) lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]) # You need to return these variables correctly. error_train = np.zeros(lambda_vec.size) error_val = np.zeros(lambda_vec.size) # ====================== YOUR CODE HERE ====================== # Instructions: Fill in this function to return training errors in # error_train and the validation errors in error_val. The # vector lambda_vec contains the different lambda parameters # to use for each calculation of the errors, i.e, # error_train(i), and error_val(i) should give # you the errors obtained after training with # lambda = lambda_vec(i) # # Note: You can loop over lambda_vec with the following: # # for i in range(lambda_vec.size): # lambda = lambda_vec(i) # # Compute train / val errors when training linear # # regression with regularization parameter lambda # # You should store the result in error_train(i) # # and error_val(i) # .... # # end # # # for i in range(lambda_vec.size): # Lambda = lambda_vec[i] # error_train_i, error_val_i = learningCurve(X, y, Xval, yval, Lambda) # error_train[i] = np.mean(error_train_i) # error_val[i] = np.mean(error_val_i) for i in range(lambda_vec.size): lambda_val = lambda_vec[i] theta = trainLinearReg(X, y, lambda_val) error_train[i] = linearRegCostFunction(X, y, theta, 0)[0] error_val[i] = linearRegCostFunction(Xval, yval, theta, 0)[0] # ========================================================================= return lambda_vec, error_train, error_val
def ValidationCurve(X, y, Xval, yval): lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]) error_train = np.zeros_like(lambda_vec) error_val = np.zeros_like(lambda_vec) for i in range(0, lambda_vec.shape[0]): lambda_ = lambda_vec[i] theta = tlr.TrainLinearRegression(X, y, lambda_) error_train[i] = lrcf.linearRegCostFunction(X, y, theta, 0)[0] error_val[i] = lrcf.linearRegCostFunction(Xval, yval, theta, 0)[0] return lambda_vec, error_train, error_val
def validationCurve(X, y, Xval, yval): """returns the train and validation errors (in error_train, error_val) for different values of lambda. You are given the training set (X, y) and validation set (Xval, yval). """ # Selected values of lambda (you should not change this) lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]) m=lambda_vec.size # You need to return these variables correctly. error_train = np.zeros(lambda_vec.size) error_val = np.zeros(lambda_vec.size) # ====================== YOUR CODE HERE ====================== # Instructions: Fill in this function to return training errors in # error_train and the validation errors in error_val. The # vector lambda_vec contains the different lambda parameters # to use for each calculation of the errors, i.e, # error_train(i), and error_val(i) should give # you the errors obtained after training with # lambda = lambda_vec(i) # # Note: You can loop over lambda_vec with the following: # # for i = 1:length(lambda_vec) # lambda = lambda_vec(i) # # Compute train / val errors when training linear # # regression with regularization parameter lambda # # You should store the result in error_train(i) # # and error_val(i) # .... # # end # # # ========================================================================= error_train=np.zeros(m) error_val=np.zeros(m) for Lambda in lambda_vec: theta=trainLinearReg(X,y,Lambda) error_train[np.where(Lambda == lambda_vec)] = linearRegCostFunction(X,y, theta, 0)[0] error_val[np.where(Lambda == lambda_vec)] = linearRegCostFunction(Xval,yval, theta, 0)[0] return lambda_vec, error_train, error_val
def validationCurve(x, y, xval, yval): lambda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10] error_train = np.zeros((len(lambda_vec), 1)) error_val = np.zeros((len(lambda_vec), 1)) theta = np.zeros((len(lambda_vec), x.shape[1])) for i in range(0, len(lambda_vec)): result = trainLinearReg(x, y, lambda_vec[i]) #print(result.x) theta[i] = result.x result_val = trainLinearReg(xval, yval, lambda_vec[i]) error_train[i] = linearRegCostFunction(theta[i], x, y, 0) error_val[i] = linearRegCostFunction(theta[i], xval, yval, 0) return lambda_vec, error_train, error_val, theta
def learningCurve(X, y, Xval, yval, lamb): m = len(y) erro_train = np.zeros((m, 1)) erro_val = np.zeros((m, 1)) theta = np.zeros(2) for i in range(m): res = trainLinearReg(theta, X[:i + 1], y[:i + 1], lamb) erro_train[i] = linearRegCostFunction(res, X[:i + 1], y[:i + 1], lamb) erro_val[i] = linearRegCostFunction(res, Xval, yval, lamb) return(erro_train, erro_val)
def learningCurve(X, y, Xval, yval, lambda_val): m = len(X) error_train = np.zeros((m, 1)) error_val = np.zeros((m, 1)) for i in xrange(1, m + 1): X_train = X[:i] y_train = y[:i] theta = tlr.trainLinearReg(X_train, y_train, 1) error_train[i - 1] = lrcf.linearRegCostFunction( X_train, y_train, theta, 0) error_val[i - 1] = lrcf.linearRegCostFunction(Xval, yval, theta, 0) return error_train, error_val
def learningCurve(X, y, Xval, yval, xlambda): m = X.shape[0] # zero initial error_train = np.zeros([m, 1]) error_val = np.zeros([m, 1]) # start for n in range(0, m): theta = tLR.trainLinearReg(X[:n + 1, :], y[:n + 1], xlambda) error_train[n], grad = lRCF.linearRegCostFunction( theta, X[:n + 1, :], y[:n + 1], 0) error_val[n], grad = lRCF.linearRegCostFunction(theta, Xval, yval, 0) return error_train, error_val
def learningCurve(X, y, Xval, yval, lmbda): # Number of training examples m = X.shape[0] # You need to return these values correctly error_train = np.zeros((m, 1)) error_val = np.zeros((m, 1)) # ====================== YOUR CODE HERE ====================== # Instructions: Fill in this function to return training errors in # error_train and the cross validation errors in error_val. # i.e., error_train[i] and # error_val[i] should give you the errors # obtained after training on i examples. # # Note: You should evaluate the training error on the first i training # examples (i.e., X[:i+1, :] and y[:i+1]). # # For the cross-validation error, you should instead evaluate on # the _entire_ cross validation set (Xval and yval). # # Note: If you are using your cost function (linearRegCostFunction) # to compute the training and cross validation error, you should # call the function with the lmbda argument set to 0. # Do note that you will still need to use lmbda when running # the training to obtain the theta parameters. # # Hint: You can loop over the examples with the following: # # for i in range(m): # % Compute train/cross validation errors using training examples # % X[:i+1, :] and y[:i+1], storing the result in # % error_train[i] and error_val[i] # .... # # # ---------------------- Sample Solution ---------------------- for i in range(m): theta = trainLinearReg(X[:i + 1, :], y[:i + 1, :], lmbda) error_train[i] = linearRegCostFunction(theta, X[:i + 1, :], y[:i + 1, :], 0)[0] error_val[i] = linearRegCostFunction(theta, Xval, yval, 0)[0] # ------------------------------------------------------------- # ============================================================= return error_train, error_val
def validationCurve(X, y, Xval, yval): """" VALIDATIONCURVE Generate the train and validation errors needed to plot a validation curve that we can use to select lambda lambda_vec, error_train, error_val = VALIDATIONCURVE(X, y, Xval, yval) returns the train and validation errors (in error_train, error_val) for different values of lmbda. You are given the training set (X, y) and validation set (Xval, yval). """ # Selected values of lambda (you should not change this) lmbda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10] # You need to return these variables correctly. error_train = np.zeros((len(lmbda_vec), 1)) error_val = np.zeros((len(lmbda_vec), 1)) # ====================== YOUR CODE HERE ====================== # Instructions: Fill in this function to return training errors in # error_train and the validation errors in error_val. The # vector lmbda_vec contains the different lambda parameters # to use for each calculation of the errors, i.e, # error_train[i], and error_val[i] should give # you the errors obtained after training with # lmbda = lmbda_vec(i) # # Note: You can loop over lmbda_vec with the following: # # for i in range(len(lmbda_vec)): # lmbda = lmbda_vec[i] # % Compute train / val errors when training linear # % regression with regularization parameter lambda # % You should store the result in error_train[i] # % and error_val[i] # .... # # for i in range(len(lmbda_vec)): lmbda = lmbda_vec[i] theta = trainLinearReg(X, y, lmbda) error_train[i] = linearRegCostFunction(theta, X, y, 0)[0] error_val[i] = linearRegCostFunction(theta, Xval, yval, 0)[0] # ========================================================================= return (lmbda_vec, error_train, error_val)
def validationCurve(X, y, Xval, yval): # Selected values of lambda (you should not change this) lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]).reshape(-1,1) # You need to return these variables correctly. error_train = np.zeros(lambda_vec.shape[0]) error_val = np.zeros(lambda_vec.shape[0]) for i in range(lambda_vec.shape[0]): _lambda = lambda_vec[i] theta = trainLinearReg(X, y, _lambda) error_train[i], _ = linearRegCostFunction(X, y, theta, 0) error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0) return lambda_vec, error_train, error_val
def trainLinearReg(X, y, lambda_): #TRAINLINEARREG Trains linear regression given a dataset (X, y) and a #regularization parameter lambda # [theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using # the dataset (X, y) and regularization parameter lambda. Returns the # trained parameters theta. # # Initialize Theta initial_theta = zeros(size(X, 1)) # Create "short hand" for the cost function to be minimized costFunction = lambda t: linearRegCostFunction(X, y, t, lambda_) # Now, costFunction is a function that takes in only one argument # setup an iteration counter counter = count() # Minimize using CG algorithm res = minimize(costFunction, initial_theta, method='CG', jac=True, options={'maxiter': 200}, callback=lambda _: counter.next()) theta = res.x print "Iteration %5d | Cost: %e" % (counter.next(), res.fun) return theta
def trainLinearReg(X, y, lambda_value): #TRAINLINEARREG Trains linear regression given a dataset (X, y) and a #regularization parameter lambda # [theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using # the dataset (X, y) and regularization parameter lambda. Returns the # trained parameters theta. # # Initialize Theta initial_theta = np.zeros((X.shape[1], 1)) # Create "short hand" for the cost function to be minimized costFunction = lambda t: linearRegCostFunction(X, y, t, lambda_value) # Now, costFunction is a function that takes in only one argument options = {'maxiter': 200} # Minimize using fmincg theta = optimize.minimize(costFunction, initial_theta, jac=True, method='TNC', options=options).x return theta
def trainLinearReg(X, y, lamda): # Initialize Theta initial_theta = np.zeros((X.shape[1], 1)) # Create "short hand" for the cost function to be minimized costFunc = lambda w: linearRegCostFunction(X, y, w, lamda) # Now, costFunction is a function that takes in only one argument minimize_method = "CG" # CG, L-BFGS-B, TNC, ... opts = {'maxiter': 200} if minimize_method == "TNC": opts['maxCGit'] = 0 opts['stepmx'] = 500 elif minimize_method == 'L-BFGS-B': opts['eps'] = 1e-8 else: pass # Minimize using fmincg Result = op.minimize(fun=linearRegCostFunction, x0=initial_theta, method=minimize_method, args=(X, y, lamda), jac=True, options=opts) return Result.x
def learningCurve(X, y, Xval, yval, _lambda): # Number of training examples m = X.shape[0] # You need to return these values correctly error_train = np.zeros(m) error_val = np.zeros(m) for i in range(m): Xtrain = X[0:i+1] ytrain = y[0:i+1] theta = trainLinearReg(Xtrain, ytrain, _lambda) error_train[i], _ = linearRegCostFunction(Xtrain, ytrain, theta, 0) error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0) return error_train, error_val
def trainLinearReg(X, y, Lambda, method='CG', maxiter=200): """trains linear regression using the dataset (X, y) and regularization parameter lambda. Returns the trained parameters theta. """ # Initialize Theta initial_theta = np.zeros(X.shape[1]) # Create "short hand" for the cost function to be minimized costFunction = lambda t: linearRegCostFunction(X, y, t, Lambda)[0] gradFunction = lambda t: linearRegCostFunction(X, y, t, Lambda)[1] result = minimize(costFunction, initial_theta, method=method, jac=None, options={'disp': True, 'maxiter': maxiter}) return result.x
def validationCurve(X, y, Xval, yval): """ VALIDATIONCURVE Generate the train and validation errors needed to plot a validation curve that we can use to select lambda [lambda_vec, error_train, error_val] = ... VALIDATIONCURVE(X, y, Xval, yval) returns the train and validation errors (in error_train, error_val) for different values of lambda. You are given the training set (X, y) and validation set (Xval, yval). Args X: numpy array (X train) y: numpy array (y train) Xval: numpy array (X validation) yval: numpy array (y validation) Return lambda_vec: numpy array (selected values of lambda) error_train: numpy array (train errors) error_val: numpy array (val errors) """ # Selected values of lambda (you should not change this) lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]) # You need to return these variables correctly. error_train = np.zeros((len(lambda_vec), 1)) error_val = np.zeros((len(lambda_vec), 1)) # ====================== YOUR CODE HERE ====================== # Instructions: Fill in this function to return training errors in # error_train and the validation errors in error_val. The # vector lambda_vec contains the different lambda parameters # to use for each calculation of the errors, i.e, # error_train(i), and error_val(i) should give # you the errors obtained after training with # lambda = lambda_vec(i) for i in range(len(lambda_vec)): lbd = lambda_vec[i] theta = trainLinearReg(X, y, lbd) error_train[i] = linearRegCostFunction(X, y, theta, 0)[0] # lambda = 0 error_val[i] = linearRegCostFunction(Xval, yval, theta, 0)[0] # lambda = 0 return lambda_vec, error_train, error_val
def validationCurve(X, y, Xval, yval): lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]).reshape(10, 1) error_train = np.zeros((lambda_vec.shape[0], 1)) error_val = np.zeros((lambda_vec.shape[0], 1)) for i in range(lambda_vec.shape[0]): lambda_ = lambda_vec[i] # train with lambda[i] to get theta[i] theta = trainLinearReg(X, y, lambda_) # Get error training and error validation error_train[i] = linearRegCostFunction(theta, X, y, 0)[0] error_val[i] = linearRegCostFunction(theta, Xval, yval, 0)[0] return lambda_vec, error_train, error_val
def LearningCurve(X, y, Xval, yval, lambda_): #In this function, you will compute the train and test errors for #dataset sizes from 1 up to m. In practice, when working with larger #datasets, you might want to do this in larger intervals. [m, n] = X.shape error_train = np.zeros((m + 1, 1)) error_cross_validation = np.zeros((m + 1, 1)) for i in range(1, m + 1): X_train = X[0:i, :] y_train = y[0:i] theta = tlr.TrainLinearRegression(X_train, y_train, lambda_) error_train[i] = lrcf.linearRegCostFunction(X_train, y_train, theta, lambda_)[0] error_cross_validation[i] = lrcf.linearRegCostFunction( Xval, yval, theta, lambda_)[0] return error_train, error_cross_validation
def gd_reglinear_reg(X, y, alpha, lamb, epochs, theta): cost = np.zeros(epochs) for i in range(epochs): gradient = gradiente_linear_reg(X, y, lamb, theta) theta = theta - (alpha * gradient) cost[i] = linearRegCostFunction(X, y, theta, lamb) return cost[-1], theta
def trainLinearReg(X, y, _lambda): # Initialize Theta initial_theta = np.zeros(X.shape[1]) # Create "short hand" for the cost function to be minimized costFunction = lambda t: linearRegCostFunction(X, y, t, _lambda) res = minimize(costFunction, initial_theta, method='BFGS', jac=True, tol=1e-2) print(res) return res.x
def output(partId): # Random Test Cases X = column_stack([ones(10), sin(arange(1,15,1.5)), cos(arange(1,15,1.5))]) y = sin(arange(1,30,3)) Xval = column_stack([ones(10), sin(arange(0,14,1.5)), cos(arange(0,14,1.5))]) yval = sin(arange(1,11)) if partId == '1': J, _ = linearRegCostFunction(X, y, hstack([0.1, 0.2, 0.3]), 0.5) return sprintf('%0.5f ', J) elif partId == '2': J, grad = linearRegCostFunction(X, y, hstack([0.1, 0.2, 0.3]), 0.5) return sprintf('%0.5f ', grad) elif partId == '3': error_train, error_val = \ learningCurve(X, y, Xval, yval, 1); return sprintf('%0.5f ', vstack([error_train, error_val])) elif partId == '4': X_poly = polyFeatures(X[1,:], 8) return sprintf('%0.5f ', X_poly) elif partId == '5': lambda_vec, error_train, error_val = \ validationCurve(X, y, Xval, yval) return sprintf('%0.5f ', \ vstack([lambda_vec, error_train, error_val]))
def trainLinearReg(X, y, lambda_): #TRAINLINEARREG Trains linear regression given a dataset (X, y) and a #regularization parameter lambda # [theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using # the dataset (X, y) and regularization parameter lambda. Returns the # trained parameters theta. # # Initialize Theta initial_theta = zeros(size(X, 1)) # Create "short hand" for the cost function to be minimized costFunction = lambda t: linearRegCostFunction(X, y, t, lambda_) # Now, costFunction is a function that takes in only one argument # setup an iteration counter counter = count() # Minimize using CG algorithm res = minimize(costFunction, initial_theta, method='CG', jac=True, options={'maxiter': 200}, callback=lambda _:counter.next()) theta = res.x print "Iteration %5d | Cost: %e" % (counter.next(), res.fun) return theta
# Plot training data plt.figure() plt.plot(X, y, 'rx', ms=5, linewidth=1.5) plt.xlabel('Change in water level (x)') plt.ylabel('Water flowing out of the dam (y)') input('Program paused. Press enter to continue.\n') ## =========== Part 2: Regularized Linear Regression Cost ============= # You should now implement the cost function for regularized linear # regression. # theta = np.ones(2) J, _ = linearRegCostFunction(_X, y, theta, 1) print('Cost at theta = [1 ; 1]: %f ' \ '\n(this value should be about 303.993192)\n'%J) input('Program paused. Press enter to continue.\n') ## =========== Part 3: Regularized Linear Regression Gradient ============= # You should now implement the gradient for regularized linear # regression. # theta = np.ones(2) J, grad = linearRegCostFunction(_X, y, theta, 1) print(grad)
# Plot training data fig = plt.figure() plt.plot(X, y, 'rx', linewidth=1.5, markersize=10) plt.xlabel('Change in water level (x)') plt.ylabel('Water flowing out of the dam (y)') input('Program paused. Press enter to continue.') # =========== Part 2: Regularized Linear Regression Cost ============= # You should now implement the cost function for regularized linear # regression. # theta = np.ones((n + 1)) J, grad = linearRegCostFunction( theta, np.hstack((np.ones((m, 1)), X)), y, 1) print('Cost at theta = [1, 1]: %f' % (J)) print('(this value should be about 303.993192)') input('Program paused. Press enter to continue.') # =========== Part 3: Regularized Linear Regression Gradient ============= # You should now implement the gradient for regularized linear # regression. # theta = np.ones((n + 1)) J, grad = linearRegCostFunction( theta, np.hstack((np.ones((m, 1)), X)), y, 1)
def learningCurve(X, y, Xval, yval, Lambda): """returns the train and cross validation set errors for a learning curve. In particular, it returns two vectors of the same length - error_train and error_val. Then, error_train(i) contains the training error for i examples (and similarly for error_val(i)). In this function, you will compute the train and test errors for dataset sizes from 1 up to m. In practice, when working with larger datasets, you might want to do this in larger intervals. """ # Number of training examples m, _ = X.shape # You need to return these values correctly error_train = np.zeros(m) error_val = np.zeros(m) # ====================== YOUR CODE HERE ====================== # Instructions: Fill in this function to return training errors in # error_train and the cross validation errors in error_val. # i.e., error_train(i) and # error_val(i) should give you the errors # obtained after training on i examples. # # Note: You should evaluate the training error on the first i training # examples (i.e., X(1:i, :) and y(1:i)). # # For the cross-validation error, you should instead evaluate on # the _entire_ cross validation set (Xval and yval). # # Note: If you are using your cost function (linearRegCostFunction) # to compute the training and cross validation error, you should # call the function with the lambda argument set to 0. # Do note that you will still need to use lambda when running # the training to obtain the theta parameters. # # Hint: You can loop over the examples with the following: # # for i = 1:m # # Compute train/cross validation errors using training examples # # X(1:i, :) and y(1:i), storing the result in # # error_train(i) and error_val(i) # .... # # end # # ---------------------- Sample Solution ---------------------- #X = np.column_stack((np.ones(m), X)) #Xval = np.column_stack((np.ones(Xval.shape), Xval)) for i in range(m): theta = trainLinearReg(X[:i+1],y[:i+1],Lambda) error_train[i] = linearRegCostFunction(X[:i+1],y[:i+1], theta, 0)[0] error_val[i] = linearRegCostFunction(Xval,yval, theta, 0)[0] # ------------------------------------------------------------------------- # ========================================================================= return error_train, error_val
m = X.size # Plot training data plt.scatter(X, y, marker='x', s=60, edgecolor='r', lw=1.5) plt.ylabel('Water flowing out of the dam (y)') # Set the y-axis label plt.xlabel('Change in water level (x)') # Set the x-axis label raw_input("Program paused. Press Enter to continue...") ## =========== Part 2: Regularized Linear Regression Cost ============= # You should now implement the cost function for regularized linear # regression. # theta = np.array([1, 1]) J = linearRegCostFunction(np.column_stack((np.ones(m), X)), y, theta, 1)[0] print 'Cost at theta = [1 1]: %f \n(this value should be about 303.993192)\n' % J raw_input("Program paused. Press Enter to continue...") ## =========== Part 3: Regularized Linear Regression Gradient ============= # You should now implement the gradient for regularized linear # regression. # theta = np.array([1, 1]) J, grad = linearRegCostFunction(np.column_stack((np.ones(m), X)), y, theta, 1) print 'Gradient at theta = [1 1]: [%f %f] \n(this value should be about [-15.303016 598.250744])\n' %(grad[0], grad[1])
plt.show(block=False) print('Program paused. Press enter to continue.\n') pause() """ =========== Part 2: Regularized Linear Regression Cost ============= You should now implement the cost function for regularized linear regression. """ theta = np.array([[1], [1]]) new_input = np.c_[np.ones((m, 1)), X] J, grad = linearRegCostFunction(new_input, y, theta, 1) print('Cost at theta = [1 ; 1]: {} \n(this value should be about 303.993192)\n'.format(J)) print('Program paused. Press enter to continue.\n') pause() """ =========== Part 3: Regularized Linear Regression Gradient ============= You should now implement the gradient for regularized linear regression. """ print('Gradient at theta = [1 ; 1]: [{}; {}] \n(this value should be about [-15.303016; 598.250744])\n'.format(grad[0], grad[1]))
# Plot training data fig = figure() plot(X, y, 'rx', markersize=10, linewidth=1.5) xlabel('Change in water level (x)') ylabel('Water flowing out of the dam (y)') fig.show() print 'Program paused. Press enter to continue.' raw_input() ## =========== Part 2: Regularized Linear Regression Cost ============= # You should now implement the cost function for regularized linear # regression. # theta = array([1., 1.]) J, _ = linearRegCostFunction(addOnes(X), y, theta, 1.) print 'Cost at theta = [1, 1]: %f ' % J print '(this value should be about 303.993192)' print 'Program paused. Press enter to continue.' raw_input() ## =========== Part 3: Regularized Linear Regression Gradient ============= # You should now implement the gradient for regularized linear # regression. # theta = array([1., 1.]) J, grad = linearRegCostFunction(addOnes(X), y, theta, 1.)