def validationCurve(X, y, Xval, yval):
'''
[lambda_vec, error_train, error_val] =
VALIDATIONCURVE(X, y, Xval, yval) returns the train
and validation errors (in error_train, error_val)
for different values of lambda. You are given the training set (X,
y) and validation set (Xval, yval).
'''
import numpy as np
from trainLinearReg import trainLinearReg
from linearRegCostFunction import linearRegCostFunction
# Selected values of lambda (you should not change this)
lambda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
# You need to return these variables correctly.
error_train = np.zeros((len(lambda_vec), 1))
error_val = np.zeros((len(lambda_vec), 1))
for i in range(len(lambda_vec)):
lambda_par = lambda_vec[i]
# Compute train / val errors when training linear
# regression with regularization parameter lambda
theta = trainLinearReg(X,y,lambda_par)
error_train[i], _ = linearRegCostFunction(X, y, theta, 0)
error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0)
return lambda_vec, error_train, error_val
def polynomialDegreeCurve(X, y, Xval, yval, reg_lambda):
"""Error cruve in function of degree of polynimal d
"""
dimensions = np.arange(1, 80).reshape(-1, 1)
# You need to return these variables correctly.
error_train = np.zeros((len(dimensions), 1))
error_val = np.zeros((len(dimensions), 1))
m_train_set = X.shape[0]
m_val_set = Xval.shape[0]
for i in range(len(dimensions)):
dimension = dimensions[i]
X_poly = polyFeatures(X, dimension)
X_poly, mu, sigma = featureNormalize(X_poly) # Normalize
X_poly = np.c_[np.ones((m_train_set, 1)), X_poly]
X_poly_val = polyFeatures(Xval, dimension)
X_poly_val = X_poly_val - mu
X_poly_val = X_poly_val / sigma
X_poly_val = np.c_[np.ones((m_val_set, 1)), X_poly_val]
theta = trainLinearReg(X_poly, y, reg_lambda)
error_train[i], tmp = linearRegCostFunction(X_poly, y, theta, 0)
error_val[i], tmp = linearRegCostFunction(X_poly_val, yval, theta, 0)
return dimensions, error_train, error_val
def learningCurve(X, y, Xval, yval, lambda_par):
'''
LEARNINGCURVE(X, y, Xval, yval, lambda) returns the train and
cross validation set errors for a learning curve. In particular,
it returns two vectors of the same length - error_train and
error_val. Then, error_train(i) contains the training error for
i examples (and similarly for error_val(i)).
'''
import numpy as np
from trainLinearReg import trainLinearReg
from linearRegCostFunction import linearRegCostFunction
# In this function, you will compute the train and test errors for
# dataset sizes from 1 up to m. In practice, when working with larger
# datasets, you might want to do this in larger intervals.
# Number of training examples
m = X.shape[0]
# You need to return these values correctly
error_train = np.zeros((m, 1))
error_val = np.zeros((m, 1))
# Loop over the examples with the following:
for i in range(1, m + 1):
# Use the trainLinearReg to find the theta parameters
theta = trainLinearReg(X[0:i, :], y[0:i, :], lambda_par)
# Compute train/cross validation errors
J_train, _ = linearRegCostFunction(X[0:i, :], y[0:i, :], theta, 0)
J_CV, _ = linearRegCostFunction(Xval, yval, theta, 0)
# store the results
error_train[i - 1] = J_train
error_val[i - 1] = J_CV
return error_train, error_val
def polynomialDegreeCurve(X, y, Xval, yval, reg_lambda):
"""Error cruve in function of degree of polynimal d
"""
dimensions = np.arange(1, 80).reshape(-1, 1)
# You need to return these variables correctly.
error_train = np.zeros((len(dimensions), 1))
error_val = np.zeros((len(dimensions), 1))
m_train_set = X.shape[0]
m_val_set = Xval.shape[0]
for i in range(len(dimensions)):
dimension = dimensions[i]
X_poly = polyFeatures(X, dimension)
X_poly, mu, sigma = featureNormalize(X_poly) # Normalize
X_poly = np.c_[np.ones((m_train_set, 1)), X_poly]
X_poly_val = polyFeatures(Xval, dimension)
X_poly_val = X_poly_val - mu
X_poly_val = X_poly_val / sigma
X_poly_val = np.c_[np.ones((m_val_set, 1)), X_poly_val]
theta = trainLinearReg(X_poly, y, reg_lambda)
error_train[i], tmp = linearRegCostFunction(X_poly, y, theta, 0)
error_val[i], tmp = linearRegCostFunction(X_poly_val, yval, theta, 0)
return dimensions, error_train, error_val
def validationCurve(X, y, Xval, yval):
"""Generates the train and validation errors needed to
plot a validation curve that we can use to select lambda
VALIDATIONCURVE(X, y, Xval, yval) returns the train
and validation errors (in error_train, error_val)
for different values of lambda. You are given the training set (X,
y) and validation set (Xval, yval).
"""
# Selected values of lambda (you should not change this)
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]).reshape(-1, 1)
# You need to return these variables correctly.
error_train = np.zeros((len(lambda_vec), 1))
error_val = np.zeros((len(lambda_vec), 1))
for i in range(len(lambda_vec)):
curr_lambda = lambda_vec[i]
theta = trainLinearReg(X, y, curr_lambda)
error_train[i], tmp = linearRegCostFunction(X, y, theta, 0)
error_val[i], tmp = linearRegCostFunction(Xval, yval, theta, 0)
return lambda_vec, error_train, error_val
def learningCurve(X, y, Xval, yval, Lambda):
"""returns the train and
cross validation set errors for a learning curve. In particular,
it returns two vectors of the same length - error_train and
error_val. Then, error_train(i) contains the training error for
i examples (and similarly for error_val(i)).
In this function, you will compute the train and test errors for
dataset sizes from 1 up to m. In practice, when working with larger
datasets, you might want to do this in larger intervals.
"""
# Number of training examples
m, _ = X.shape
# You need to return these values correctly
error_train = np.zeros(m)
error_val = np.zeros(m)
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return training errors in
# error_train and the cross validation errors in error_val.
# i.e., error_train(i) and
# error_val(i) should give you the errors
# obtained after training on i examples.
#
# Note: You should evaluate the training error on the first i training
# examples (i.e., X(1:i, :) and y(1:i)).
#
# For the cross-validation error, you should instead evaluate on
# the _entire_ cross validation set (Xval and yval).
#
# Note: If you are using your cost function (linearRegCostFunction)
# to compute the training and cross validation error, you should
# call the function with the lambda argument set to 0.
# Do note that you will still need to use lambda when running
# the training to obtain the theta parameters.
#
# Hint: You can loop over the examples with the following:
#
# for i = 1:m
# # Compute train/cross validation errors using training examples
# # X(1:i, :) and y(1:i), storing the result in
# # error_train(i) and error_val(i)
# ....
#
# end
#
# ---------------------- Sample Solution ----------------------
for i in xrange(1, m + 1):
theta = trainLinearReg(X[:i], y[:i], Lambda)
error_train[i - 1] = linearRegCostFunction(X[:i], y[:i], theta, 0)[0]
error_val[i - 1] = linearRegCostFunction(Xval, yval, theta, 0)[0]
# -------------------------------------------------------------------------
# =========================================================================
return error_train, error_val
def learningCurve(X, y, Xval, yval, Lambda):
"""returns the train and
cross validation set errors for a learning curve. In particular,
it returns two vectors of the same length - error_train and
error_val. Then, error_train(i) contains the training error for
i examples (and similarly for error_val(i)).
In this function, you will compute the train and test errors for
dataset sizes from 1 up to m. In practice, when working with larger
datasets, you might want to do this in larger intervals.
"""
m, _ = X.shape
error_train = np.zeros(m)
error_val = np.zeros(m)
for i in range(1, m):
theta = trainLinearReg(X[:i], y[:i], 0)
error_train[i], _ = linearRegCostFunction(X[:i], y[:i], theta, 0)
error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0)
# Note: You should evaluate the training error on the first i training
# examples (i.e., X(1:i, :) and y(1:i)).
# For the cross-validation error, you should instead evaluate on
# the _entire_ cross validation set (Xval and yval).
# Note: If you are using your cost function (linearRegCostFunction)
# to compute the training and cross validation error, you should
# call the function with the lambda argument set to 0.
# Do note that you will still need to use lambda when running
# the training to obtain the theta parameters.
return error_train, error_val
def trainLinearReg(X, y, reg_lambda):
"""TRAINLINEARREG Trains linear regression given a dataset (X, y) and a
regularization parameter lambda
[theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using
the dataset (X, y) and regularization parameter lambda. Returns the
trained parameters theta.
"""
#Initialize Theta
initial_theta = np.zeros((X.shape[1], 1))
# Create "short hand" for the cost function to be minimized
costFunction = lambda theta : linearRegCostFunction(X, y, theta, reg_lambda, returnOnlyCost = True)
gradFunc = lambda theta : linearRegCostFunction(X, y, theta, reg_lambda, returnOnlyGrad = True, flattenResult = True)
#should finish learning (reach local minima) in 4 iterations
max_iter = 200
# Run fmincg to obtain the optimal theta
Result = op.minimize(fun = costFunction, x0 = initial_theta, method = 'TNC', jac = gradFunc,
options={'maxiter' : max_iter, 'disp': True})
optimal_theta = Result.x
return optimal_theta
def validationCurve(theta_init, X, X_val, y, y_val):
lmdaa = np.array([0., 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10])
error_train = np.zeros(lmdaa.size)
error_val = np.zeros(lmdaa.size)
for i in range(lmdaa.size):
lmd = lmdaa[i]
theta = line_trainLinearReg(theta_init, X, y, lmd)
error_train[i],_ ,_ = linearRegCostFunction(X, y, theta, lmd)
error_val[i],_ ,_ = linearRegCostFunction(X_val, y_val, theta, lmd)
return error_train, error_val
def trainLinearReg(X, y, lamda):
initial_theta = np.zeros((X.shape[1], 1))
cost_function = lambda theta: linearRegCostFunction(X, y, theta, lamda)[0]
grad_function = lambda theta: linearRegCostFunction(X, y, theta, lamda)[1]
theta = fmin_cg(cost_function, initial_theta, fprime=grad_function,
maxiter=50, disp=False)
return theta
def trainLinearReg(X, y, lambda_):
initial_theta = np.zeros((X.shape[1]))
theta = opt.fmin_cg(
f=lambda x: linearRegCostFunction(x, X, y, lambda_)[0],
x0=initial_theta,
fprime=lambda x: linearRegCostFunction(x, X, y, lambda_)[1],
maxiter=200,
disp=0)
return theta
def learningCurve(X, y, Xval, yval, lambda_):
m = len(y)
error_train = np.zeros((X.shape[0], 1)).ravel()
error_val = np.zeros((X.shape[0], 1)).ravel()
theta = np.zeros((X.shape[1], 1))
for i in range(m):
theta = trainLinearReg(X[:i+1, :], y[:i+1], lambda_)
error_train[i] = linearRegCostFunction(theta, X[:i+1, :], y[:i+1], 0)[0]
error_val[i] = linearRegCostFunction(theta, Xval, yval, 0)[0]
return error_train, error_val
def learningCurve(x, y, xval, yval, _lambda):
m, n = x.shape
error_train = np.zeros((m, 1))
error_val = np.zeros((m, 1))
for i in range(0, m):
x1 = x[0:i + 1, :]
y1 = y[0:i + 1, :]
result = trainLinearReg(x1, y1, _lambda)
theta = result.x
error_train[i] = linearRegCostFunction(theta, x1, y1, _lambda)
error_val[i] = linearRegCostFunction(theta, xval, yval, _lambda)
return error_train, error_val
def validationCurve(X, y, Xval, yval):
#VALIDATIONCURVE Generate the train and validation errors needed to
#plot a validation curve that we can use to select lambda
# [lambda_vec, error_train, error_val] = ...
# VALIDATIONCURVE(X, y, Xval, yval) returns the train
# and validation errors (in error_train, error_val)
# for different values of lambda. You are given the training set (X,
# y) and validation set (Xval, yval).
#
# Selected values of lambda (you should not change this)
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10])
# You need to return these variables correctly.
error_train = np.zeros((len(lambda_vec), 1))
error_val = np.zeros((len(lambda_vec), 1))
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return training errors in
# error_train and the validation errors in error_val. The
# vector lambda_vec contains the different lambda parameters
# to use for each calculation of the errors, i.e,
# error_train(i), and error_val(i) should give
# you the errors obtained after training with
# lambda = lambda_vec(i)
#
# Note: You can loop over lambda_vec with the following:
#
# for i = 1:length(lambda_vec)
# lambda = lambda_vec(i);
# # Compute train / val errors when training linear
# # regression with regularization parameter lambda
# # You should store the result in error_train(i)
# # and error_val(i)
# ....
#
# end
#
#
for i in xrange(len(lambda_vec)):
lambda_val = lambda_vec[i]
# learn theta parameters with current lambda value
theta = tlr.trainLinearReg(X, y, lambda_val)
# fill in error_train[i] and error_val[i]
# note that for error computation, we set lambda = 0 in the last argument
error_train[i] = lrcf.linearRegCostFunction(X, y, theta, 0)
error_val[i] = lrcf.linearRegCostFunction(Xval, yval, theta, 0)
return lambda_vec, error_train, error_val
def validationCurve(X, y, Xval, yval):
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10])
error_train = np.zeros(lambda_vec.size)
error_val = np.zeros(lambda_vec.size)
for i in range(lambda_vec.size):
lamda = lambda_vec[i]
theta = trainLinearReg(X, y, lamda)
error_train[i], _ = linearRegCostFunction(X, y, theta, 0)
error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0)
return lambda_vec, error_train, error_val
def validationCurve(X, y, Xval, yval):
#VALIDATIONCURVE Generate the train and validation errors needed to
#plot a validation curve that we can use to select lambda
# [lambda_vec, error_train, error_val] = ...
# VALIDATIONCURVE(X, y, Xval, yval) returns the train
# and validation errors (in error_train, error_val)
# for different values of lambda. You are given the training set (X,
# y) and validation set (Xval, yval).
#
# Selected values of lambda (you should not change this)
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10])
# You need to return these variables correctly.
error_train = np.zeros((len(lambda_vec), 1))
error_val = np.zeros((len(lambda_vec), 1))
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return training errors in
# error_train and the validation errors in error_val. The
# vector lambda_vec contains the different lambda parameters
# to use for each calculation of the errors, i.e,
# error_train(i), and error_val(i) should give
# you the errors obtained after training with
# lambda = lambda_vec(i)
#
# Note: You can loop over lambda_vec with the following:
#
# for i = 1:length(lambda_vec)
# lambda = lambda_vec(i);
# # Compute train / val errors when training linear
# # regression with regularization parameter lambda
# # You should store the result in error_train(i)
# # and error_val(i)
# ....
#
# end
#
#
for i in xrange(len(lambda_vec)):
lambda_val = lambda_vec[i]
# learn theta parameters with current lambda value
theta = tlr.trainLinearReg(X, y, lambda_val)
# fill in error_train[i] and error_val[i]
# note that for error computation, we set lambda = 0 in the last argument
error_train[i] = lrcf.linearRegCostFunction(X, y, theta, 0)
error_val[i] = lrcf.linearRegCostFunction(Xval, yval, theta, 0)
return lambda_vec, error_train, error_val
def learningCurve(X, y, Xval, yval, reg):
m = y.size
error_train = np.zeros((m, 1))
error_val = np.zeros((m, 1))
for i in np.arange(m):
res = trainLinearReg(X[:i + 1], y[:i + 1], reg)
error_train[i] = linearRegCostFunction(res.x, X[:i + 1], y[:i + 1],
reg)
error_val[i] = linearRegCostFunction(res.x, Xval, yval, reg)
return (error_train, error_val)
def trainLinearReg(X, y, lamda):
initial_theta = np.zeros((X.shape[1], 1))
cost_function = lambda theta: linearRegCostFunction(X, y, theta, lamda)[0]
grad_function = lambda theta: linearRegCostFunction(X, y, theta, lamda)[1]
theta = fmin_cg(cost_function,
initial_theta,
fprime=grad_function,
maxiter=50,
disp=False)
return theta
def learningCurve(X, y, Xval, yval, lamda):
m = y.size
error_train = np.zeros(m)
error_val = np.zeros(m)
for i in range(m):
Xtrain = X[:i + 1, :]
ytrain = y[:i + 1, :]
theta = trainLinearReg(Xtrain, ytrain, lamda)
error_train[i], _ = linearRegCostFunction(Xtrain, ytrain, theta, 0)
error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0)
return error_train, error_val
def validationCurve(X, y, Xval, yval):
"""returns the train
and validation errors (in error_train, error_val)
for different values of lambda. You are given the training set (X,
y) and validation set (Xval, yval).
"""
# Selected values of lambda (you should not change this)
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10])
# You need to return these variables correctly.
error_train = np.zeros(lambda_vec.size)
error_val = np.zeros(lambda_vec.size)
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return training errors in
# error_train and the validation errors in error_val. The
# vector lambda_vec contains the different lambda parameters
# to use for each calculation of the errors, i.e,
# error_train(i), and error_val(i) should give
# you the errors obtained after training with
# lambda = lambda_vec(i)
#
# Note: You can loop over lambda_vec with the following:
#
# for i in range(lambda_vec.size):
# lambda = lambda_vec(i)
# # Compute train / val errors when training linear
# # regression with regularization parameter lambda
# # You should store the result in error_train(i)
# # and error_val(i)
# ....
#
# end
#
#
# for i in range(lambda_vec.size):
# Lambda = lambda_vec[i]
# error_train_i, error_val_i = learningCurve(X, y, Xval, yval, Lambda)
# error_train[i] = np.mean(error_train_i)
# error_val[i] = np.mean(error_val_i)
for i in range(lambda_vec.size):
lambda_val = lambda_vec[i]
theta = trainLinearReg(X, y, lambda_val)
error_train[i] = linearRegCostFunction(X, y, theta, 0)[0]
error_val[i] = linearRegCostFunction(Xval, yval, theta, 0)[0]
# =========================================================================
return lambda_vec, error_train, error_val
def ValidationCurve(X, y, Xval, yval):
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10])
error_train = np.zeros_like(lambda_vec)
error_val = np.zeros_like(lambda_vec)
for i in range(0, lambda_vec.shape[0]):
lambda_ = lambda_vec[i]
theta = tlr.TrainLinearRegression(X, y, lambda_)
error_train[i] = lrcf.linearRegCostFunction(X, y, theta, 0)[0]
error_val[i] = lrcf.linearRegCostFunction(Xval, yval, theta, 0)[0]
return lambda_vec, error_train, error_val
def validationCurve(X, y, Xval, yval):
"""returns the train
and validation errors (in error_train, error_val)
for different values of lambda. You are given the training set (X,
y) and validation set (Xval, yval).
"""
# Selected values of lambda (you should not change this)
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10])
m=lambda_vec.size
# You need to return these variables correctly.
error_train = np.zeros(lambda_vec.size)
error_val = np.zeros(lambda_vec.size)
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return training errors in
# error_train and the validation errors in error_val. The
# vector lambda_vec contains the different lambda parameters
# to use for each calculation of the errors, i.e,
# error_train(i), and error_val(i) should give
# you the errors obtained after training with
# lambda = lambda_vec(i)
#
# Note: You can loop over lambda_vec with the following:
#
# for i = 1:length(lambda_vec)
# lambda = lambda_vec(i)
# # Compute train / val errors when training linear
# # regression with regularization parameter lambda
# # You should store the result in error_train(i)
# # and error_val(i)
# ....
#
# end
#
#
# =========================================================================
error_train=np.zeros(m)
error_val=np.zeros(m)
for Lambda in lambda_vec:
theta=trainLinearReg(X,y,Lambda)
error_train[np.where(Lambda == lambda_vec)] = linearRegCostFunction(X,y, theta, 0)[0]
error_val[np.where(Lambda == lambda_vec)] = linearRegCostFunction(Xval,yval, theta, 0)[0]
return lambda_vec, error_train, error_val
def validationCurve(x, y, xval, yval):
lambda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
error_train = np.zeros((len(lambda_vec), 1))
error_val = np.zeros((len(lambda_vec), 1))
theta = np.zeros((len(lambda_vec), x.shape[1]))
for i in range(0, len(lambda_vec)):
result = trainLinearReg(x, y, lambda_vec[i])
#print(result.x)
theta[i] = result.x
result_val = trainLinearReg(xval, yval, lambda_vec[i])
error_train[i] = linearRegCostFunction(theta[i], x, y, 0)
error_val[i] = linearRegCostFunction(theta[i], xval, yval, 0)
return lambda_vec, error_train, error_val, theta
def learningCurve(X, y, Xval, yval, lamb):
m = len(y)
erro_train = np.zeros((m, 1))
erro_val = np.zeros((m, 1))
theta = np.zeros(2)
for i in range(m):
res = trainLinearReg(theta, X[:i + 1], y[:i + 1], lamb)
erro_train[i] = linearRegCostFunction(res, X[:i + 1],
y[:i + 1], lamb)
erro_val[i] = linearRegCostFunction(res, Xval, yval, lamb)
return(erro_train, erro_val)
def learningCurve(X, y, Xval, yval, lambda_val):
m = len(X)
error_train = np.zeros((m, 1))
error_val = np.zeros((m, 1))
for i in xrange(1, m + 1):
X_train = X[:i]
y_train = y[:i]
theta = tlr.trainLinearReg(X_train, y_train, 1)
error_train[i - 1] = lrcf.linearRegCostFunction(
X_train, y_train, theta, 0)
error_val[i - 1] = lrcf.linearRegCostFunction(Xval, yval, theta, 0)
return error_train, error_val
def learningCurve(X, y, Xval, yval, xlambda):
m = X.shape[0]
# zero initial
error_train = np.zeros([m, 1])
error_val = np.zeros([m, 1])
# start
for n in range(0, m):
theta = tLR.trainLinearReg(X[:n + 1, :], y[:n + 1], xlambda)
error_train[n], grad = lRCF.linearRegCostFunction(
theta, X[:n + 1, :], y[:n + 1], 0)
error_val[n], grad = lRCF.linearRegCostFunction(theta, Xval, yval, 0)
return error_train, error_val
def learningCurve(X, y, Xval, yval, lmbda):
# Number of training examples
m = X.shape[0]
# You need to return these values correctly
error_train = np.zeros((m, 1))
error_val = np.zeros((m, 1))
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return training errors in
# error_train and the cross validation errors in error_val.
# i.e., error_train[i] and
# error_val[i] should give you the errors
# obtained after training on i examples.
#
# Note: You should evaluate the training error on the first i training
# examples (i.e., X[:i+1, :] and y[:i+1]).
#
# For the cross-validation error, you should instead evaluate on
# the _entire_ cross validation set (Xval and yval).
#
# Note: If you are using your cost function (linearRegCostFunction)
# to compute the training and cross validation error, you should
# call the function with the lmbda argument set to 0.
# Do note that you will still need to use lmbda when running
# the training to obtain the theta parameters.
#
# Hint: You can loop over the examples with the following:
#
# for i in range(m):
# % Compute train/cross validation errors using training examples
# % X[:i+1, :] and y[:i+1], storing the result in
# % error_train[i] and error_val[i]
# ....
#
#
# ---------------------- Sample Solution ----------------------
for i in range(m):
theta = trainLinearReg(X[:i + 1, :], y[:i + 1, :], lmbda)
error_train[i] = linearRegCostFunction(theta, X[:i + 1, :], y[:i + 1, :], 0)[0]
error_val[i] = linearRegCostFunction(theta, Xval, yval, 0)[0]
# -------------------------------------------------------------
# =============================================================
return error_train, error_val
def validationCurve(X, y, Xval, yval):
""""
VALIDATIONCURVE Generate the train and validation errors needed to
plot a validation curve that we can use to select lambda
lambda_vec, error_train, error_val =
VALIDATIONCURVE(X, y, Xval, yval) returns the train
and validation errors (in error_train, error_val)
for different values of lmbda. You are given the training set (X,
y) and validation set (Xval, yval).
"""
# Selected values of lambda (you should not change this)
lmbda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
# You need to return these variables correctly.
error_train = np.zeros((len(lmbda_vec), 1))
error_val = np.zeros((len(lmbda_vec), 1))
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return training errors in
# error_train and the validation errors in error_val. The
# vector lmbda_vec contains the different lambda parameters
# to use for each calculation of the errors, i.e,
# error_train[i], and error_val[i] should give
# you the errors obtained after training with
# lmbda = lmbda_vec(i)
#
# Note: You can loop over lmbda_vec with the following:
#
# for i in range(len(lmbda_vec)):
# lmbda = lmbda_vec[i]
# % Compute train / val errors when training linear
# % regression with regularization parameter lambda
# % You should store the result in error_train[i]
# % and error_val[i]
# ....
#
#
for i in range(len(lmbda_vec)):
lmbda = lmbda_vec[i]
theta = trainLinearReg(X, y, lmbda)
error_train[i] = linearRegCostFunction(theta, X, y, 0)[0]
error_val[i] = linearRegCostFunction(theta, Xval, yval, 0)[0]
# =========================================================================
return (lmbda_vec, error_train, error_val)
def validationCurve(X, y, Xval, yval):
# Selected values of lambda (you should not change this)
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]).reshape(-1,1)
# You need to return these variables correctly.
error_train = np.zeros(lambda_vec.shape[0])
error_val = np.zeros(lambda_vec.shape[0])
for i in range(lambda_vec.shape[0]):
_lambda = lambda_vec[i]
theta = trainLinearReg(X, y, _lambda)
error_train[i], _ = linearRegCostFunction(X, y, theta, 0)
error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0)
return lambda_vec, error_train, error_val
def validationCurve(X, y, Xval, yval):
# Selected values of lambda (you should not change this)
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]).reshape(-1,1)
# You need to return these variables correctly.
error_train = np.zeros(lambda_vec.shape[0])
error_val = np.zeros(lambda_vec.shape[0])
for i in range(lambda_vec.shape[0]):
_lambda = lambda_vec[i]
theta = trainLinearReg(X, y, _lambda)
error_train[i], _ = linearRegCostFunction(X, y, theta, 0)
error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0)
return lambda_vec, error_train, error_val
def trainLinearReg(X, y, lambda_):
#TRAINLINEARREG Trains linear regression given a dataset (X, y) and a
#regularization parameter lambda
# [theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using
# the dataset (X, y) and regularization parameter lambda. Returns the
# trained parameters theta.
#
# Initialize Theta
initial_theta = zeros(size(X, 1))
# Create "short hand" for the cost function to be minimized
costFunction = lambda t: linearRegCostFunction(X, y, t, lambda_)
# Now, costFunction is a function that takes in only one argument
# setup an iteration counter
counter = count()
# Minimize using CG algorithm
res = minimize(costFunction,
initial_theta,
method='CG',
jac=True,
options={'maxiter': 200},
callback=lambda _: counter.next())
theta = res.x
print "Iteration %5d | Cost: %e" % (counter.next(), res.fun)
return theta
def trainLinearReg(X, y, lambda_value):
#TRAINLINEARREG Trains linear regression given a dataset (X, y) and a
#regularization parameter lambda
# [theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using
# the dataset (X, y) and regularization parameter lambda. Returns the
# trained parameters theta.
#
# Initialize Theta
initial_theta = np.zeros((X.shape[1], 1))
# Create "short hand" for the cost function to be minimized
costFunction = lambda t: linearRegCostFunction(X, y, t, lambda_value)
# Now, costFunction is a function that takes in only one argument
options = {'maxiter': 200}
# Minimize using fmincg
theta = optimize.minimize(costFunction,
initial_theta,
jac=True,
method='TNC',
options=options).x
return theta
def trainLinearReg(X, y, lamda):
# Initialize Theta
initial_theta = np.zeros((X.shape[1], 1))
# Create "short hand" for the cost function to be minimized
costFunc = lambda w: linearRegCostFunction(X, y, w, lamda)
# Now, costFunction is a function that takes in only one argument
minimize_method = "CG" # CG, L-BFGS-B, TNC, ...
opts = {'maxiter': 200}
if minimize_method == "TNC":
opts['maxCGit'] = 0
opts['stepmx'] = 500
elif minimize_method == 'L-BFGS-B':
opts['eps'] = 1e-8
else:
pass
# Minimize using fmincg
Result = op.minimize(fun=linearRegCostFunction,
x0=initial_theta,
method=minimize_method,
args=(X, y, lamda),
jac=True,
options=opts)
return Result.x
def learningCurve(X, y, Xval, yval, _lambda):
# Number of training examples
m = X.shape[0]
# You need to return these values correctly
error_train = np.zeros(m)
error_val = np.zeros(m)
for i in range(m):
Xtrain = X[0:i+1]
ytrain = y[0:i+1]
theta = trainLinearReg(Xtrain, ytrain, _lambda)
error_train[i], _ = linearRegCostFunction(Xtrain, ytrain, theta, 0)
error_val[i], _ = linearRegCostFunction(Xval, yval, theta, 0)
return error_train, error_val
def trainLinearReg(X, y, Lambda, method='CG', maxiter=200):
"""trains linear regression using
the dataset (X, y) and regularization parameter lambda. Returns the
trained parameters theta.
"""
# Initialize Theta
initial_theta = np.zeros(X.shape[1])
# Create "short hand" for the cost function to be minimized
costFunction = lambda t: linearRegCostFunction(X, y, t, Lambda)[0]
gradFunction = lambda t: linearRegCostFunction(X, y, t, Lambda)[1]
result = minimize(costFunction, initial_theta, method=method, jac=None, options={'disp': True, 'maxiter': maxiter})
return result.x
def validationCurve(X, y, Xval, yval):
"""
VALIDATIONCURVE Generate the train and validation errors needed to
plot a validation curve that we can use to select lambda
[lambda_vec, error_train, error_val] = ...
VALIDATIONCURVE(X, y, Xval, yval) returns the train
and validation errors (in error_train, error_val)
for different values of lambda. You are given the training set (X,
y) and validation set (Xval, yval).
Args
X: numpy array (X train)
y: numpy array (y train)
Xval: numpy array (X validation)
yval: numpy array (y validation)
Return
lambda_vec: numpy array (selected values of lambda)
error_train: numpy array (train errors)
error_val: numpy array (val errors)
"""
# Selected values of lambda (you should not change this)
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10])
# You need to return these variables correctly.
error_train = np.zeros((len(lambda_vec), 1))
error_val = np.zeros((len(lambda_vec), 1))
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return training errors in
# error_train and the validation errors in error_val. The
# vector lambda_vec contains the different lambda parameters
# to use for each calculation of the errors, i.e,
# error_train(i), and error_val(i) should give
# you the errors obtained after training with
# lambda = lambda_vec(i)
for i in range(len(lambda_vec)):
lbd = lambda_vec[i]
theta = trainLinearReg(X, y, lbd)
error_train[i] = linearRegCostFunction(X, y, theta, 0)[0] # lambda = 0
error_val[i] = linearRegCostFunction(Xval, yval, theta,
0)[0] # lambda = 0
return lambda_vec, error_train, error_val
def validationCurve(X, y, Xval, yval):
lambda_vec = np.array([0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3,
10]).reshape(10, 1)
error_train = np.zeros((lambda_vec.shape[0], 1))
error_val = np.zeros((lambda_vec.shape[0], 1))
for i in range(lambda_vec.shape[0]):
lambda_ = lambda_vec[i]
# train with lambda[i] to get theta[i]
theta = trainLinearReg(X, y, lambda_)
# Get error training and error validation
error_train[i] = linearRegCostFunction(theta, X, y, 0)[0]
error_val[i] = linearRegCostFunction(theta, Xval, yval, 0)[0]
return lambda_vec, error_train, error_val
def LearningCurve(X, y, Xval, yval, lambda_):
#In this function, you will compute the train and test errors for
#dataset sizes from 1 up to m. In practice, when working with larger
#datasets, you might want to do this in larger intervals.
[m, n] = X.shape
error_train = np.zeros((m + 1, 1))
error_cross_validation = np.zeros((m + 1, 1))
for i in range(1, m + 1):
X_train = X[0:i, :]
y_train = y[0:i]
theta = tlr.TrainLinearRegression(X_train, y_train, lambda_)
error_train[i] = lrcf.linearRegCostFunction(X_train, y_train, theta,
lambda_)[0]
error_cross_validation[i] = lrcf.linearRegCostFunction(
Xval, yval, theta, lambda_)[0]
return error_train, error_cross_validation
def gd_reglinear_reg(X, y, alpha, lamb, epochs, theta):
cost = np.zeros(epochs)
for i in range(epochs):
gradient = gradiente_linear_reg(X, y, lamb, theta)
theta = theta - (alpha * gradient)
cost[i] = linearRegCostFunction(X, y, theta, lamb)
return cost[-1], theta
def trainLinearReg(X, y, _lambda):
# Initialize Theta
initial_theta = np.zeros(X.shape[1])
# Create "short hand" for the cost function to be minimized
costFunction = lambda t: linearRegCostFunction(X, y, t, _lambda)
res = minimize(costFunction, initial_theta, method='BFGS', jac=True, tol=1e-2)
print(res)
return res.x
def output(partId):
# Random Test Cases
X = column_stack([ones(10), sin(arange(1,15,1.5)), cos(arange(1,15,1.5))])
y = sin(arange(1,30,3))
Xval = column_stack([ones(10), sin(arange(0,14,1.5)), cos(arange(0,14,1.5))])
yval = sin(arange(1,11))
if partId == '1':
J, _ = linearRegCostFunction(X, y, hstack([0.1, 0.2, 0.3]), 0.5)
return sprintf('%0.5f ', J)
elif partId == '2':
J, grad = linearRegCostFunction(X, y, hstack([0.1, 0.2, 0.3]), 0.5)
return sprintf('%0.5f ', grad)
elif partId == '3':
error_train, error_val = \
learningCurve(X, y, Xval, yval, 1);
return sprintf('%0.5f ', vstack([error_train, error_val]))
elif partId == '4':
X_poly = polyFeatures(X[1,:], 8)
return sprintf('%0.5f ', X_poly)
elif partId == '5':
lambda_vec, error_train, error_val = \
validationCurve(X, y, Xval, yval)
return sprintf('%0.5f ', \
vstack([lambda_vec, error_train, error_val]))
def trainLinearReg(X, y, lambda_):
#TRAINLINEARREG Trains linear regression given a dataset (X, y) and a
#regularization parameter lambda
# [theta] = TRAINLINEARREG (X, y, lambda) trains linear regression using
# the dataset (X, y) and regularization parameter lambda. Returns the
# trained parameters theta.
#
# Initialize Theta
initial_theta = zeros(size(X, 1))
# Create "short hand" for the cost function to be minimized
costFunction = lambda t: linearRegCostFunction(X, y, t, lambda_)
# Now, costFunction is a function that takes in only one argument
# setup an iteration counter
counter = count()
# Minimize using CG algorithm
res = minimize(costFunction, initial_theta, method='CG', jac=True,
options={'maxiter': 200}, callback=lambda _:counter.next())
theta = res.x
print "Iteration %5d | Cost: %e" % (counter.next(), res.fun)
return theta
# Plot training data
plt.figure()
plt.plot(X, y, 'rx', ms=5, linewidth=1.5)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
input('Program paused. Press enter to continue.\n')
## =========== Part 2: Regularized Linear Regression Cost =============
# You should now implement the cost function for regularized linear
# regression.
#
theta = np.ones(2)
J, _ = linearRegCostFunction(_X, y, theta, 1)
print('Cost at theta = [1 ; 1]: %f ' \
'\n(this value should be about 303.993192)\n'%J)
input('Program paused. Press enter to continue.\n')
## =========== Part 3: Regularized Linear Regression Gradient =============
# You should now implement the gradient for regularized linear
# regression.
#
theta = np.ones(2)
J, grad = linearRegCostFunction(_X, y, theta, 1)
print(grad)
# Plot training data
fig = plt.figure()
plt.plot(X, y, 'rx', linewidth=1.5, markersize=10)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
input('Program paused. Press enter to continue.')
# =========== Part 2: Regularized Linear Regression Cost =============
# You should now implement the cost function for regularized linear
# regression.
#
theta = np.ones((n + 1))
J, grad = linearRegCostFunction(
theta, np.hstack((np.ones((m, 1)), X)), y, 1)
print('Cost at theta = [1, 1]: %f' % (J))
print('(this value should be about 303.993192)')
input('Program paused. Press enter to continue.')
# =========== Part 3: Regularized Linear Regression Gradient =============
# You should now implement the gradient for regularized linear
# regression.
#
theta = np.ones((n + 1))
J, grad = linearRegCostFunction(
theta, np.hstack((np.ones((m, 1)), X)), y, 1)
def learningCurve(X, y, Xval, yval, Lambda):
"""returns the train and
cross validation set errors for a learning curve. In particular,
it returns two vectors of the same length - error_train and
error_val. Then, error_train(i) contains the training error for
i examples (and similarly for error_val(i)).
In this function, you will compute the train and test errors for
dataset sizes from 1 up to m. In practice, when working with larger
datasets, you might want to do this in larger intervals.
"""
# Number of training examples
m, _ = X.shape
# You need to return these values correctly
error_train = np.zeros(m)
error_val = np.zeros(m)
# ====================== YOUR CODE HERE ======================
# Instructions: Fill in this function to return training errors in
# error_train and the cross validation errors in error_val.
# i.e., error_train(i) and
# error_val(i) should give you the errors
# obtained after training on i examples.
#
# Note: You should evaluate the training error on the first i training
# examples (i.e., X(1:i, :) and y(1:i)).
#
# For the cross-validation error, you should instead evaluate on
# the _entire_ cross validation set (Xval and yval).
#
# Note: If you are using your cost function (linearRegCostFunction)
# to compute the training and cross validation error, you should
# call the function with the lambda argument set to 0.
# Do note that you will still need to use lambda when running
# the training to obtain the theta parameters.
#
# Hint: You can loop over the examples with the following:
#
# for i = 1:m
# # Compute train/cross validation errors using training examples
# # X(1:i, :) and y(1:i), storing the result in
# # error_train(i) and error_val(i)
# ....
#
# end
#
# ---------------------- Sample Solution ----------------------
#X = np.column_stack((np.ones(m), X))
#Xval = np.column_stack((np.ones(Xval.shape), Xval))
for i in range(m):
theta = trainLinearReg(X[:i+1],y[:i+1],Lambda)
error_train[i] = linearRegCostFunction(X[:i+1],y[:i+1], theta, 0)[0]
error_val[i] = linearRegCostFunction(Xval,yval, theta, 0)[0]
# -------------------------------------------------------------------------
# =========================================================================
return error_train, error_val
m = X.size
# Plot training data
plt.scatter(X, y, marker='x', s=60, edgecolor='r', lw=1.5)
plt.ylabel('Water flowing out of the dam (y)') # Set the y-axis label
plt.xlabel('Change in water level (x)') # Set the x-axis label
raw_input("Program paused. Press Enter to continue...")
## =========== Part 2: Regularized Linear Regression Cost =============
# You should now implement the cost function for regularized linear
# regression.
#
theta = np.array([1, 1])
J = linearRegCostFunction(np.column_stack((np.ones(m), X)), y, theta, 1)[0]
print 'Cost at theta = [1 1]: %f \n(this value should be about 303.993192)\n' % J
raw_input("Program paused. Press Enter to continue...")
## =========== Part 3: Regularized Linear Regression Gradient =============
# You should now implement the gradient for regularized linear
# regression.
#
theta = np.array([1, 1])
J, grad = linearRegCostFunction(np.column_stack((np.ones(m), X)), y, theta, 1)
print 'Gradient at theta = [1 1]: [%f %f] \n(this value should be about [-15.303016 598.250744])\n' %(grad[0], grad[1])
plt.show(block=False)
print('Program paused. Press enter to continue.\n')
pause()
""" =========== Part 2: Regularized Linear Regression Cost =============
You should now implement the cost function for regularized linear
regression.
"""
theta = np.array([[1], [1]])
new_input = np.c_[np.ones((m, 1)), X]
J, grad = linearRegCostFunction(new_input, y, theta, 1)
print('Cost at theta = [1 ; 1]: {} \n(this value should be about 303.993192)\n'.format(J))
print('Program paused. Press enter to continue.\n')
pause()
""" =========== Part 3: Regularized Linear Regression Gradient =============
You should now implement the gradient for regularized linear
regression.
"""
print('Gradient at theta = [1 ; 1]: [{}; {}] \n(this value should be about [-15.303016; 598.250744])\n'.format(grad[0], grad[1]))
# Plot training data
fig = figure()
plot(X, y, 'rx', markersize=10, linewidth=1.5)
xlabel('Change in water level (x)')
ylabel('Water flowing out of the dam (y)')
fig.show()
print 'Program paused. Press enter to continue.'
raw_input()
## =========== Part 2: Regularized Linear Regression Cost =============
# You should now implement the cost function for regularized linear
# regression.
#
theta = array([1., 1.])
J, _ = linearRegCostFunction(addOnes(X), y, theta, 1.)
print 'Cost at theta = [1, 1]: %f ' % J
print '(this value should be about 303.993192)'
print 'Program paused. Press enter to continue.'
raw_input()
## =========== Part 3: Regularized Linear Regression Gradient =============
# You should now implement the gradient for regularized linear
# regression.
#
theta = array([1., 1.])
J, grad = linearRegCostFunction(addOnes(X), y, theta, 1.)
