def validationCurve(X, y, Xval, yval):
"""
Instructions
------------
Fill in this function to return training errors in `error_train` and
the validation errors in `error_val`. The vector `lambda_vec` contains
the different lambda parameters to use for each calculation of the
errors, i.e, `error_train[i]`, and `error_val[i]` should give you the
errors obtained after training with `lambda_ = lambda_vec[i]`.
"""
# Selected values of lambda (you should not change this)
lambda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
# You need to return these variables correctly.
error_train = np.zeros(len(lambda_vec))
error_val = np.zeros(len(lambda_vec))
# ====================== YOUR CODE HERE ======================
for ii in range(0, len(lambda_vec)):
# Choosing lambda
lambda_ = lambda_vec[ii]
# Optimizing theta
theta = utils.trainLinearReg(linearRegCostFunction, X, y, lambda_)
# Calculating errors
error_train[ii], _ = linearRegCostFunction(X, y, theta)
error_val[ii], _ = linearRegCostFunction(Xval, yval, theta)
# ============================================================
return lambda_vec, error_train, error_val
def validationCurve(X, y, Xval, yval):
lambda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
# You need to return these variables correctly.
error_train = np.zeros(len(lambda_vec))
error_val = np.zeros(len(lambda_vec))
for i in range(len(lambda_vec)):
# print(i)
theta_lamda_i = utils.trainLinearReg(linearRegCostFunction, X, y,
lambda_vec[i])
J_train_err_i, _ = linearRegCostFunction(X, y, theta_lamda_i)
J_val_err_i, _ = linearRegCostFunction(
Xval, yval, theta_lamda_i) # print(J_val_err_i)
error_train[i - 1] = J_train_err_i
error_val[i - 1] = J_val_err_i
return lambda_vec, error_train, error_val
def learningCurve(X, y, Xval, yval, lambda_=0):
m = len(y)
error_train = np.zeros(m)
error_val = np.zeros(m)
Xval_aug = np.concatenate([np.ones((Xval.shape[0],1)),Xval],axis=1)
for i in range(m):
X_i = X[:i+1]
X_i_aug = np.concatenate([np.ones((X_i.shape[0],1)),X_i],axis=1)
y_i = y[:i+1]
theta_i = utils.trainLinearReg(LinearRegCostFunction,X_i_aug,y_i,lambda_=lambda_)
error_train[i] = LinearRegCostFunction(X_i_aug,y_i,theta_i,lambda_=0)[0]
if i==0:
error_val[i] = np.nan
else:
error_val[i] = LinearRegCostFunction(Xval_aug,yval,theta_i,lambda_=0)[0]
return error_train, error_val
def validationCurve(X, y, Xval, yval):
# Selected values of lambda (you should not change this)
lambda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
# You need to return these variables correctly.
error_train = np.zeros(len(lambda_vec))
error_val = np.zeros(len(lambda_vec))
# ====================== YOUR CODE HERE ======================
for i in range(len(lambda_vec)):
lambda_ = lambda_vec[i]
theta = utils.trainLinearReg(linearRegCostFunction, X, y, lambda_)
error_train[i], _ = linearRegCostFunction(X, y, theta)
error_val[i], _ = linearRegCostFunction(Xval, yval, theta)
# ============================================================
return lambda_vec, error_train, error_val
def learningCurve(X, y, Xval, yval, lambda_=0):
# Number of training examples
m = y.size
# You need to return these values correctly
error_train = np.zeros(m)
error_val = np.zeros(m)
# ====================== YOUR CODE HERE ======================
for i in range(1, m + 1):
#theta = utils.trainLinearReg(linearRegCostFunction, X[:i,:], y[:i],lambda_)
#error_train[i-1], _ = linearRegCostFunction(X, y, theta,lambda_)
#error_val[i-1], _ = linearRegCostFunction(Xval, yval, theta,lambda_)
theta = utils.trainLinearReg(linearRegCostFunction, X[:i, :], y[:i],
lambda_)
error_train[i - 1], _ = linearRegCostFunction(X[:i, :], y[:i], theta)
error_val[i - 1], _ = linearRegCostFunction(Xval, yval, theta)
# =============================================================
return error_train, error_val
def learningCurve(X, y, Xval, yval, lambda_=0):
m = y.size # Number of training examples
error_train = np.zeros(m)
error_val = np.zeros(m)
print(X.shape)
print(y.shape)
for i in range(1, m + 1):
# findTheta at first i examples in train data set
X_train_temp = X[:i] #X[:i, :]
# print(X_train_temp.shape)
y_train_temp = y[:i]
theta_i = utils.trainLinearReg(linearRegCostFunction, X_train_temp,
y_train_temp, lambda_)
# J_train_err_i, _ = linearRegCostFunction(X_train_temp, y_train_temp,
# theta_i, lambda_)
J_train_err_i, _ = linearRegCostFunction(X_train_temp, y_train_temp,
theta_i)
J_val_err_i, _ = linearRegCostFunction(Xval, yval, theta_i)
error_train[i - 1] = J_train_err_i
error_val[i - 1] = J_val_err_i
return error_train, error_val
def learningCurve(X, y, Xval, yval, lambda_=0):
"""
Instructions
------------
Fill in this function to return training errors in error_train and the
cross validation errors in error_val. i.e., error_train[i] and
error_val[i] should give you the errors obtained after training on i examples.
"""
# Number of training examples
m = y.size
# You need to return these values correctly
error_train = np.zeros(m)
error_val = np.zeros(m)
# ====================== YOUR CODE HERE ======================
for ii in range(1, m + 1):
Xtrain = X[:ii, :]
ytrain = y[:ii]
theta = utils.trainLinearReg(linearRegCostFunction, Xtrain, ytrain,
lambda_)
error_train[ii - 1], _ = linearRegCostFunction(Xtrain,
ytrain,
theta,
lambda_=0)
error_val[ii - 1], _ = linearRegCostFunction(Xval,
yval,
theta,
lambda_=0)
# =============================================================
return error_train, error_val
# lambda_vec, error_train, error_val = validationCurve(X_poly, y, X_poly_val, yval)
# print(error_train.shape)
# pyplot.plot(lambda_vec, error_train, '-o', lambda_vec, error_val, '-o', lw=2)
# pyplot.legend(['Train', 'Cross Validation'])
# pyplot.xlabel('lambda')
# pyplot.ylabel('Error')
# pyplot.show()
# print('lambda\t\tTrain Error\tValidation Error')
# for i in range(len(lambda_vec)):
# print(' %f\t%f\t%f' % (lambda_vec[i], error_train[i], error_val[i]))
# Optional 1
theta_with_lamda_3 = utils.trainLinearReg(linearRegCostFunction, X_poly, y, 3)
J_test_err, _ = linearRegCostFunction(X_poly_test, ytest, theta_with_lamda_3)
print(J_test_err)
# Optional 2
# the training error and cross validation error for i examples
# lambda_val value for this step
lambda_val = 0.01
# number of iterations
times = 50
# initialize error matrices
error_train_rand = np.zeros((m, times))
error_val_rand = np.zeros((m, times))
def validationCurve(X, y, Xval, yval):
"""
Generate the train and validation errors needed to plot a validation
curve that we can use to select lambda_.
Parameters
----------
X : array_like
The training dataset. Matrix with shape (m x n) where m is the
total number of training examples, and n is the number of features
including any polynomial features.
y : array_like
The functions values at each training datapoint. A vector of
shape (m, ).
Xval : array_like
The validation dataset. Matrix with shape (m_val x n) where m is the
total number of validation examples, and n is the number of features
including any polynomial features.
yval : array_like
The functions values at each validation datapoint. A vector of
shape (m_val, ).
Returns
-------
lambda_vec : list
The values of the regularization parameters which were used in
cross validation.
error_train : list
The training error computed at each value for the regularization
parameter.
error_val : list
The validation error computed at each value for the regularization
parameter.
Instructions
------------
Fill in this function to return training errors in `error_train` and
the validation errors in `error_val`. The vector `lambda_vec` contains
the different lambda parameters to use for each calculation of the
errors, i.e, `error_train[i]`, and `error_val[i]` should give you the
errors obtained after training with `lambda_ = lambda_vec[i]`.
Note
----
You can loop over lambda_vec with the following:
for i in range(len(lambda_vec))
lambda = lambda_vec[i]
# Compute train / val errors when training linear
# regression with regularization parameter lambda_
# You should store the result in error_train[i]
# and error_val[i]
....
"""
# Selected values of lambda (you should not change this)
lambda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
# You need to return these variables correctly.
error_train = np.zeros(len(lambda_vec))
error_val = np.zeros(len(lambda_vec))
# ====================== YOUR CODE HERE ======================
for i in range(len(lambda_vec)):
lambda_ = lambda_vec[i]
theta = utils.trainLinearReg(linearRegCostFunction, X, y, lambda_)
error_train[i] = linearRegCostFunction(X, y, theta, 0)[0]
error_val[i] = linearRegCostFunction(Xval, yval, theta, 0)[0]
# ============================================================
return lambda_vec, error_train, error_val
def learningCurve(X, y, Xval, yval, lambda_=0):
"""
Generates the train and cross validation set errors needed to plot a learning curve
returns the train and cross validation set errors for a learning curve.
In this function, you will compute the train and test errors for
dataset sizes from 1 up to m. In practice, when working with larger
datasets, you might want to do this in larger intervals.
Parameters
----------
X : array_like
The training dataset. Matrix with shape (m x n + 1) where m is the
total number of examples, and n is the number of features
before adding the bias term.
y : array_like
The functions values at each training datapoint. A vector of
shape (m, ).
Xval : array_like
The validation dataset. Matrix with shape (m_val x n + 1) where m is the
total number of examples, and n is the number of features
before adding the bias term.
yval : array_like
The functions values at each validation datapoint. A vector of
shape (m_val, ).
lambda_ : float, optional
The regularization parameter.
Returns
-------
error_train : array_like
A vector of shape m. error_train[i] contains the training error for
i examples.
error_val : array_like
A vecotr of shape m. error_val[i] contains the validation error for
i training examples.
Instructions
------------
Fill in this function to return training errors in error_train and the
cross validation errors in error_val. i.e., error_train[i] and
error_val[i] should give you the errors obtained after training on i examples.
Notes
-----
- You should evaluate the training error on the first i training
examples (i.e., X[:i, :] and y[:i]).
For the cross-validation error, you should instead evaluate on
the _entire_ cross validation set (Xval and yval).
- If you are using your cost function (linearRegCostFunction) to compute
the training and cross validation error, you should call the function with
the lambda argument set to 0. Do note that you will still need to use
lambda when running the training to obtain the theta parameters.
Hint
----
You can loop over the examples with the following:
for i in range(1, m+1):
# Compute train/cross validation errors using training examples
# X[:i, :] and y[:i], storing the result in
# error_train[i-1] and error_val[i-1]
....
"""
# Number of training examples
m = y.size
# You need to return these values correctly
error_train = np.zeros(m)
error_val = np.zeros(m)
# ====================== YOUR CODE HERE ======================
for i in range(1, m + 1):
theta = utils.trainLinearReg(linearRegCostFunction, X[:i, :], y[:i],
lambda_)
error_train[i - 1] = linearRegCostFunction(X[:i, :], y[:i], theta,
0.0)[0]
error_val[i - 1] = linearRegCostFunction(Xval, yval, theta, 0.0)[0]
# =============================================================
return error_train, error_val
# Once your cost function and gradient are working correctly, the next cell will run the code in `trainLinearReg` (found in the module `utils.py`) to compute the optimal values of $\theta$. This training function uses `scipy`'s optimization module to minimize the cost function.
#
# In this part, we set regularization parameter $\lambda$ to zero. Because our current implementation of linear regression is trying to fit a 2-dimensional $\theta$, regularization will not be incredibly helpful for a $\theta$ of such low dimension. In the later parts of the exercise, you will be using polynomial regression with regularization.
#
# Finally, the code in the next cell should also plot the best fit line, which should look like the figure below.
#
# 
#
# The best fit line tells us that the model is not a good fit to the data because the data has a non-linear pattern. While visualizing the best fit as shown is one possible way to debug your learning algorithm, it is not always easy to visualize the data and model. In the next section, you will implement a function to generate learning curves that can help you debug your learning algorithm even if it is not easy to visualize the
# data.
# In[21]:
# add a columns of ones for the y-intercept
X_aug = np.concatenate([np.ones((m, 1)), X], axis=1)
theta = utils.trainLinearReg(linearRegCostFunction, X_aug, y, lambda_=0)
# Plot fit over the data
pyplot.plot(X, y, 'ro', ms=10, mec='k', mew=1.5)
pyplot.xlabel('Change in water level (x)')
pyplot.ylabel('Water flowing out of the dam (y)')
pyplot.plot(X, np.dot(X_aug, theta), '--', lw=2)
# <a id="section3"></a>
# ## 2 Bias-variance
#
# An important concept in machine learning is the bias-variance tradeoff. Models with high bias are not complex enough for the data and tend to underfit, while models with high variance overfit to the training data.
#
# In this part of the exercise, you will plot training and test errors on a learning curve to diagnose bias-variance problems.
#
# ### 2.1 Learning Curves
theta_ = theta.copy()
theta_[0] = 0
J = (0.5/m) * ((h - y)**2).sum() + lambda_ * (0.5/m)*(theta_**2).sum()
grad = (1./m)*np.dot(h-y,X) + lambda_ * (1/m) * theta_
return J, grad
initial_theta = np.array([1,1])
X_ = np.concatenate([np.ones((m, 1)), X], axis=1)
J ,grad = LinearRegCostFunction(X_, y, initial_theta, lambda_=1.0)
print("Cost is: {:.6f}".format(J))
print("Gradient is: [{:.6f},{:.6f}]".format(*grad)) ### passed!
###===================Part 1.4 Fitting linear regression ===========================================
X_ = np.concatenate([np.ones((m, 1)), X], axis=1)
optimized_theta = utils.trainLinearReg(LinearRegCostFunction,X_,y,lambda_=0.)
# plt.plot(X, y, "ro", ms=10, mec="k", mew=1)
# plt.plot(X,np.dot(X_,optimized_theta),"--",lw=2) ### passed!
# plt.show()
###=================== Part 2.1 Learning Curves ===================================================
def learningCurve(X, y, Xval, yval, lambda_=0):
m = len(y)
error_train = np.zeros(m)
error_val = np.zeros(m)
Xval_aug = np.concatenate([np.ones((Xval.shape[0],1)),Xval],axis=1)
for i in range(m):
X_i = X[:i+1]
X_i_aug = np.concatenate([np.ones((X_i.shape[0],1)),X_i],axis=1)
def learningCurveAverage(X, y, Xval, yval):
X_limit = X.shape[0]
Xval_limit = Xval.shape[0]
training_costs_for_each_i = []
val_costs_for_each_i = []
for i in range(1, min(X_limit, Xval_limit) + 1):
training_costs_for_i = []
val_costs_for_i = []
for iter in range(50):
X_random_indices = np.random.choice(X_limit, size=i, replace=False)
X_temp = []
y_temp = []
for idx in X_random_indices:
X_temp.append(X[idx])
y_temp.append(y[idx])
X_temp = np.asarray(X_temp)
y_temp = np.asarray(y_temp)
theta = utils.trainLinearReg(linearRegCostFunction,
np.asarray(X_temp),
np.asarray(y_temp),
lambda_=0.01,
maxiter=55)
Xval_random_indices = np.random.choice(Xval_limit,
size=i,
replace=False)
Xval_temp = []
yval_temp = []
for index in Xval_random_indices:
Xval_temp.append(Xval[index])
yval_temp.append(yval[index])
Xval_temp = np.asarray(Xval_temp)
yval_temp = np.asarray(yval_temp)
train_cost, _ = linearRegCostFunction(X_temp,
y_temp,
theta,
lambda_=0)
val_cost, _ = linearRegCostFunction(Xval_temp,
yval_temp,
theta,
lambda_=0)
training_costs_for_i.append(train_cost)
val_costs_for_i.append(val_cost)
training_costs_for_each_i.append(
np.mean(np.asarray(training_costs_for_i)))
val_costs_for_each_i.append(np.mean(np.asarray(val_costs_for_i)))
pyplot.clf()
training_costs_for_each_i = np.asarray(training_costs_for_each_i)
val_costs_for_each_i = np.asarray(val_costs_for_each_i)
pyplot.plot(range(1, 13),
training_costs_for_each_i,
'-o',
range(1, 13),
val_costs_for_each_i,
'-o',
lw=2)
pyplot.legend(['Train', 'Cross Validation'])
pyplot.xlabel('Number of Training Examples')
pyplot.ylabel('Error')
pyplot.axis([0, 13, 0, 150])
pyplot.show()
# Once your cost function and gradient are working correctly, the next cell will run the code in `trainLinearReg` (found in the module `utils.py`) to compute the optimal values of $\theta$. This training function uses `scipy`'s optimization module to minimize the cost function.
#
# In this part, we set regularization parameter $\lambda$ to zero. Because our current implementation of linear regression is trying to fit a 2-dimensional $\theta$, regularization will not be incredibly helpful for a $\theta$ of such low dimension. In the later parts of the exercise, you will be using polynomial regression with regularization.
#
# Finally, the code in the next cell should also plot the best fit line, which should look like the figure below.
#
# 
#
# The best fit line tells us that the model is not a good fit to the data because the data has a non-linear pattern. While visualizing the best fit as shown is one possible way to debug your learning algorithm, it is not always easy to visualize the data and model. In the next section, you will implement a function to generate learning curves that can help you debug your learning algorithm even if it is not easy to visualize the
# data.
# In[ ]:
# add a columns of ones for the y-intercept
X_aug = np.concatenate([np.ones((m, 1)), X], axis=1)
theta = utils.trainLinearReg(linearRegCostFunction, X_aug, y, lambda_=0)
# Plot fit over the data
# pyplot.plot(X, y, 'ro', ms=10, mec='k', mew=1.5)
# pyplot.xlabel('Change in water level (x)')
# pyplot.ylabel('Water flowing out of the dam (y)')
# pyplot.plot(X, np.dot(X_aug, theta), '--', lw=2);
# <a id="section3"></a>
# ## 2 Bias-variance
#
# An important concept in machine learning is the bias-variance tradeoff. Models with high bias are not complex enough for the data and tend to underfit, while models with high variance overfit to the training data.
#
# In this part of the exercise, you will plot training and test errors on a learning curve to diagnose bias-variance problems.
#
# ### 2.1 Learning Curves
