def cross_validation(y, x, k_fold, lambda_, degree):
"""
Return the loss for ridge regression for this given lambda_ and given degree
Arguments:
- y: the column of ground truth results
- x: the features matrix
- k_fold: the number of fold to do cross validation
- lambda_: penalizing parameter for ridge regression
- degree: the degree of the polynomial data augmentation
"""
k_indices = build_k_indices(y, k_fold, 1)
x_k, y_k = x[k_indices], y[k_indices]
Loss_tr = []
Loss_te = []
for k in range(k_fold):
x_train, y_train, x_test, y_test = [], [], [], []
x_test = x_k[k]
y_test = y_k[k]
x_train = np.delete(x_k, k, axis=0)
y_train = np.delete(y_k, k, axis=0)
phi_x_train = build_poly(x_train, degree)
phi_x_test = build_poly(x_test, degree)
loss_tr, weights = implementations.ridge_regression(
y_train, phi_x_train, lambda_)
loss_te = implementations.compute_mse(y_test, phi_x_test, weights)
Loss_tr.append(loss_tr)
Loss_te.append(loss_te)
Loss_tr = np.array(Loss_tr)
Loss_te = np.array(Loss_te)
return Loss_tr.mean(), Loss_te.mean()
def lasso_reg(y, tx, initial_w, max_iters, gamma, LAMBDA):
"""
L1 regularized linear regression = Lasso Regression
:param y: (n,) array
:param tx: (n,d) matrix
:param intial_w: (d,) array; initial weights
:param max_iters: int; number of iterations
:param gamma: float; learning rate
:return: final weights vector and loss
"""
w = initial_w
for n_iter in range(max_iters):
# retrieve gradient and cost
grd = compute_gradient_mse(y, tx, w)
# prepare the regularization factor
reg = np.sign(w) * (-1)
# update the weights
w = w - (grd + reg * LAMBDA) * gamma
# set small weights to 0
w[w < 0.005] = 0
# print(f"Step loss: {compute_mse(e)}")
# calculate the final loss
loss = compute_mse(y, tx, w) + np.sum(np.abs(w) * LAMBDA)
return w, loss
def cross_validation_ls_GD(y, x, k_indices, k, gamma, max_iters, w_initial):
"""train and test least square GD model using cross validation"""
x_test = x[k_indices[k]]
x_train = np.delete(x, [k_indices[k]], axis=0)
y_test = y[k_indices[k]]
y_train = np.delete(y, [k_indices[k]], axis=0)
opt_w, mse_tr = imp.least_squares_GD(y_train, x_train, w_initial, max_iters,gamma)
mse_te = imp.compute_mse(y_test, x_test,opt_w)
return mse_te, opt_w
def cross_validation_ls(y, x, k_indices, k):
"""train and test least square model using cross validation"""
x_test = x[k_indices[k]]
x_train = np.delete(x, [k_indices[k]], axis=0)
y_test = y[k_indices[k]]
y_train = np.delete(y, [k_indices[k]], axis=0)
opt_w, mse_tr = imp.least_squares(y_train,x_train)
mse_te = imp.compute_mse(y_test, x_test, opt_w)
return mse_te, opt_w
def cross_validation_rr(y, x, k_indices, k, lambda_, degree):
"""train and test ridge regression model using cross validation"""
x_test = x[k_indices[k]]
x_train = np.delete(x, [k_indices[k]], axis=0)
y_test = y[k_indices[k]]
y_train = np.delete(y, [k_indices[k]], axis=0)
x_tr_poly = helpers.build_poly(x_train, degree)
x_te_poly = helpers.build_poly(x_test, degree)
w, loss_tr = imp.ridge_regression(y_train, x_tr_poly, lambda_)
loss_te = imp.compute_mse(y_test, x_te_poly, w)
return loss_tr, loss_te