def cross_validation(y, x, k_fold, lambda_, degree): """ Return the loss for ridge regression for this given lambda_ and given degree Arguments: - y: the column of ground truth results - x: the features matrix - k_fold: the number of fold to do cross validation - lambda_: penalizing parameter for ridge regression - degree: the degree of the polynomial data augmentation """ k_indices = build_k_indices(y, k_fold, 1) x_k, y_k = x[k_indices], y[k_indices] Loss_tr = [] Loss_te = [] for k in range(k_fold): x_train, y_train, x_test, y_test = [], [], [], [] x_test = x_k[k] y_test = y_k[k] x_train = np.delete(x_k, k, axis=0) y_train = np.delete(y_k, k, axis=0) phi_x_train = build_poly(x_train, degree) phi_x_test = build_poly(x_test, degree) loss_tr, weights = implementations.ridge_regression( y_train, phi_x_train, lambda_) loss_te = implementations.compute_mse(y_test, phi_x_test, weights) Loss_tr.append(loss_tr) Loss_te.append(loss_te) Loss_tr = np.array(Loss_tr) Loss_te = np.array(Loss_te) return Loss_tr.mean(), Loss_te.mean()
def lasso_reg(y, tx, initial_w, max_iters, gamma, LAMBDA): """ L1 regularized linear regression = Lasso Regression :param y: (n,) array :param tx: (n,d) matrix :param intial_w: (d,) array; initial weights :param max_iters: int; number of iterations :param gamma: float; learning rate :return: final weights vector and loss """ w = initial_w for n_iter in range(max_iters): # retrieve gradient and cost grd = compute_gradient_mse(y, tx, w) # prepare the regularization factor reg = np.sign(w) * (-1) # update the weights w = w - (grd + reg * LAMBDA) * gamma # set small weights to 0 w[w < 0.005] = 0 # print(f"Step loss: {compute_mse(e)}") # calculate the final loss loss = compute_mse(y, tx, w) + np.sum(np.abs(w) * LAMBDA) return w, loss
def cross_validation_ls_GD(y, x, k_indices, k, gamma, max_iters, w_initial): """train and test least square GD model using cross validation""" x_test = x[k_indices[k]] x_train = np.delete(x, [k_indices[k]], axis=0) y_test = y[k_indices[k]] y_train = np.delete(y, [k_indices[k]], axis=0) opt_w, mse_tr = imp.least_squares_GD(y_train, x_train, w_initial, max_iters,gamma) mse_te = imp.compute_mse(y_test, x_test,opt_w) return mse_te, opt_w
def cross_validation_ls(y, x, k_indices, k): """train and test least square model using cross validation""" x_test = x[k_indices[k]] x_train = np.delete(x, [k_indices[k]], axis=0) y_test = y[k_indices[k]] y_train = np.delete(y, [k_indices[k]], axis=0) opt_w, mse_tr = imp.least_squares(y_train,x_train) mse_te = imp.compute_mse(y_test, x_test, opt_w) return mse_te, opt_w
def cross_validation_rr(y, x, k_indices, k, lambda_, degree): """train and test ridge regression model using cross validation""" x_test = x[k_indices[k]] x_train = np.delete(x, [k_indices[k]], axis=0) y_test = y[k_indices[k]] y_train = np.delete(y, [k_indices[k]], axis=0) x_tr_poly = helpers.build_poly(x_train, degree) x_te_poly = helpers.build_poly(x_test, degree) w, loss_tr = imp.ridge_regression(y_train, x_tr_poly, lambda_) loss_te = imp.compute_mse(y_test, x_te_poly, w) return loss_tr, loss_te