alpha_max = (X_train.T @ y_train).max() / n_samples
p_alpha = 0.9
alpha = p_alpha * alpha_max
log_alpha = np.log(alpha)
log_alphas = np.log(alpha_max * np.geomspace(1, 0.1))
tol = 1e-16
dict_log_alpha = {}
dict_log_alpha["lasso"] = log_alpha
tab = np.linspace(1, 1000, n_features)
dict_log_alpha["wlasso"] = log_alpha + np.log(tab / tab.max())
models = {}
models["lasso"] = Lasso(X_train, y_train, estimator=None)
models["wlasso"] = WeightedLasso(X_train, y_train, estimator=None)
def get_v(mask, dense):
return 2 * (
X_val[:, mask].T @ (X_val[:, mask] @ dense - y_val)) / X_val.shape[0]
def test_beta_jac():
#########################################################################
# check that the methods computing the full Jacobian compute the same sol
# maybe we could add a test comparing with sklearn
for key in models.keys():
supp1, dense1, jac1 = get_beta_jac_iterdiff(X_train,
y_train,
dict_log_alpha[key],
max_iter=100,
cv=2,
n_jobs=2).fit(X_train_val, y_train_val)
# Measure mse on test
mse_cv = mean_squared_error(y_test, model_cv.predict(X_test))
print("Vanilla LassoCV: Mean-squared error on test data %f" % mse_cv)
##############################################################################
##############################################################################
# Weighted Lasso with sparse-ho.
# We use the vanilla lassoCV coefficients as a starting point
alpha0 = np.log(model_cv.alpha_) * np.ones(X_train.shape[1])
# Weighted Lasso: Sparse-ho: 1 param per feature
estimator = Lasso(fit_intercept=False, max_iter=10, warm_start=True)
model = WeightedLasso(X_train, y_train, estimator=estimator)
criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = ImplicitForward()
monitor = Monitor()
grad_search(algo, criterion, alpha0, monitor, n_outer=20, tol=1e-6)
##############################################################################
##############################################################################
# MSE on validation set
mse_sho_val = mean_squared_error(y_val, estimator.predict(X_val))
# MSE on test set, ie unseen data
mse_sho_test = mean_squared_error(y_test, estimator.predict(X_test))
print("Sparse-ho: Mean-squared error on validation data %f" % mse_sho_val)
print("Sparse-ho: Mean-squared error on test (unseen) data %f" % mse_sho_test)
alpha_max = (np.abs(X[idx_train, :].T @ y[idx_train])).max() / n_samples
p_alpha = 0.9
alpha = p_alpha * alpha_max
log_alpha = np.log(alpha)
log_alphas = np.log(alpha_max * np.geomspace(1, 0.1))
tol = 1e-16
dict_log_alpha = {}
dict_log_alpha["lasso"] = log_alpha
tab = np.linspace(1, 1000, n_features)
dict_log_alpha["wlasso"] = log_alpha + np.log(tab / tab.max())
models = {}
models["lasso"] = Lasso(estimator=None)
models["wlasso"] = WeightedLasso(estimator=None)
def get_v(mask, dense):
return 2 * (X[np.ix_(idx_val, mask)].T @ (
X[np.ix_(idx_val, mask)] @ dense - y[idx_val])) / len(idx_val)
def test_beta_jac():
#########################################################################
# check that the methods computing the full Jacobian compute the same sol
# maybe we could add a test comparing with sklearn
for key in models.keys():
supp1, dense1, jac1 = get_beta_jac_iterdiff(X[idx_train, :],
y[idx_train],
dict_log_alpha[key],
max_iter=100,
cv=2,
n_jobs=2).fit(X, y)
# Measure mse on test
mse_cv = mean_squared_error(y_test, model_cv.predict(X_test))
print("Vanilla LassoCV: Mean-squared error on test data %f" % mse_cv)
##############################################################################
##############################################################################
# Weighted Lasso with sparse-ho.
# We use the vanilla lassoCV coefficients as a starting point
log_alpha0 = np.log(model_cv.alpha_) * np.ones(n_features)
# Weighted Lasso: Sparse-ho: 1 param per feature
estimator = Lasso(fit_intercept=False, max_iter=10, warm_start=True)
model = WeightedLasso(estimator=estimator)
criterion = HeldOutMSE(idx_train, idx_val)
algo = ImplicitForward()
monitor = Monitor()
grad_search(algo,
criterion,
model,
X,
y,
log_alpha0,
monitor,
n_outer=20,
tol=1e-6)
##############################################################################
##############################################################################
log_alpha1 = np.log(alpha_1)
log_alpha2 = np.log(alpha_2)
dict_log_alpha = {}
dict_log_alpha["lasso"] = log_alpha
dict_log_alpha["enet"] = np.array([log_alpha1, log_alpha2])
tab = np.linspace(1, 1000, n_features)
dict_log_alpha["wLasso"] = log_alpha + np.log(tab / tab.max())
dict_log_alpha["logreg"] = (log_alpha - np.log(2))
dict_log_alpha["svm"] = 1e-4
dict_log_alpha["svr"] = np.array([1e-2, 1e-2])
# Set models to be tested
models = {}
models["lasso"] = Lasso(estimator=None)
models["enet"] = ElasticNet(estimator=None)
models["wLasso"] = WeightedLasso(estimator=None)
models["logreg"] = SparseLogreg(estimator=None)
models["svm"] = SVM(estimator=None)
models["svr"] = SVR(estimator=None)
custom_models = {}
custom_models["lasso"] = Lasso(estimator=celer.Lasso(
warm_start=True, fit_intercept=False))
custom_models["enet"] = ElasticNet(
estimator=linear_model.ElasticNet(warm_start=True, fit_intercept=False))
custom_models["logreg"] = SparseLogreg(
estimator=celer.LogisticRegression(warm_start=True, fit_intercept=False))
# Compute "ground truth" with cvxpylayer
dict_cvxpy_func = {
'lasso': lasso_cvxpy,