def test_group_lasso_lasso(sparse_X, fit_intercept, normalize):
# check that group Lasso with groups of size 1 gives Lasso
n_features = 1000
X, y = build_dataset(n_samples=100,
n_features=n_features,
sparse_X=sparse_X)[:2]
alpha_max = norm(X.T @ y, ord=np.inf) / len(y)
alpha = alpha_max / 10
clf = Lasso(alpha,
tol=1e-12,
fit_intercept=fit_intercept,
normalize=normalize,
verbose=0)
clf.fit(X, y)
# take groups of size 1:
clf1 = GroupLasso(alpha=alpha,
groups=1,
tol=1e-12,
fit_intercept=fit_intercept,
normalize=normalize,
verbose=0)
clf1.fit(X, y)
np.testing.assert_allclose(clf1.coef_, clf.coef_, atol=1e-6)
np.testing.assert_allclose(clf1.intercept_, clf.intercept_, rtol=1e-4)
def test_rw_cvg():
X, y, _ = make_correlated_data(20, 40, random_state=0)
alpha = np.max(np.abs(X.T @ y)) / 5
w, E = reweighted(X, y, alpha, max_iter=1000, n_adapt=5)
clf = Lasso(fit_intercept=False, alpha=alpha / len(y)).fit(X, y)
np.testing.assert_allclose(w, clf.coef_, atol=5e-4)
np.testing.assert_allclose(E[-1] / E[0], E[-2] / E[0], atol=5e-4)
w, E = reweighted(X, y, alpha, deriv_MCP)
np.testing.assert_allclose(E[-1] / E[0], E[-2] / E[0], atol=5e-4)
def test_cd_ista_fista():
np.random.seed(0)
X, y, _ = make_correlated_data(20, 40, random_state=0)
alpha = np.max(np.abs(X.T @ y)) / 5
w, _, _ = cd(X, y, alpha, max_iter=100)
clf = Lasso(fit_intercept=False, alpha=alpha / len(y)).fit(X, y)
np.testing.assert_allclose(w, clf.coef_, atol=5e-4)
w, _, _ = ista(X, y, alpha, max_iter=1_000)
np.testing.assert_allclose(w, clf.coef_, atol=5e-4)
w, _, _ = fista(X, y, alpha, max_iter=1_000)
np.testing.assert_allclose(w, clf.coef_, atol=5e-4)
def fit_weights(self, y, phi, reg_param, tol_factor=1e-4):
obs = self.compute_obs(phi, version=1)
mat = np.array([np.sum(obs[i], axis=0) for i in range(self.num_atoms)])
mat = mat.reshape((self.num_atoms, -1)).T
tol = tol_factor * np.linalg.norm(y)**2 / y.size
perimeters = np.array([self.atoms[i].support.compute_perimeter() for i in range(self.num_atoms)])
lasso = Lasso(alpha=reg_param/y.size, fit_intercept=False, tol=tol, weights=perimeters)
lasso.fit(mat, y.reshape(-1))
new_weights = lasso.coef_
self.atoms = [WeightedIndicatorFunction(new_weights[i], self.atoms[i].support)
for i in range(self.num_atoms) if np.abs(new_weights[i]) > 1e-2]
def set_objective(self, X, y, lmbd):
self.X, self.y, self.lmbd = X, y, lmbd
warnings.filterwarnings('ignore', category=ConvergenceWarning)
n_samples = self.X.shape[0]
self.lasso = Lasso(
alpha=self.lmbd / n_samples,
max_iter=1,
max_epochs=100000,
tol=1e-12,
prune=True,
fit_intercept=False,
normalize=False,
warm_start=False,
positive=False,
verbose=False,
)
tol=1e-7,
max_iter=100,
cv=2,
n_jobs=2).fit(X_train_val, y_train_val)
# Measure mse on test
mse_cv = mean_squared_error(y_test, model_cv.predict(X_test))
print("Vanilla LassoCV: Mean-squared error on test data %f" % mse_cv)
##############################################################################
##############################################################################
# Weighted Lasso with sparse-ho.
# We use the vanilla lassoCV coefficients as a starting point
alpha0 = np.log(model_cv.alpha_) * np.ones(X_train.shape[1])
# Weighted Lasso: Sparse-ho: 1 param per feature
estimator = Lasso(fit_intercept=False, max_iter=10, warm_start=True)
model = WeightedLasso(X_train, y_train, estimator=estimator)
criterion = HeldOutMSE(X_val, y_val, model, X_test=X_test, y_test=y_test)
algo = ImplicitForward()
monitor = Monitor()
grad_search(algo, criterion, alpha0, monitor, n_outer=20, tol=1e-6)
##############################################################################
##############################################################################
# MSE on validation set
mse_sho_val = mean_squared_error(y_val, estimator.predict(X_val))
# MSE on test set, ie unseen data
mse_sho_test = mean_squared_error(y_test, estimator.predict(X_test))
print("Sparse-ho: Mean-squared error on validation data %f" % mse_sho_val)