# map form full to filtered, -1 means not selected map_full_to_filtered = -np.ones(Xtrain.shape[1], dtype=int) map_full_to_filtered[filter_] = np.arange((K + 1)) groups_filtered = [map_full_to_filtered[g] for g in groups] groups_filtered = [g[g != -1] for g in groups_filtered] groups_filtered = [g for g in groups_filtered if len(g) >= 1] weights_filtered = [len(g) for g in groups_filtered] weights_filtered = np.sqrt(np.asarray(weights_filtered)) Agl = gl.linear_operator_from_groups(Xval_filtered.shape[1], groups=groups_filtered, weights=weights_filtered, penalty_start=1) enet_gl = estimators.LinearRegressionL1L2GL( l1=l1, l2=l2, gl=lgl, A=Agl, algorithm=algorithm, penalty_start=1) # enet_gl2=ElasticNet(alpha=l2, l1_ratio=l1,) enet_gl.fit(Xtr_filtered, ytr) y_pred_test = enet_gl.predict(Xval_filtered) test_acc = r2_score(yval, y_pred_test) print test_acc inner_param[(l1, l2, lgl)].append(test_acc) inner_param_mean = { k: np.mean(inner_param[k]) for k in inner_param.keys() } print inner_param_mean l1, l2, lgl = max(inner_param_mean.iteritems(),
mean=True) enet_PP.fit(X_res, z) print "Compute beta values" beta = enet_PP.beta beta = beta[11:] #do not consider covariates print "Compute the weights using Parsimony's ElasticNet algorithm." weights = [ math.pow(abs(beta[j[0]]) + 1 / float(n), -gamma) for j in groups ] # Adaptive Elasticnet algorithm adaptive_enet = estimators.LinearRegressionL1L2GL( l1=0, l2=0.8, gl=0.006, A=gl.A_from_groups(p, groups, weights=weights, penalty_start=11), algorithm=proximal.FISTA(), algorithm_params=dict(max_iter=10000), penalty_start=11, mean=True) stime = time.time() print "=================================================================" print "Now fitting the model" adaptive_enet.fit(X_res, z) print "Fit duration : ", time.time() - stime print "=================================================================" # Interpretation beta_w = adaptive_enet.beta plt.plot(beta_w[11:])
l1_max =0.1* np.max(s)/Xtr.shape[0] print "l1 max is", l1_max ################################# l1, l2, lgl =l1_max * np.array((0.1, 0.1, 0.01)) weights = [np.sqrt(len(group)) for group in groups] weights = 1./np.sqrt(np.asarray(weights)) Agl = gl.linear_operator_from_groups(p, groups=groups, weights=weights) algorithm = algorithms.proximal.CONESTA(eps=consts.TOLERANCE, max_iter=15000) enet_gl = estimators.LinearRegressionL1L2GL(l1, l2, lgl , Agl, algorithm=algorithm) yte_pred_enetgl = enet_gl.fit(Xtr, ytr).predict(Xte) print " r carré vaut", r2_score(yte, yte_pred_enetgl) Xnon_res = sklearn.preprocessing.scale(Xnon_res, axis=0, with_mean=True, with_std=False) Ynon_res = Ynon_res-Ynon_res.mean() n_train = int(X.shape[0]/1.75) Xtr_res = Xnon_res[:n_train, :] ytr_res = Ynon_res[:n_train] Xte_res = Xnon_res[n_train:, :]
def test_group_lasso(self): random_state = np.random.RandomState(42) state = random_state.get_state() rng01 = simulate.utils.RandomUniform(0, 1, random_state=random_state) rng11 = simulate.utils.RandomUniform(-1, 1, random_state=random_state) # Generate start values. n, p = 48, 64 + 1 # Define the groups. groups = [range(1, 2 * p / 3), range(p / 3, p)] # Generate candidate data. beta = simulate.beta.random((p - 1, 1), density=0.5, sort=True, rng=rng01) # Add the intercept. beta = np.vstack((random_state.rand(1, 1), beta)) Sigma = simulate.correlation_matrices.constant_correlation( p=p - 1, rho=0.01, eps=0.001, random_state=random_state) X0 = random_state.multivariate_normal(np.zeros(p - 1), Sigma, n) # Add the intercept. X0 = np.hstack((np.ones((n, 1)), X0)) e = random_state.randn(n, 1) # Create linear operator. A = simulate.functions.SmoothedGroupLasso.A_from_groups( p, groups, weights=None, penalty_start=1) # Define regularisation parameters. l = 0.618 # L1 coefficient. k = 1.0 - l # Ridge (L2) coefficient. g = 1.618 # TV coefficient. mu = 5e-4 # Create optimisation problem. l1 = simulate.functions.L1(l, rng=rng11) l2 = simulate.functions.L2Squared(k) gl = simulate.functions.SmoothedGroupLasso(g, A, mu=simulate.utils.TOLERANCE) lr = simulate.LinearRegressionData([l1, l2, gl], X0, e, snr=2.0, intercept=True) # Generate simulated data. random_state.set_state(state) X, y, beta_star, e = lr.load(beta) try: import parsimony.estimators as estimators import parsimony.algorithms.proximal as proximal except ImportError: print "pylearn-parsimony is not properly installed. Will not be " \ "able to run this test." return e = estimators.LinearRegressionL1L2GL( l, k, g, A, mu, algorithm=proximal.FISTA(max_iter=10000), penalty_start=1, mean=False) beta_sim = e.fit(X, y, beta).parameters()["beta"] assert (np.linalg.norm(beta_sim - beta_star) < 0.0001)
mean=True) enet_PP.fit(X_res, z) print "Compute beta values" beta = enet_PP.beta beta = beta[11:] #do not consider covariates print "Compute the weights using Parsimony's ElasticNet algorithm." weights = [ math.pow(abs(beta[j[0]]) + 1 / float(n), -gamma) for j in groups ] # Adaptive Elasticnet algorithm A = gl.A_from_groups(p, groups, weights=weights) adaptive_enet = estimators.LinearRegressionL1L2GL( l1=0, l2=0.8, gl=0.006, A=A, algorithm=proximal.FISTA(), algorithm_params=dict(max_iter=10000), penalty_start=11, mean=True) stime = time.time() print "=================================================================" print "Now fitting the model" adaptive_enet.fit(X_res, z) print "Fit duration : ", time.time() - stime print "=================================================================" # Interpretation beta_w = adaptive_enet.beta[11:] plt.plot(beta_w)
# Create the loss function. function = LinearRegressionL1L2GL(X, y, l, k, g, A=A, penalty_start=1, mean=False) # Create the estimator. lr = estimators.LinearRegressionL1L2GL(l1=l, l2=k, gl=g, A=A, algorithm=CONESTA( max_iter=max_iter, eps=eps), penalty_start=1, mean=False) # Fit data with the new regularisation parameters. beta = lr.fit(X, y, beta).beta # Compute output. err_beta[i, j] = np.linalg.norm(beta - beta_star) err_f[i, j] = np.linalg.norm(function.f(beta) - function.f(beta_star)) print "l: %.3f, g: %.3f, err_f: %.12f" % (l, g, err_f[i, j]) print("err_beta:\n", err_beta) print("err_f:\n", err_f)