# map form full to filtered, -1 means not selected
map_full_to_filtered = -np.ones(Xtrain.shape[1], dtype=int)
map_full_to_filtered[filter_] = np.arange((K + 1))
groups_filtered = [map_full_to_filtered[g] for g in groups]
groups_filtered = [g[g != -1] for g in groups_filtered]
groups_filtered = [g for g in groups_filtered if len(g) >= 1]
weights_filtered = [len(g) for g in groups_filtered]
weights_filtered = np.sqrt(np.asarray(weights_filtered))
Agl = gl.linear_operator_from_groups(Xval_filtered.shape[1],
groups=groups_filtered,
weights=weights_filtered,
penalty_start=1)
enet_gl = estimators.LinearRegressionL1L2GL(
l1=l1,
l2=l2,
gl=lgl,
A=Agl,
algorithm=algorithm,
penalty_start=1)
# enet_gl2=ElasticNet(alpha=l2, l1_ratio=l1,)
enet_gl.fit(Xtr_filtered, ytr)
y_pred_test = enet_gl.predict(Xval_filtered)
test_acc = r2_score(yval, y_pred_test)
print test_acc
inner_param[(l1, l2, lgl)].append(test_acc)
inner_param_mean = {
k: np.mean(inner_param[k])
for k in inner_param.keys()
}
print inner_param_mean
l1, l2, lgl = max(inner_param_mean.iteritems(),
mean=True)
enet_PP.fit(X_res, z)
print "Compute beta values"
beta = enet_PP.beta
beta = beta[11:] #do not consider covariates
print "Compute the weights using Parsimony's ElasticNet algorithm."
weights = [
math.pow(abs(beta[j[0]]) + 1 / float(n), -gamma) for j in groups
]
# Adaptive Elasticnet algorithm
adaptive_enet = estimators.LinearRegressionL1L2GL(
l1=0,
l2=0.8,
gl=0.006,
A=gl.A_from_groups(p, groups, weights=weights, penalty_start=11),
algorithm=proximal.FISTA(),
algorithm_params=dict(max_iter=10000),
penalty_start=11,
mean=True)
stime = time.time()
print "================================================================="
print "Now fitting the model"
adaptive_enet.fit(X_res, z)
print "Fit duration : ", time.time() - stime
print "================================================================="
# Interpretation
beta_w = adaptive_enet.beta
plt.plot(beta_w[11:])
l1_max =0.1* np.max(s)/Xtr.shape[0]
print "l1 max is", l1_max
#################################
l1, l2, lgl =l1_max * np.array((0.1, 0.1, 0.01))
weights = [np.sqrt(len(group)) for group in groups]
weights = 1./np.sqrt(np.asarray(weights))
Agl = gl.linear_operator_from_groups(p, groups=groups, weights=weights)
algorithm = algorithms.proximal.CONESTA(eps=consts.TOLERANCE, max_iter=15000)
enet_gl = estimators.LinearRegressionL1L2GL(l1, l2, lgl , Agl, algorithm=algorithm)
yte_pred_enetgl = enet_gl.fit(Xtr, ytr).predict(Xte)
print " r carré vaut", r2_score(yte, yte_pred_enetgl)
Xnon_res = sklearn.preprocessing.scale(Xnon_res,
axis=0,
with_mean=True,
with_std=False)
Ynon_res = Ynon_res-Ynon_res.mean()
n_train = int(X.shape[0]/1.75)
Xtr_res = Xnon_res[:n_train, :]
ytr_res = Ynon_res[:n_train]
Xte_res = Xnon_res[n_train:, :]
def test_group_lasso(self):
random_state = np.random.RandomState(42)
state = random_state.get_state()
rng01 = simulate.utils.RandomUniform(0, 1, random_state=random_state)
rng11 = simulate.utils.RandomUniform(-1, 1, random_state=random_state)
# Generate start values.
n, p = 48, 64 + 1
# Define the groups.
groups = [range(1, 2 * p / 3), range(p / 3, p)]
# Generate candidate data.
beta = simulate.beta.random((p - 1, 1),
density=0.5,
sort=True,
rng=rng01)
# Add the intercept.
beta = np.vstack((random_state.rand(1, 1), beta))
Sigma = simulate.correlation_matrices.constant_correlation(
p=p - 1, rho=0.01, eps=0.001, random_state=random_state)
X0 = random_state.multivariate_normal(np.zeros(p - 1), Sigma, n)
# Add the intercept.
X0 = np.hstack((np.ones((n, 1)), X0))
e = random_state.randn(n, 1)
# Create linear operator.
A = simulate.functions.SmoothedGroupLasso.A_from_groups(
p, groups, weights=None, penalty_start=1)
# Define regularisation parameters.
l = 0.618 # L1 coefficient.
k = 1.0 - l # Ridge (L2) coefficient.
g = 1.618 # TV coefficient.
mu = 5e-4
# Create optimisation problem.
l1 = simulate.functions.L1(l, rng=rng11)
l2 = simulate.functions.L2Squared(k)
gl = simulate.functions.SmoothedGroupLasso(g,
A,
mu=simulate.utils.TOLERANCE)
lr = simulate.LinearRegressionData([l1, l2, gl],
X0,
e,
snr=2.0,
intercept=True)
# Generate simulated data.
random_state.set_state(state)
X, y, beta_star, e = lr.load(beta)
try:
import parsimony.estimators as estimators
import parsimony.algorithms.proximal as proximal
except ImportError:
print "pylearn-parsimony is not properly installed. Will not be " \
"able to run this test."
return
e = estimators.LinearRegressionL1L2GL(
l,
k,
g,
A,
mu,
algorithm=proximal.FISTA(max_iter=10000),
penalty_start=1,
mean=False)
beta_sim = e.fit(X, y, beta).parameters()["beta"]
assert (np.linalg.norm(beta_sim - beta_star) < 0.0001)
mean=True)
enet_PP.fit(X_res, z)
print "Compute beta values"
beta = enet_PP.beta
beta = beta[11:] #do not consider covariates
print "Compute the weights using Parsimony's ElasticNet algorithm."
weights = [
math.pow(abs(beta[j[0]]) + 1 / float(n), -gamma) for j in groups
]
# Adaptive Elasticnet algorithm
A = gl.A_from_groups(p, groups, weights=weights)
adaptive_enet = estimators.LinearRegressionL1L2GL(
l1=0,
l2=0.8,
gl=0.006,
A=A,
algorithm=proximal.FISTA(),
algorithm_params=dict(max_iter=10000),
penalty_start=11,
mean=True)
stime = time.time()
print "================================================================="
print "Now fitting the model"
adaptive_enet.fit(X_res, z)
print "Fit duration : ", time.time() - stime
print "================================================================="
# Interpretation
beta_w = adaptive_enet.beta[11:]
plt.plot(beta_w)
# Create the loss function.
function = LinearRegressionL1L2GL(X,
y,
l,
k,
g,
A=A,
penalty_start=1,
mean=False)
# Create the estimator.
lr = estimators.LinearRegressionL1L2GL(l1=l,
l2=k,
gl=g,
A=A,
algorithm=CONESTA(
max_iter=max_iter, eps=eps),
penalty_start=1,
mean=False)
# Fit data with the new regularisation parameters.
beta = lr.fit(X, y, beta).beta
# Compute output.
err_beta[i, j] = np.linalg.norm(beta - beta_star)
err_f[i, j] = np.linalg.norm(function.f(beta) - function.f(beta_star))
print "l: %.3f, g: %.3f, err_f: %.12f" % (l, g, err_f[i, j])
print("err_beta:\n", err_beta)
print("err_f:\n", err_f)