def test_group_lasso_separable():
"""
This test verifies that the specification of a separable
penalty yields the same results as having two linear_atoms
with selector matrices. The penalty here is a group_lasso, i.e. l2
penalty.
"""
X = np.random.standard_normal((100,20))
Y = np.random.standard_normal((100,)) + np.dot(X, np.random.standard_normal(20))
penalty1 = rr.l2norm(10, lagrange=.2)
penalty2 = rr.l2norm(10, lagrange=.2)
penalty = rr.separable((20,), [penalty1, penalty2], [slice(0,10), slice(10,20)])
# solve using separable
loss = rr.quadratic.affine(X, -Y, coef=0.5)
problem = rr.separable_problem.fromatom(penalty, loss)
solver = rr.FISTA(problem)
solver.fit(min_its=200, tol=1.0e-12)
coefs = solver.composite.coefs
# solve using the selectors
penalty_s = [rr.linear_atom(p, rr.selector(g, (20,))) for p, g in
zip(penalty.atoms, penalty.groups)]
problem_s = rr.container(loss, *penalty_s)
solver_s = rr.FISTA(problem_s)
solver_s.fit(min_its=200, tol=1.0e-12)
coefs_s = solver_s.composite.coefs
np.testing.assert_almost_equal(coefs, coefs_s)
def test_group_lasso_separable():
"""
This test verifies that the specification of a separable
penalty yields the same results as having two linear_atoms
with selector matrices. The penalty here is a group_lasso, i.e. l2
penalty.
"""
X = np.random.standard_normal((100,20))
Y = np.random.standard_normal((100,)) + np.dot(X, np.random.standard_normal(20))
penalty1 = rr.l2norm(10, lagrange=.2)
penalty2 = rr.l2norm(10, lagrange=.2)
penalty = rr.separable((20,), [penalty1, penalty2], [slice(0,10), slice(10,20)])
# solve using separable
loss = rr.quadratic_loss.affine(X, -Y, coef=0.5)
problem = rr.separable_problem.fromatom(penalty, loss)
solver = rr.FISTA(problem)
solver.fit(min_its=200, tol=1.0e-12)
coefs = solver.composite.coefs
# solve using the selectors
penalty_s = [rr.linear_atom(p, rr.selector(g, (20,))) for p, g in
zip(penalty.atoms, penalty.groups)]
problem_s = rr.container(loss, *penalty_s)
solver_s = rr.FISTA(problem_s)
solver_s.fit(min_its=200, tol=1.0e-12)
coefs_s = solver_s.composite.coefs
np.testing.assert_almost_equal(coefs, coefs_s)
def test_path_group_lasso():
'''
this test looks at the paths of three different parameterizations
of the same problem
'''
n = 100
X = np.random.standard_normal((n, 10))
U = np.random.standard_normal((n, 2))
Y = np.random.standard_normal(100)
betaX = np.array([3, 4, 5, 0, 0] + [0] * 5)
betaU = np.array([10, -5])
Y += (np.dot(X, betaX) + np.dot(U, betaU)) * 5
Xn = rr.normalize(np.hstack([np.ones((100, 1)), X]),
inplace=True,
center=True,
scale=True,
intercept_column=0).normalized_array()
lasso = mixed_lasso.mixed_lasso_path.gaussian(Xn[:, 1:],
Y,
penalty_structure=[0] * 7 +
[1] * 3,
nstep=10)
sol = lasso.main(inner_tol=1.e-12, verbose=True)
beta = np.array(sol['beta'].todense())
sols = []
sols_sep = []
for l in sol['lagrange']:
loss = rr.glm.gaussian(Xn, Y)
penalty = rr.mixed_lasso([mixed_lasso.UNPENALIZED] + [0] * 7 + [1] * 3,
lagrange=l) # matrix contains an intercept...
problem = rr.simple_problem(loss, penalty)
sols.append(problem.solve(tol=1.e-12).copy())
sep = rr.separable((11, ), [
rr.l2norm((7, ),
np.sqrt(7) * l),
rr.l2norm((3, ),
np.sqrt(3) * l)
], [np.arange(1, 8), np.arange(8, 11)])
sep_problem = rr.simple_problem(loss, sep)
sols_sep.append(sep_problem.solve(tol=1.e-12).copy())
sols = np.array(sols).T
sols_sep = np.array(sols_sep).T
nt.assert_true(
np.linalg.norm(beta - sols) / (1 + np.linalg.norm(beta)) <= 1.e-4)
nt.assert_true(
np.linalg.norm(beta - sols_sep) / (1 + np.linalg.norm(beta)) <= 1.e-4)
def group_lasso_example():
def selector(p, slice):
return np.identity(p)[slice]
penalties = [R.l2norm(selector(500, slice(i*100,(i+1)*100)), lagrange=.1) for i in range(5)]
penalties[0].lagrange = 250.
penalties[1].lagrange = 225.
penalties[2].lagrange = 150.
penalties[3].lagrange = 100.
X = np.random.standard_normal((1000,500))
Y = np.random.standard_normal((1000,))
loss = R.quadratic.affine(X, -Y, coef=0.5)
group_lasso = R.container(loss, *penalties)
solver=R.FISTA(group_lasso)
solver.debug = True
vals = solver.fit(max_its=2000, min_its=20,tol=1e-10)
soln = solver.composite.coefs
# solution
pylab.figure(num=1)
pylab.clf()
pylab.plot(soln, c='g')
# objective values
pylab.figure(num=2)
pylab.clf()
pylab.plot(vals)
def test_path_group_lasso():
"""
this test looks at the paths of three different parameterizations
of the same problem
"""
n = 100
X = np.random.standard_normal((n, 10))
U = np.random.standard_normal((n, 2))
Y = np.random.standard_normal(100)
betaX = np.array([3, 4, 5, 0, 0] + [0] * 5)
betaU = np.array([10, -5])
Y += (np.dot(X, betaX) + np.dot(U, betaU)) * 5
Xn = rr.normalize(
np.hstack([np.ones((100, 1)), X]), inplace=True, center=True, scale=True, intercept_column=0
).normalized_array()
lasso = rr.lasso.squared_error(Xn[:, 1:], Y, penalty_structure=[0] * 7 + [1] * 3, nstep=10)
sol = lasso.main(inner_tol=1.0e-12, verbose=True)
beta = np.array(sol["beta"].todense())
sols = []
sols_sep = []
for l in sol["lagrange"]:
loss = rr.squared_error(Xn, Y, coef=1.0 / n)
penalty = rr.mixed_lasso([rr.UNPENALIZED] + [0] * 7 + [1] * 3, lagrange=l) # matrix contains an intercept...
problem = rr.simple_problem(loss, penalty)
sols.append(problem.solve(tol=1.0e-12).copy())
sep = rr.separable(
(11,),
[rr.l2norm((7,), np.sqrt(7) * l), rr.l2norm((3,), np.sqrt(3) * l)],
[np.arange(1, 8), np.arange(8, 11)],
)
sep_problem = rr.simple_problem(loss, sep)
sols_sep.append(sep_problem.solve(tol=1.0e-12).copy())
sols = np.array(sols).T
sols_sep = np.array(sols_sep).T
nt.assert_true(np.linalg.norm(beta - sols) / (1 + np.linalg.norm(beta)) <= 1.0e-4)
nt.assert_true(np.linalg.norm(beta - sols_sep) / (1 + np.linalg.norm(beta)) <= 1.0e-4)
def group_lasso_signal_approx():
def selector(p, slice):
return np.identity(p)[slice]
penalties = [R.l2norm(selector(500, slice(i*100,(i+1)*100)), lagrange=10.) for i in range(5)]
loss = R.quadratic.shift(-x, coef=0.5)
group_lasso = R.container(loss, **penalties)
x = np.random.standard_normal(500)
solver = R.FISTA(group_lasso)
solver.fit()
a = solver.composite.coefs
Y = np.random.standard_normal((500,)) + np.dot(X, beta)
Xnorm = scipy.linalg.eigvalsh(np.dot(X.T,X), eigvals=(998,999)).max()
import regreg.api as R
from regreg.smooth import linear
smooth_linf_constraint = R.smoothed_atom(R.maxnorm(1000, bound=1),
epsilon=0.01,
store_argmin=True)
loss = R.affine_smooth(smooth_linf_constraint, -X.T, None)
smooth_f = R.smooth_function(loss, linear(Y))
norm_Y = np.linalg.norm(Y)
l2_constraint_value = np.sqrt(0.1) * norm_Y
l2_lagrange = R.l2norm(500, lagrange=l2_constraint_value)
basis_pursuit = R.container(smooth_f, l2_lagrange)
solver = R.FISTA(basis_pursuit.composite(initial=np.random.standard_normal(500)))
tol = 1.0e-08
solver = R.FISTA(basis_pursuit.composite(initial=np.random.standard_normal(500)))
for epsilon in [0.6**i for i in range(20)]:
smooth_linf_constraint.epsilon = epsilon
solver.composite.lipshitz = 1.1/epsilon * Xnorm
solver.fit(max_its=2000, tol=tol, min_its=10, backtrack=False)
basis_pursuit_soln = smooth_linf_constraint.argmin
sparsity = R.l1norm(1000, bound=np.fabs(basis_pursuit_soln).sum())
loss = R.l2normsq.affine(X, -Y)
"""
Solving basis pursuit with TFOCS
"""
import regreg.api as rr
import numpy as np
import nose.tools as nt
n, p = 100, 200
X = np.random.standard_normal((n, p))
beta = np.zeros(p)
beta[:4] = 3
Y = np.random.standard_normal(n) + np.dot(X, beta)
lscoef = np.dot(np.linalg.pinv(X), Y)
minimum_l2 = np.linalg.norm(Y - np.dot(X, lscoef))
maximum_l2 = np.linalg.norm(Y)
l2bound = (minimum_l2 + maximum_l2) * 0.5
l2 = rr.l2norm(n, bound=l2bound)
T = rr.affine_transform(X, -Y)
l1 = rr.l1norm(p, lagrange=1)
primal, dual = rr.tfocs(l1, T, l2, tol=1.e-10)
nt.assert_true(
np.fabs(np.linalg.norm(Y - np.dot(X, primal)) - l2bound) <= l2bound *
1.e-5)
beta[:100] = 3 * np.sqrt(2 * np.log(1000))
Y = np.random.standard_normal((500, )) + np.dot(X, beta)
Xnorm = scipy.linalg.eigvalsh(np.dot(X.T, X), eigvals=(998, 999)).max()
import regreg.api as R
from regreg.smooth import linear
smooth_linf_constraint = R.smoothed_atom(R.supnorm(1000, bound=1),
epsilon=0.01,
store_argmin=True)
transform = R.linear_transform(-X.T)
loss = R.affine_smooth(smooth_linf_constraint, transform)
norm_Y = np.linalg.norm(Y)
l2_constraint_value = np.sqrt(0.1) * norm_Y
l2_lagrange = R.l2norm(500, lagrange=l2_constraint_value)
basis_pursuit = R.container(loss, linear(Y), l2_lagrange)
solver = R.FISTA(basis_pursuit)
tol = 1.0e-08
for epsilon in [0.6**i for i in range(20)]:
smooth_linf_constraint.epsilon = epsilon
solver.composite.lipschitz = 1.1 / epsilon * Xnorm
solver.fit(max_its=2000, tol=tol, min_its=10, backtrack=False)
basis_pursuit_soln = smooth_linf_constraint.sm_atom.argmin
sparsity = R.l1norm(1000, bound=np.fabs(basis_pursuit_soln).sum())
loss = R.quadratic.affine(X, -Y)
lasso = R.container(loss, sparsity)
def ridge_bound(self):
return rr.l2norm(self.p,bound=self.bound2)
""" Solving basis pursuit with TFOCS """ import regreg.api as rr import numpy as np import nose.tools as nt n, p = 100, 200 X = np.random.standard_normal((n, p)) beta = np.zeros(p) beta[:4] = 3 Y = np.random.standard_normal(n) + np.dot(X, beta) lscoef = np.dot(np.linalg.pinv(X), Y) minimum_l2 = np.linalg.norm(Y - np.dot(X, lscoef)) maximum_l2 = np.linalg.norm(Y) l2bound = (minimum_l2 + maximum_l2) * 0.5 l2 = rr.l2norm(n, bound=l2bound) T = rr.affine_transform(X, -Y) l1 = rr.l1norm(p, lagrange=1) primal, dual = rr.tfocs(l1, T, l2, tol=1.0e-10) nt.assert_true(np.fabs(np.linalg.norm(Y - np.dot(X, primal)) - l2bound) <= l2bound * 1.0e-5)