def test_lasso():
"""
this test verifies that the l1 prox can be solved
by a primal/dual specification
obviously, we don't to solve the l1 prox this way,
but it verifies that specification is working correctly
"""
l1 = rr.l1norm(4, lagrange=2.0)
l11 = rr.l1norm(4, lagrange=1.0)
l12 = rr.l1norm(4, lagrange=1.0)
X = np.random.standard_normal((10, 4))
Y = np.random.standard_normal(10) + 3
loss = rr.quadratic.affine(X, -Y)
p1 = rr.container(l11, loss, l12)
solver1 = rr.FISTA(p1)
solver1.fit(tol=1.0e-12, min_its=500)
p2 = rr.separable_problem.singleton(l1, loss)
solver2 = rr.FISTA(p2)
solver2.fit(tol=1.0e-12)
f = p2.objective
ans = scipy.optimize.fmin_powell(f, np.zeros(4), ftol=1.0e-12)
print f(solver2.composite.coefs), f(ans)
print f(solver1.composite.coefs), f(ans)
yield ac, ans, solver2.composite.coefs, "singleton solver"
yield ac, solver1.composite.coefs, solver2.composite.coefs, "container solver"
def test_multiple_lasso_dual(n=500):
"""
Check that the solution of the lasso signal approximator dual composite is soft-thresholding even when specified with multiple seminorms
"""
l1 = 1
sparsity1 = R.l1norm(n, lagrange=l1 * 0.75)
sparsity2 = R.l1norm(n, lagrange=l1 * 0.25)
x = np.random.normal(0, 1, n)
loss = R.quadratic.shift(-x, coef=0.5)
p = R.dual_problem.fromprimal(loss, sparsity1, sparsity2)
t1 = time.time()
solver = R.FISTA(p)
solver.debug = True
vals = solver.fit(tol=1.0e-16)
soln = p.primal
t2 = time.time()
print t2 - t1
st = np.maximum(np.fabs(x) - l1, 0) * np.sign(x)
np.testing.assert_almost_equal(soln, st, decimal=3)
p = R.container(loss, sparsity1, sparsity2)
t1 = time.time()
solver = R.FISTA(p)
solver.debug = True
vals = solver.fit(tol=1.0e-16)
soln = p.primal
t2 = time.time()
print t2 - t1
st = np.maximum(np.fabs(x) - l1, 0) * np.sign(x)
print soln[range(10)]
print st[range(10)]
np.testing.assert_almost_equal(soln, st, decimal=3)
def test_lasso():
'''
this test verifies that the l1 prox can be solved
by a primal/dual specification
obviously, we don't to solve the l1 prox this way,
but it verifies that specification is working correctly
'''
l1 = rr.l1norm(4, lagrange=2.)
l11 = rr.l1norm(4, lagrange=1.)
l12 = rr.l1norm(4, lagrange=1.)
X = np.random.standard_normal((10, 4))
Y = np.random.standard_normal(10) + 3
loss = rr.quadratic.affine(X, -Y)
p1 = rr.container(l11, loss, l12)
solver1 = rr.FISTA(p1)
solver1.fit(tol=1.0e-12, min_its=500)
p2 = rr.separable_problem.singleton(l1, loss)
solver2 = rr.FISTA(p2)
solver2.fit(tol=1.0e-12)
f = p2.objective
ans = scipy.optimize.fmin_powell(f, np.zeros(4), ftol=1.0e-12)
print(f(solver2.composite.coefs), f(ans))
print(f(solver1.composite.coefs), f(ans))
yield all_close, ans, solver2.composite.coefs, 'singleton solver', None
yield all_close, solver1.composite.coefs, solver2.composite.coefs, 'container solver', None
def test_conjugate_solver():
# Solve Lagrange problem
Y = np.random.standard_normal(500)
Y[100:150] += 7
Y[250:300] += 14
loss = R.quadratic.shift(-Y, coef=0.5)
sparsity = R.l1norm(len(Y), lagrange=1.4)
D = sparse.csr_matrix((np.identity(500) + np.diag([-1] * 499, k=1))[:-1])
fused = R.l1norm.linear(D, lagrange=25.5)
problem = R.container(loss, sparsity, fused)
solver = R.FISTA(problem)
solver.fit(max_its=500, tol=1e-10)
solution = solver.composite.coefs
# Solve constrained version
delta1 = np.fabs(D * solution).sum()
delta2 = np.fabs(solution).sum()
fused_constraint = R.l1norm.linear(D, bound=delta1)
sparsity_constraint = R.l1norm(500, bound=delta2)
constrained_problem = R.container(loss, fused_constraint,
sparsity_constraint)
constrained_solver = R.FISTA(constrained_problem)
vals = constrained_solver.fit(max_its=500, tol=1e-10)
constrained_solution = constrained_solver.composite.coefs
npt.assert_almost_equal(np.fabs(constrained_solution).sum(), delta2, 3)
npt.assert_almost_equal(np.fabs(D * constrained_solution).sum(), delta1, 3)
# Solve with (shifted) conjugate function
loss = R.quadratic.shift(-Y, coef=0.5)
true_conjugate = R.quadratic.shift(Y, coef=0.5)
problem = R.container(loss, fused_constraint, sparsity_constraint)
solver = R.FISTA(problem.conjugate_composite(true_conjugate))
solver.fit(max_its=500, tol=1e-10)
conjugate_coefs = problem.conjugate_primal_from_dual(
solver.composite.coefs)
# Solve with generic conjugate function
loss = R.quadratic.shift(-Y, coef=0.5)
problem = R.container(loss, fused_constraint, sparsity_constraint)
solver2 = R.FISTA(problem.conjugate_composite(conjugate_tol=1e-12))
solver2.fit(max_its=500, tol=1e-10)
conjugate_coefs_gen = problem.conjugate_primal_from_dual(
solver2.composite.coefs)
d1 = np.linalg.norm(solution -
constrained_solution) / np.linalg.norm(solution)
d2 = np.linalg.norm(solution - conjugate_coefs) / np.linalg.norm(solution)
d3 = np.linalg.norm(solution -
conjugate_coefs_gen) / np.linalg.norm(solution)
npt.assert_array_less(d1, 0.01)
npt.assert_array_less(d2, 0.01)
npt.assert_array_less(d3, 0.01)
def test_lasso_separable():
"""
This test verifies that the specification of a separable
penalty yields the same results as having two linear_atoms
with selector matrices. The penalty here is a lasso, i.e. l1
penalty.
"""
X = np.random.standard_normal((100,20))
Y = np.random.standard_normal((100,)) + np.dot(X, np.random.standard_normal(20))
penalty1 = rr.l1norm(10, lagrange=1.2)
penalty2 = rr.l1norm(10, lagrange=1.2)
penalty = rr.separable((20,), [penalty1, penalty2], [slice(0,10), slice(10,20)], test_for_overlap=True)
# ensure code is tested
print(penalty1.latexify())
print(penalty.latexify())
print(penalty.conjugate)
print(penalty.dual)
print(penalty.seminorm(np.ones(penalty.shape)))
print(penalty.constraint(np.ones(penalty.shape), bound=2.))
pencopy = copy(penalty)
pencopy.set_quadratic(rr.identity_quadratic(1,0,0,0))
pencopy.conjugate
# solve using separable
loss = rr.quadratic.affine(X, -Y, coef=0.5)
problem = rr.separable_problem.fromatom(penalty, loss)
solver = rr.FISTA(problem)
solver.fit(min_its=200, tol=1.0e-12)
coefs = solver.composite.coefs
# solve using the usual composite
penalty_all = rr.l1norm(20, lagrange=1.2)
problem_all = rr.container(loss, penalty_all)
solver_all = rr.FISTA(problem_all)
solver_all.fit(min_its=100, tol=1.0e-12)
coefs_all = solver_all.composite.coefs
# solve using the selectors
penalty_s = [rr.linear_atom(p, rr.selector(g, (20,))) for p, g in
zip(penalty.atoms, penalty.groups)]
problem_s = rr.container(loss, *penalty_s)
solver_s = rr.FISTA(problem_s)
solver_s.fit(min_its=500, tol=1.0e-12)
coefs_s = solver_s.composite.coefs
np.testing.assert_almost_equal(coefs, coefs_all)
np.testing.assert_almost_equal(coefs, coefs_s)
def test_lasso_separable():
"""
This test verifies that the specification of a separable
penalty yields the same results as having two linear_atoms
with selector matrices. The penalty here is a lasso, i.e. l1
penalty.
"""
X = np.random.standard_normal((100,20))
Y = np.random.standard_normal((100,)) + np.dot(X, np.random.standard_normal(20))
penalty1 = rr.l1norm(10, lagrange=1.2)
penalty2 = rr.l1norm(10, lagrange=1.2)
penalty = rr.separable((20,), [penalty1, penalty2], [slice(0,10), slice(10,20)], test_for_overlap=True)
# ensure code is tested
print(penalty1.latexify())
print(penalty.latexify())
print(penalty.conjugate)
print(penalty.dual)
print(penalty.seminorm(np.ones(penalty.shape)))
print(penalty.constraint(np.ones(penalty.shape), bound=2.))
pencopy = copy(penalty)
pencopy.set_quadratic(rr.identity_quadratic(1,0,0,0))
pencopy.conjugate
# solve using separable
loss = rr.quadratic_loss.affine(X, -Y, coef=0.5)
problem = rr.separable_problem.fromatom(penalty, loss)
solver = rr.FISTA(problem)
solver.fit(min_its=200, tol=1.0e-12)
coefs = solver.composite.coefs
# solve using the usual composite
penalty_all = rr.l1norm(20, lagrange=1.2)
problem_all = rr.container(loss, penalty_all)
solver_all = rr.FISTA(problem_all)
solver_all.fit(min_its=100, tol=1.0e-12)
coefs_all = solver_all.composite.coefs
# solve using the selectors
penalty_s = [rr.linear_atom(p, rr.selector(g, (20,))) for p, g in
zip(penalty.atoms, penalty.groups)]
problem_s = rr.container(loss, *penalty_s)
solver_s = rr.FISTA(problem_s)
solver_s.fit(min_its=500, tol=1.0e-12)
coefs_s = solver_s.composite.coefs
np.testing.assert_almost_equal(coefs, coefs_all)
np.testing.assert_almost_equal(coefs, coefs_s)
def test_conjugate_solver():
# Solve Lagrange problem
Y = np.random.standard_normal(500); Y[100:150] += 7; Y[250:300] += 14
loss = R.quadratic.shift(-Y, coef=0.5)
sparsity = R.l1norm(len(Y), lagrange = 1.4)
D = sparse.csr_matrix((np.identity(500) + np.diag([-1]*499,k=1))[:-1])
fused = R.l1norm.linear(D, lagrange = 25.5)
problem = R.container(loss, sparsity, fused)
solver = R.FISTA(problem)
solver.fit(max_its=500, tol=1e-10)
solution = solver.composite.coefs
# Solve constrained version
delta1 = np.fabs(D * solution).sum()
delta2 = np.fabs(solution).sum()
fused_constraint = R.l1norm.linear(D, bound = delta1)
sparsity_constraint = R.l1norm(500, bound = delta2)
constrained_problem = R.container(loss, fused_constraint, sparsity_constraint)
constrained_solver = R.FISTA(constrained_problem)
vals = constrained_solver.fit(max_its=500, tol=1e-10)
constrained_solution = constrained_solver.composite.coefs
npt.assert_almost_equal(np.fabs(constrained_solution).sum(), delta2, 3)
npt.assert_almost_equal(np.fabs(D * constrained_solution).sum(), delta1, 3)
# Solve with (shifted) conjugate function
loss = R.quadratic.shift(-Y, coef=0.5)
true_conjugate = R.quadratic.shift(Y, coef=0.5)
problem = R.container(loss, fused_constraint, sparsity_constraint)
solver = R.FISTA(problem.conjugate_composite(true_conjugate))
solver.fit(max_its=500, tol=1e-10)
conjugate_coefs = problem.conjugate_primal_from_dual(solver.composite.coefs)
# Solve with generic conjugate function
loss = R.quadratic.shift(-Y, coef=0.5)
problem = R.container(loss, fused_constraint, sparsity_constraint)
solver2 = R.FISTA(problem.conjugate_composite(conjugate_tol=1e-12))
solver2.fit(max_its=500, tol=1e-10)
conjugate_coefs_gen = problem.conjugate_primal_from_dual(solver2.composite.coefs)
d1 = np.linalg.norm(solution - constrained_solution) / np.linalg.norm(solution)
d2 = np.linalg.norm(solution - conjugate_coefs) / np.linalg.norm(solution)
d3 = np.linalg.norm(solution - conjugate_coefs_gen) / np.linalg.norm(solution)
npt.assert_array_less(d1, 0.01)
npt.assert_array_less(d2, 0.01)
npt.assert_array_less(d3, 0.01)
def example4(lambda1=10):
#Example with an initial value for backtracking
# In the previous examples you'll see a lot of "Increasing inv_step" iterations - these are trying to find an approximate Lipschitz constant in a backtracking loop.
# For your problem the Lipschitz constant is just the largest eigenvalue of X^TX, so you can precompute this with a few power iterations.
n = 100
p = 1000
X = np.random.standard_normal(n*p).reshape((n,p))
Y = 10*np.random.standard_normal(n)
v = np.random.standard_normal(p)
for i in range(10):
v = np.dot(X.T, np.dot(X,v))
norm = np.linalg.norm(v)
v /= norm
print "Approximate Lipschitz constant is", norm
loss = rr.l2normsq.affine(X,-Y,coef=1.)
sparsity = rr.l1norm(p, lagrange = lambda1)
nonnegative = rr.nonnegative(p)
problem = rr.container(loss, sparsity, nonnegative)
solver = rr.FISTA(problem)
#Give approximate Lipschitz constant to solver
solver.fit(debug=True, start_inv_step=norm)
solution = solver.composite.coefs
def test_conjugate_l1norm():
'''
this test verifies that numerically computing the conjugate
is essentially the same as using the smooth_conjugate
of the atom
'''
q = rr.identity_quadratic(1.2, 0, 0, 0)
l1 = rr.l1norm(4, lagrange=0.3)
pen2 = copy(l1)
pen2.set_quadratic(q)
v1 = rr.smooth_conjugate(l1, q)
v2 = rr.conjugate(l1, q, tol=1.e-12, min_its=100)
v3 = rr.conjugate(pen2, None, tol=1.e-12, min_its=100)
w = np.random.standard_normal(4)
u11, u12 = v1.smooth_objective(w)
u21, u22 = v2.smooth_objective(w)
u31, u32 = v3.smooth_objective(w)
np.testing.assert_approx_equal(u11, u21)
np.testing.assert_allclose(u12, u22, rtol=1.0e-05)
np.testing.assert_approx_equal(u11, u31)
np.testing.assert_allclose(u12, u32, rtol=1.0e-05)
v2.smooth_objective(w, mode='func')
v2.smooth_objective(w, mode='grad')
nt.assert_raises(ValueError, v2.smooth_objective, w, 'blah')
def choose_lambda_CVr(self, scale=1., loss=None):
"""
Minimizes CV error curve without randomization and the one with residual randomization
"""
if loss is None:
loss = self.loss
if not hasattr(self, 'scale'):
self.scale = scale
CV_curve = []
X, _ = loss.data
p = X.shape[1]
for lam in self.lam_seq:
penalty = rr.l1norm(p, lagrange=lam)
# CV_curve.append(self.CV_err(penalty, loss) + (lam,))
CV_curve.append(
self.CV_err(penalty,
loss,
residual_randomization=True,
scale=self.scale))
CV_curve = np.array(CV_curve)
CV_val = CV_curve[:, 0]
CV_val_randomized = CV_curve[:, 2]
lam_CV = self.lam_seq[np.argmin(CV_val)]
lam_CV_randomized = self.lam_seq[np.argmin(CV_val_randomized)]
SD_val = CV_curve[:, 1]
SD_val_randomized = CV_curve[:, 3]
return lam_CV, CV_val, SD_val, lam_CV_randomized, CV_val_randomized, SD_val_randomized
def test_lasso(n=100):
l1 = 1.
sparsity = R.l1norm(n, lagrange=l1)
X = np.random.standard_normal((5000,n))
Y = np.random.standard_normal((5000,))
regloss = R.quadratic.affine(-X,Y)
p=R.container(regloss, sparsity)
solver=R.FISTA(p)
solver.debug = True
t1 = time.time()
vals1 = solver.fit(max_its=800)
t2 = time.time()
dt1 = t2 - t1
soln = solver.composite.coefs
time.sleep(5)
print soln[range(10)]
print solver.composite.objective(soln)
print "Times", dt1
def test_conjugate_l1norm():
'''
this test verifies that numerically computing the conjugate
is essentially the same as using the smooth_conjugate
of the atom
'''
q = rr.identity_quadratic(1.2,0,0,0)
l1 = rr.l1norm(4, lagrange=0.3)
pen2 = copy(l1)
pen2.set_quadratic(q)
v1 = rr.smooth_conjugate(l1, q)
v2 = rr.conjugate(l1, q, tol=1.e-12, min_its=100)
v3 = rr.conjugate(pen2, None, tol=1.e-12, min_its=100)
w = np.random.standard_normal(4)
u11, u12 = v1.smooth_objective(w)
u21, u22 = v2.smooth_objective(w)
u31, u32 = v3.smooth_objective(w)
np.testing.assert_approx_equal(u11, u21)
np.testing.assert_allclose(u12, u22, rtol=1.0e-05)
np.testing.assert_approx_equal(u11, u31)
np.testing.assert_allclose(u12, u32, rtol=1.0e-05)
v2.smooth_objective(w, mode='func')
v2.smooth_objective(w, mode='grad')
nt.assert_raises(ValueError, v2.smooth_objective, w, 'blah')
def test_nesta_nonnegative():
state = np.random.get_state()
np.random.seed(10)
n, p, q = 1000, 20, 5
X = np.random.standard_normal((n, p))
A = np.random.standard_normal((q,p))
coef = 10 * np.fabs(np.random.standard_normal(q)) + 1
coef[:2] = -0.2
beta = np.dot(np.linalg.pinv(A), coef)
print(r'\beta', beta)
print(r'A\beta', np.dot(A, beta))
Y = np.random.standard_normal(n) + np.dot(X, beta)
loss = rr.squared_error(X,Y)
penalty = rr.l1norm(p, lagrange=0.2)
constraint = rr.nonnegative.linear(A)
primal, dual = rr.nesta(loss, penalty, constraint, max_iters=300, coef_tol=1.e-10, tol=1.e-10)
print(r'A \hat{\beta}', np.dot(A, primal))
assert_almost_nonnegative(np.dot(A,primal), tol=1.e-3)
np.random.set_state(state)
def test_changepoint():
p = 150
M = multiscale(p)
M.minsize = 10
X = ra.adjoint(M)
Y = np.random.standard_normal(p)
Y[20:50] += 8
Y += 2
meanY = Y.mean()
lammax = np.fabs(X.adjoint_map(Y)).max()
penalty = rr.l1norm(X.input_shape, lagrange=0.5*lammax)
loss = rr.squared_error(X, Y - meanY)
problem = rr.simple_problem(loss, penalty)
soln = problem.solve()
Yhat = X.linear_map(soln)
Yhat += meanY
if INTERACTIVE:
plt.scatter(np.arange(p), Y)
plt.plot(np.arange(p), Yhat)
plt.show()
def test_nesta_lasso():
n, p = 1000, 20
X = np.random.standard_normal((n, p))
beta = np.zeros(p)
beta[:4] = 30
Y = np.random.standard_normal(n) + np.dot(X, beta)
loss = rr.squared_error(X, Y)
penalty = rr.l1norm(p, lagrange=4.)
# using nesta
z = rr.zero(p)
primal, dual = rr.nesta(loss,
z,
penalty,
tol=1.e-10,
epsilon=2.**(-np.arange(30)))
# using simple problem
problem = rr.simple_problem(loss, penalty)
problem.solve()
nt.assert_true(
np.linalg.norm(primal - problem.coefs) /
np.linalg.norm(problem.coefs) < 1.e-3)
def test_l1prox_bound():
'''
this test verifies that the l1 prox in bound form can be solved
by a primal/dual specification
obviously, we don't to solve the l1 prox this way,
but it verifies that specification is working correctly
'''
l1 = rr.l1norm(4, bound=2.)
ww = np.random.standard_normal(4)*2
ab = l1.proximal(rr.identity_quadratic(0.5, ww, 0, 0))
l1c = copy(l1)
l1c.quadratic = rr.identity_quadratic(0.5, ww, None, 0.)
a = rr.simple_problem.nonsmooth(l1c)
solver = rr.FISTA(a)
solver.fit(min_its=100)
l1c = copy(l1)
l1c.quadratic = rr.identity_quadratic(0.5, ww, None, 0.)
a = rr.dual_problem.fromprimal(l1c)
solver = rr.FISTA(a)
solver.fit(min_its=100)
ac = a.primal
np.testing.assert_allclose(ac + 0.1, ab + 0.1, rtol=1.e-4)
def test_lasso():
'''
this test verifies that the l1 prox can be solved
by a primal/dual specification
obviously, we don't to solve the l1 prox this way,
but it verifies that specification is working correctly
'''
l1 = rr.l1norm(4, lagrange=2.)
l1.quadratic = rr.identity_quadratic(0.5, 0, None, 0.)
X = np.random.standard_normal((10,4))
Y = np.random.standard_normal(10) + 3
loss = rr.quadratic.affine(X, -Y, coef=0.5)
p2 = rr.separable_problem.singleton(l1, loss)
solver2 = rr.FISTA(p2)
solver2.fit(tol=1.0e-14, min_its=100)
f = p2.objective
ans = scipy.optimize.fmin_powell(f, np.zeros(4), ftol=1.0e-12, xtol=1.e-10)
print(f(solver2.composite.coefs), f(ans))
np.testing.assert_allclose(ans + 0.1, solver2.composite.coefs + 0.1, rtol=1.e-3)
def test_l1prox():
'''
this test verifies that the l1 prox in lagrange form can be solved
by a primal/dual specification
obviously, we don't to solve the l1 prox this way,
but it verifies that specification is working correctly
'''
l1 = rr.l1norm(4, lagrange=0.3)
ww = np.random.standard_normal(4)*3
ab = l1.proximal(rr.identity_quadratic(0.5, ww, 0,0))
l1c = copy(l1)
l1c.quadratic = rr.identity_quadratic(0.5, ww, None, 0.)
a = rr.simple_problem.nonsmooth(l1c)
solver = rr.FISTA(a)
solver.fit(tol=1.e-10)
ad = a.coefs
l1c = copy(l1)
l1c.quadratic = rr.identity_quadratic(0.5, ww, None, 0.)
a = rr.dual_problem.fromprimal(l1c)
solver = rr.FISTA(a)
solver.fit(tol=1.0e-14)
ac = a.primal
np.testing.assert_allclose(ac, ab, rtol=1.0e-4)
np.testing.assert_allclose(ac, ad, rtol=1.0e-4)
def test_lasso():
'''
this test verifies that the l1 prox can be solved
by a primal/dual specification
obviously, we don't to solve the l1 prox this way,
but it verifies that specification is working correctly
'''
l1 = rr.l1norm(4, lagrange=2.)
l1.quadratic = rr.identity_quadratic(0.5, 0, None, 0.)
X = np.random.standard_normal((10, 4))
Y = np.random.standard_normal(10) + 3
loss = rr.quadratic_loss.affine(X, -Y, coef=0.5)
p2 = rr.separable_problem.singleton(l1, loss)
solver2 = rr.FISTA(p2)
solver2.fit(tol=1.0e-14, min_its=100)
f = p2.objective
ans = scipy.optimize.fmin_powell(f, np.zeros(4), ftol=1.0e-12, xtol=1.e-10)
print(f(solver2.composite.coefs), f(ans))
np.testing.assert_allclose(ans + 0.1,
solver2.composite.coefs + 0.1,
rtol=1.e-3)
def test_nesta_lasso():
n, p = 1000, 20
X = np.random.standard_normal((n, p))
beta = np.zeros(p)
beta[:4] = 30
Y = np.random.standard_normal(n) + np.dot(X, beta)
loss = rr.squared_error(X,Y)
penalty = rr.l1norm(p, lagrange=2.)
# using nesta
z = rr.zero(p)
primal, dual = rr.nesta(loss, z, penalty, tol=1.e-10,
epsilon=2.**(-np.arange(30)),
initial_dual=np.zeros(p))
# using simple problem
problem = rr.simple_problem(loss, penalty)
problem.solve()
nt.assert_true(np.linalg.norm(primal - problem.coefs) / np.linalg.norm(problem.coefs) < 1.e-3)
# test None as smooth_atom
rr.nesta(None, z, penalty, tol=1.e-10,
epsilon=2.**(-np.arange(30)),
initial_dual=np.zeros(p))
# using coefficients to stop
rr.nesta(loss, z, penalty, tol=1.e-10,
epsilon=2.**(-np.arange(30)),
initial_dual=np.zeros(p),
coef_stop=True)
def fused_example():
x=np.random.standard_normal(500); x[100:150] += 7
sparsity = R.l1norm(500, lagrange=1.3)
D = (np.identity(500) + np.diag([-1]*499,k=1))[:-1]
fused = R.l1norm.linear(D, lagrange=10.5)
loss = R.quadratic.shift(-x, coef=0.5)
pen = R.container(loss, sparsity,fused)
solver = R.FISTA(pen)
vals = solver.fit()
soln = solver.composite.coefs
# solution
pylab.figure(num=1)
pylab.clf()
pylab.plot(soln, c='g')
pylab.scatter(np.arange(x.shape[0]), x)
# objective values
pylab.figure(num=2)
pylab.clf()
pylab.plot(vals)
def test_lasso_dual():
"""
Check that the solution of the lasso signal approximator dual composite is soft-thresholding
"""
l1 = .1
sparsity = R.l1norm(10, lagrange=l1)
x = np.arange(10) - 5
loss = R.quadratic.shift(-x, coef=0.5)
pen = R.simple_problem(loss, sparsity)
solver = R.FISTA(pen)
pen.lipschitz = 1
solver.fit(backtrack=False)
soln = solver.composite.coefs
st = np.maximum(np.fabs(x)-l1,0) * np.sign(x)
np.testing.assert_almost_equal(soln,st, decimal=3)
pen = R.simple_problem(loss, sparsity)
solver = R.FISTA(pen)
solver.fit(monotonicity_restart=False)
soln = solver.composite.coefs
st = np.maximum(np.fabs(x)-l1,0) * np.sign(x)
np.testing.assert_almost_equal(soln,st, decimal=3)
pen = R.container(loss, sparsity)
solver = R.FISTA(pen)
solver.fit()
soln = solver.composite.coefs
np.testing.assert_almost_equal(soln,st, decimal=3)
def test_nesta_nonnegative():
state = np.random.get_state()
np.random.seed(10)
n, p, q = 1000, 20, 5
X = np.random.standard_normal((n, p))
A = np.random.standard_normal((q, p))
coef = 10 * np.fabs(np.random.standard_normal(q)) + 1
coef[:2] = -0.2
beta = np.dot(np.linalg.pinv(A), coef)
print(r'\beta', beta)
print(r'A\beta', np.dot(A, beta))
Y = np.random.standard_normal(n) + np.dot(X, beta)
loss = rr.squared_error(X, Y)
penalty = rr.l1norm(p, lagrange=0.2)
constraint = rr.nonnegative.linear(A)
primal, dual = rr.nesta(loss,
penalty,
constraint,
max_iters=300,
coef_tol=1.e-10,
tol=1.e-10)
print(r'A \hat{\beta}', np.dot(A, primal))
assert_almost_nonnegative(np.dot(A, primal), tol=1.e-3)
np.random.set_state(state)
def choose_lambda_CVR(self, scale1=None, scale2=None, loss=None):
"""
Minimizes CV error curve with additive randomization (CVR=CV+R1+R2=CV1+R2)
"""
if loss is None:
loss = copy.copy(self.loss)
CV_curve = []
X, _ = loss.data
p = X.shape[1]
for lam in self.lam_seq:
penalty = rr.l1norm(p, lagrange=lam)
#CV_curve.append(self.CV_err(penalty, loss) + (lam,))
CV_curve.append(self.CV_err(penalty, loss))
CV_curve = np.array(CV_curve)
rv1, rv2 = np.zeros(self.lam_seq.shape[0]), np.zeros(
self.lam_seq.shape[0])
if scale1 is not None:
randomization1 = randomization.isotropic_gaussian(
(self.lam_seq.shape[0], ), scale=scale1)
rv1 = np.asarray(randomization1._sampler(size=(1, )))
if scale2 is not None:
randomization2 = randomization.isotropic_gaussian(
(self.lam_seq.shape[0], ), scale=scale2)
rv2 = np.asarray(randomization2._sampler(size=(1, )))
CVR_val = CV_curve[:, 0] + rv1.flatten() + rv2.flatten()
lam_CVR = self.lam_seq[np.argmin(CVR_val)] # lam_CVR minimizes CVR
CV1_val = CV_curve[:, 0] + rv1.flatten()
SD = CV_curve[:, 1]
return lam_CVR, SD, CVR_val, CV1_val, self.lam_seq
def test_nesta_nonnegative():
n, p, q = 1000, 20, 5
X = np.random.standard_normal((n, p))
A = np.random.standard_normal((q, p))
coef = 10 * np.fabs(np.random.standard_normal(q)) + 1
coef[:2] = -0.2
beta = np.dot(np.linalg.pinv(A), coef)
Y = np.random.standard_normal(n) + np.dot(X, beta)
loss = rr.squared_error(X, Y)
penalty = rr.l1norm(p, lagrange=0.2)
constraint = rr.nonnegative.linear(A)
primal, dual = rr.nesta(loss,
penalty,
constraint,
max_iters=300,
coef_tol=1.e-4,
tol=1.e-4)
print np.dot(A, primal)
assert_almost_nonnegative(np.dot(A, primal), tol=1.e-3)
def test_l1prox_bound():
'''
this test verifies that the l1 prox in bound form can be solved
by a primal/dual specification
obviously, we don't to solve the l1 prox this way,
but it verifies that specification is working correctly
'''
l1 = rr.l1norm(4, bound=2.)
ww = np.random.standard_normal(4) * 2
ab = l1.proximal(rr.identity_quadratic(0.5, ww, 0, 0))
l1c = copy(l1)
l1c.quadratic = rr.identity_quadratic(0.5, ww, None, 0.)
a = rr.simple_problem.nonsmooth(l1c)
solver = rr.FISTA(a)
solver.fit(min_its=100)
l1c = copy(l1)
l1c.quadratic = rr.identity_quadratic(0.5, ww, None, 0.)
a = rr.dual_problem.fromprimal(l1c)
solver = rr.FISTA(a)
solver.fit(min_its=100)
ac = a.primal
np.testing.assert_allclose(ac + 0.1, ab + 0.1, rtol=1.e-4)
def test_lasso_separable():
"""
This test verifies that the specification of a separable
penalty yields the same results as having two linear_atoms
with selector matrices. The penalty here is a lasso, i.e. l1
penalty.
"""
X = np.random.standard_normal((100, 20))
Y = np.random.standard_normal(
(100, )) + np.dot(X, np.random.standard_normal(20))
penalty1 = rr.l1norm(10, lagrange=1.2)
penalty2 = rr.l1norm(10, lagrange=1.2)
penalty = rr.separable((20, ), [penalty1, penalty2],
[slice(0, 10), slice(10, 20)])
# solve using separable
loss = rr.quadratic.affine(X, -Y, coef=0.5)
problem = rr.separable_problem.fromatom(penalty, loss)
solver = rr.FISTA(problem)
solver.fit(min_its=200, tol=1.0e-12)
coefs = solver.composite.coefs
# solve using the usual composite
penalty_all = rr.l1norm(20, lagrange=1.2)
problem_all = rr.container(loss, penalty_all)
solver_all = rr.FISTA(problem_all)
solver_all.fit(min_its=100, tol=1.0e-12)
coefs_all = solver_all.composite.coefs
# solve using the selectors
penalty_s = [
rr.linear_atom(p, rr.selector(g, (20, )))
for p, g in zip(penalty.atoms, penalty.groups)
]
problem_s = rr.container(loss, *penalty_s)
solver_s = rr.FISTA(problem_s)
solver_s.fit(min_its=500, tol=1.0e-12)
coefs_s = solver_s.composite.coefs
np.testing.assert_almost_equal(coefs, coefs_all)
np.testing.assert_almost_equal(coefs, coefs_s)
def test_weighted_l1_with_zero():
z = np.random.standard_normal(5)
a=rr.weighted_l1norm([0,1,1,1,1], lagrange=0.5)
b=a.dual[1]
c=rr.l1norm(4, lagrange=0.5)
npt.assert_equal(a.lagrange_prox(z), z-b.bound_prox(z))
npt.assert_equal(a.lagrange_prox(z)[0], z[0])
npt.assert_equal(a.lagrange_prox(z)[1:], c.lagrange_prox(z[1:]))
def test_gengrad():
Z = np.random.standard_normal(100) * 4
p = rr.l1norm(100, lagrange=0.13)
L = 0.14
loss = rr.quadratic_loss.shift(Z, coef=L)
problem = rr.simple_problem(loss, p)
solver = rr.FISTA(problem)
solver.fit(tol=1.0e-10, debug=True)
simple_coef = solver.composite.coefs
prox_coef = p.proximal(rr.identity_quadratic(L, Z, 0, 0))
p2 = rr.l1norm(100, lagrange=0.13)
p2 = copy(p)
p2.quadratic = rr.identity_quadratic(L, Z, 0, 0)
problem = rr.simple_problem.nonsmooth(p2)
solver = rr.FISTA(problem)
solver.fit(tol=1.0e-14, debug=True)
simple_nonsmooth_coef = solver.composite.coefs
p = rr.l1norm(100, lagrange=0.13)
p.quadratic = rr.identity_quadratic(L, Z, 0, 0)
problem = rr.simple_problem.nonsmooth(p)
simple_nonsmooth_gengrad = rr.gengrad(problem, L, tol=1.0e-10)
p = rr.l1norm(100, lagrange=0.13)
problem = rr.separable_problem.singleton(p, loss)
solver = rr.FISTA(problem)
solver.fit(tol=1.0e-10)
separable_coef = solver.composite.coefs
loss2 = rr.quadratic_loss.shift(Z, coef=0.6 * L)
loss2.quadratic = rr.identity_quadratic(0.4 * L, Z, 0, 0)
p.coefs *= 0
problem2 = rr.simple_problem(loss2, p)
loss2_coefs = problem2.solve(coef_stop=True)
solver2 = rr.FISTA(problem2)
solver2.fit(tol=1.0e-10, debug=True, coef_stop=True)
yield all_close, prox_coef, simple_nonsmooth_gengrad, 'prox to nonsmooth gengrad', None
yield all_close, prox_coef, separable_coef, 'prox to separable', None
yield all_close, prox_coef, simple_nonsmooth_coef, 'prox to simple_nonsmooth', None
yield all_close, prox_coef, simple_coef, 'prox to simple', None
yield all_close, prox_coef, loss2_coefs, 'simple where loss has quadratic 1', None
yield all_close, prox_coef, solver2.composite.coefs, 'simple where loss has quadratic 2', None
def test_simple():
Z = np.random.standard_normal(100) * 4
p = rr.l1norm(100, lagrange=0.13)
L = 0.14
loss = rr.quadratic.shift(-Z, coef=L)
problem = rr.simple_problem(loss, p)
solver = rr.FISTA(problem)
solver.fit(tol=1.0e-10, debug=True)
simple_coef = solver.composite.coefs
prox_coef = p.proximal(rr.identity_quadratic(L, Z, 0, 0))
p2 = rr.l1norm(100, lagrange=0.13)
p2 = copy(p)
p2.quadratic = rr.identity_quadratic(L, Z, 0, 0)
problem = rr.simple_problem.nonsmooth(p2)
solver = rr.FISTA(problem)
solver.fit(tol=1.0e-14, debug=True)
simple_nonsmooth_coef = solver.composite.coefs
p = rr.l1norm(100, lagrange=0.13)
p.quadratic = rr.identity_quadratic(L, Z, 0, 0)
problem = rr.simple_problem.nonsmooth(p)
simple_nonsmooth_gengrad = gengrad(problem, L, tol=1.0e-10)
p = rr.l1norm(100, lagrange=0.13)
problem = rr.separable_problem.singleton(p, loss)
solver = rr.FISTA(problem)
solver.fit(tol=1.0e-10)
separable_coef = solver.composite.coefs
loss2 = rr.quadratic.shift(-Z, coef=0.6*L)
loss2.quadratic = rr.identity_quadratic(0.4*L, Z, 0, 0)
p.coefs *= 0
problem2 = rr.simple_problem(loss2, p)
loss2_coefs = problem2.solve(coef_stop=True)
solver2 = rr.FISTA(problem2)
solver2.fit(tol=1.0e-10, debug=True, coef_stop=True)
yield ac, prox_coef, simple_nonsmooth_gengrad, 'prox to nonsmooth gengrad'
yield ac, prox_coef, separable_coef, 'prox to separable'
yield ac, prox_coef, simple_nonsmooth_coef, 'prox to simple_nonsmooth'
yield ac, prox_coef, simple_coef, 'prox to simple'
yield ac, prox_coef, loss2_coefs, 'simple where loss has quadratic 1'
yield ac, prox_coef, solver2.composite.coefs, 'simple where loss has quadratic 2'
def example3(lambda1=10):
#Example using a smooth approximation to the non-negativity constraint
# On large problems this might be faster than using the actual constraint
n = 100
p = 1000
X = np.random.standard_normal(n*p).reshape((n,p))
Y = 10*np.random.standard_normal(n)
loss = rr.l2normsq.affine(X,-Y,coef=1.)
sparsity = rr.l1norm(p, lagrange = lambda1)
nonnegative = rr.nonnegative(p)
smooth_nonnegative = rr.smoothed_atom(nonnegative, epsilon = 1e-4)
problem = rr.container(loss, sparsity, smooth_nonnegative)
solver = rr.FISTA(problem)
solver.fit(debug=True)
solution1 = solver.composite.coefs
loss = rr.l2normsq.affine(X,-Y,coef=1.)
sparsity = rr.l1norm(p, lagrange = lambda1)
nonnegative = rr.nonnegative(p)
problem = rr.container(loss, sparsity, nonnegative)
solver = rr.FISTA(problem)
solver.fit(debug=True)
solution2 = solver.composite.coefs
pl.subplot(1,2,1)
pl.hist(solution1, bins=40)
pl.subplot(1,2,2)
pl.scatter(solution2,solution1)
pl.xlabel("Constraint")
pl.ylabel("Smooth constraint")
def lasso_via_dual_split():
def selector(p, slice):
return np.identity(p)[slice]
penalties = [R.l1norm(selector(500, slice(i*100,(i+1)*100)), lagrange=0.2) for i in range(5)]
x = np.random.standard_normal(500)
loss = R.quadratic.shift(-x, coef=0.5)
lasso = R.container(loss,*penalties)
solver = R.FISTA(lasso)
np.testing.assert_almost_equal(np.maximum(np.fabs(x)-0.2, 0) * np.sign(x), solver.composite.coefs, decimal=3)
def test_multiple_lasso():
"""
Check that the solution of the lasso signal approximator dual problem is soft-thresholding even when specified with multiple seminorms
"""
p = 1000
l1 = 2
sparsity1 = R.l1norm(p, lagrange=l1 * 0.75)
sparsity2 = R.l1norm(p, lagrange=l1 * 0.25)
x = np.random.normal(0, 1, p)
loss = R.quadratic.shift(-x, coef=0.5)
p = R.container(loss, sparsity1, sparsity2)
solver = R.FISTA(p)
vals = solver.fit(tol=1.0e-10)
soln = solver.composite.coefs
st = np.maximum(np.fabs(x) - l1, 0) * np.sign(x)
npt.assert_array_almost_equal(soln, st, 3)
def test_lasso_separable():
"""
This test verifies that the specification of a separable
penalty yields the same results as having two linear_atoms
with selector matrices. The penalty here is a lasso, i.e. l1
penalty.
"""
X = np.random.standard_normal((100,20))
Y = np.random.standard_normal((100,)) + np.dot(X, np.random.standard_normal(20))
penalty1 = rr.l1norm(10, lagrange=1.2)
penalty2 = rr.l1norm(10, lagrange=1.2)
penalty = rr.separable((20,), [penalty1, penalty2], [slice(0,10), slice(10,20)])
# solve using separable
loss = rr.quadratic.affine(X, -Y, coef=0.5)
problem = rr.separable_problem.fromatom(penalty, loss)
solver = rr.FISTA(problem)
solver.fit(min_its=200, tol=1.0e-12)
coefs = solver.composite.coefs
# solve using the usual composite
penalty_all = rr.l1norm(20, lagrange=1.2)
problem_all = rr.container(loss, penalty_all)
solver_all = rr.FISTA(problem_all)
solver_all.fit(min_its=100, tol=1.0e-12)
coefs_all = solver_all.composite.coefs
# solve using the selectors
penalty_s = [rr.linear_atom(p, rr.selector(g, (20,))) for p, g in
zip(penalty.atoms, penalty.groups)]
problem_s = rr.container(loss, *penalty_s)
solver_s = rr.FISTA(problem_s)
solver_s.fit(min_its=500, tol=1.0e-12)
coefs_s = solver_s.composite.coefs
np.testing.assert_almost_equal(coefs, coefs_all)
np.testing.assert_almost_equal(coefs, coefs_s)
def test_multiple_lasso():
"""
Check that the solution of the lasso signal approximator dual problem is soft-thresholding even when specified with multiple seminorms
"""
p = 1000
l1 = 2
sparsity1 = R.l1norm(p, lagrange=l1*0.75)
sparsity2 = R.l1norm(p, lagrange=l1*0.25)
x = np.random.normal(0,1,p)
loss = R.quadratic.shift(-x, coef=0.5)
p = R.container(loss, sparsity1, sparsity2)
solver = R.FISTA(p)
vals = solver.fit(tol=1.0e-10)
soln = solver.composite.coefs
st = np.maximum(np.fabs(x)-l1,0) * np.sign(x)
npt.assert_array_almost_equal(soln, st, 3)
def __init__(self, X, initial=None, lagrange=1, rho=1):
self.X = R.affine_transform(X, None)
self.atom = R.l1norm(X.shape[1], l)
self.rho = rho
self.loss = R.quadratic.affine(X, -np.zeros(X.shape[0]), lagrange=rho/2.)
self.lasso = R.container(self.loss, self.atom)
self.solver = R.FISTA(self.lasso.problem())
if initial is None:
self.beta[:] = np.random.standard_normal(self.atom.primal_shape)
else:
self.beta[:] = initial
def meta_algorithm(XTX, XTXi, dispersion, lam, sampler):
p = XTX.shape[0]
success = np.zeros(p)
loss = rr.quadratic_loss((p, ), Q=XTX)
pen = rr.l1norm(p, lagrange=lam)
scale = 0.5
noisy_S = sampler(scale=scale)
soln = XTXi.dot(noisy_S)
solnZ = soln / (np.sqrt(np.diag(XTXi)) * np.sqrt(dispersion))
return set(np.nonzero(np.fabs(solnZ) > 2.1)[0])
def test_affine_linear_offset_l1norm():
"""
Test linear, affine and offset with the l1norm atom
"""
n = 1000
p = 10
X = np.random.standard_normal((n,p))
Y = 10*np.random.standard_normal(n)
coefs = []
loss = rr.quadratic.affine(X,-Y, coef=0.5)
sparsity = rr.l1norm(p, lagrange=5.)
problem = rr.container(loss, sparsity)
solver = rr.FISTA(problem)
solver.fit(debug=False, tol=1e-10)
coefs.append(1.*solver.composite.coefs)
loss = rr.quadratic.affine(X,-Y, coef=0.5)
sparsity = rr.l1norm.linear(np.eye(p), lagrange=5.)
problem = rr.container(loss, sparsity)
solver = rr.FISTA(problem)
solver.fit(debug=False, tol=1e-10)
coefs.append(1.*solver.composite.coefs)
loss = rr.quadratic.affine(X,-Y, coef=0.5)
sparsity = rr.l1norm.affine(np.eye(p),np.zeros(p), lagrange=5.)
problem = rr.container(loss, sparsity)
solver = rr.FISTA(problem)
solver.fit(debug=False, tol=1e-10)
coefs.append(1.*solver.composite.coefs)
loss = rr.quadratic.affine(X,-Y, coef=0.5)
sparsity = rr.l1norm.linear(np.eye(p), lagrange=5., offset=np.zeros(p))
problem = rr.container(loss, sparsity)
solver = rr.FISTA(problem)
solver.fit(debug=False, tol=1e-10)
coefs.append(1.*solver.composite.coefs)
loss = rr.quadratic.affine(X,-Y, coef=0.5)
sparsity = rr.l1norm.shift(np.zeros(p), lagrange=5.)
problem = rr.container(loss, sparsity)
solver = rr.FISTA(problem)
solver.fit(debug=False, tol=1e-10)
coefs.append(1.*solver.composite.coefs)
for i,j in itertools.combinations(range(len(coefs)), 2):
npt.assert_almost_equal(coefs[i], coefs[j])
def test_affine_linear_offset_l1norm():
"""
Test linear, affine and offset with the l1norm atom
"""
n = 1000
p = 10
X = np.random.standard_normal((n,p))
Y = 10*np.random.standard_normal(n)
coefs = []
loss = rr.quadratic.affine(X,-Y, coef=0.5)
sparsity = rr.l1norm(p, lagrange=5.)
problem = rr.container(loss, sparsity)
solver = rr.FISTA(problem)
solver.fit(debug=False, tol=1e-10)
coefs.append(1.*solver.composite.coefs)
loss = rr.quadratic.affine(X,-Y, coef=0.5)
sparsity = rr.l1norm.linear(np.eye(p), lagrange=5.)
problem = rr.container(loss, sparsity)
solver = rr.FISTA(problem)
solver.fit(debug=False, tol=1e-10)
coefs.append(1.*solver.composite.coefs)
loss = rr.quadratic.affine(X,-Y, coef=0.5)
sparsity = rr.l1norm.affine(np.eye(p),np.zeros(p), lagrange=5.)
problem = rr.container(loss, sparsity)
solver = rr.FISTA(problem)
solver.fit(debug=False, tol=1e-10)
coefs.append(1.*solver.composite.coefs)
loss = rr.quadratic.affine(X,-Y, coef=0.5)
sparsity = rr.l1norm.linear(np.eye(p), lagrange=5., offset=np.zeros(p))
problem = rr.container(loss, sparsity)
solver = rr.FISTA(problem)
solver.fit(debug=False, tol=1e-10)
coefs.append(1.*solver.composite.coefs)
loss = rr.quadratic.affine(X,-Y, coef=0.5)
sparsity = rr.l1norm.shift(np.zeros(p), lagrange=5.)
problem = rr.container(loss, sparsity)
solver = rr.FISTA(problem)
solver.fit(debug=False, tol=1e-10)
coefs.append(1.*solver.composite.coefs)
for i,j in itertools.combinations(range(len(coefs)), 2):
npt.assert_almost_equal(coefs[i], coefs[j])
def algorithm(lam, X, y):
n, p = X.shape
success = np.zeros(p)
loss = rr.quadratic_loss((p,), Q=X.T.dot(X))
pen = rr.l1norm(p, lagrange=lam)
S = -X.T.dot(y)
loss.quadratic = rr.identity_quadratic(0, 0, S, 0)
problem = rr.simple_problem(loss, pen)
soln = problem.solve(max_its=100, tol=1.e-10)
success += soln != 0
return set(np.nonzero(success)[0])
def lasso_example():
l1 = 20.
sparsity = R.l1norm(500, lagrange=l1/2.)
X = np.random.standard_normal((1000,500))
Y = np.random.standard_normal((1000,))
regloss = R.quadratic.affine(X,-Y, coef=0.5)
sparsity2 = R.l1norm(500, lagrange=l1/2.)
p=R.container(regloss, sparsity, sparsity2)
solver=R.FISTA(p)
solver.debug = True
vals = solver.fit(max_its=2000, min_its = 100)
soln = solver.composite.coefs
# solution
pylab.figure(num=1)
pylab.clf()
pylab.plot(soln, c='g')
# objective values
pylab.figure(num=2)
pylab.clf()
pylab.plot(vals)
def __init__(self, X, initial=None, lagrange=1, rho=1):
self.X = R.affine_transform(X, None)
self.atom = R.l1norm(X.shape[1], l)
self.rho = rho
self.loss = R.quadratic.affine(X,
-np.zeros(X.shape[0]),
lagrange=rho / 2.)
self.lasso = R.container(self.loss, self.atom)
self.solver = R.FISTA(self.lasso.problem())
if initial is None:
self.beta[:] = np.random.standard_normal(self.atom.primal_shape)
else:
self.beta[:] = initial
def meta_algorithm(X, XTXi, resid, lam, sampler):
p = XTX.shape[0]
success = np.zeros(p)
loss = rr.quadratic_loss((p, ), Q=XTX)
pen = rr.l1norm(p, lagrange=lam)
scale = 0.
noisy_S = sampler(scale=scale)
loss.quadratic = rr.identity_quadratic(0, 0, -noisy_S, 0)
problem = rr.simple_problem(loss, pen)
soln = problem.solve(max_its=100, tol=1.e-10)
success += soln != 0
return set(np.nonzero(success)[0])
def meta_algorithm(XTX, XTXi, dispersion, lam, sampler):
p = XTX.shape[0]
success = np.zeros(p)
loss = rr.quadratic_loss((p, ), Q=XTX)
pen = rr.l1norm(p, lagrange=lam)
scale = 0.
noisy_S = sampler(scale=scale)
soln = XTXi.dot(noisy_S)
solnZ = soln / (np.sqrt(np.diag(XTXi)) * np.sqrt(dispersion))
pval = ndist.cdf(solnZ)
pval = 2 * np.minimum(pval, 1 - pval)
return set(BHfilter(pval, q=0.2))
def test_multiple_lasso_dual(n=500):
"""
Check that the solution of the lasso signal approximator dual composite is soft-thresholding even when specified with multiple seminorms
"""
l1 = 1
sparsity1 = R.l1norm(n, lagrange=l1*0.75)
sparsity2 = R.l1norm(n, lagrange=l1*0.25)
x = np.random.normal(0,1,n)
loss = R.quadratic.shift(-x, coef=0.5)
p = R.dual_problem.fromprimal(loss, sparsity1, sparsity2)
t1 = time.time()
solver = R.FISTA(p)
solver.debug = True
vals = solver.fit(tol=1.0e-16)
soln = p.primal
t2 = time.time()
print t2-t1
st = np.maximum(np.fabs(x)-l1,0) * np.sign(x)
np.testing.assert_almost_equal(soln,st, decimal=3)
p = R.container(loss, sparsity1, sparsity2)
t1 = time.time()
solver = R.FISTA(p)
solver.debug = True
vals = solver.fit(tol=1.0e-16)
soln = p.primal
t2 = time.time()
print t2-t1
st = np.maximum(np.fabs(x)-l1,0) * np.sign(x)
print soln[range(10)]
print st[range(10)]
np.testing.assert_almost_equal(soln,st, decimal=3)
def test_nesta_nonnegative():
n, p, q = 1000, 20, 5
X = np.random.standard_normal((n, p))
beta = np.zeros(p)
beta[:4] = 3
Y = np.random.standard_normal(n) + np.dot(X, beta)
A = np.random.standard_normal((q,p))
loss = rr.squared_error(X,Y)
penalty = rr.l1norm(p, lagrange=0.2)
constraint = rr.nonnegative.linear(A)
primal, dual = rr.nesta(loss, penalty, constraint)
assert_almost_nonnegative(np.dot(A,primal))
def test_class():
"""
runs several class methods on generic instance
"""
n, p = 100, 20
X = np.random.standard_normal((n, p))
Y = np.random.standard_normal(n)
loss = rr.squared_error(X, Y)
pen = rr.l1norm(p, lagrange=1.0)
problem = rr.simple_problem(loss, pen)
problem.latexify()
for debug, coef_stop, max_its in product([True, False], [True, False], [5, 100]):
rr.gengrad(problem, rr.power_L(X) ** 2, max_its=max_its, debug=debug, coef_stop=coef_stop)
def test_nesta_nonnegative():
n, p, q = 1000, 20, 5
X = np.random.standard_normal((n, p))
beta = np.zeros(p)
beta[:4] = 3
Y = np.random.standard_normal(n) + np.dot(X, beta)
A = np.random.standard_normal((q, p))
loss = rr.squared_error(X, Y)
penalty = rr.l1norm(p, lagrange=0.2)
constraint = rr.nonnegative.linear(A)
primal, dual = rr.nesta(loss, penalty, constraint)
assert_almost_nonnegative(np.dot(A, primal))
def test_admm(n=100, p=10):
X = np.random.standard_normal((n, p))
Y = np.random.standard_normal(n)
loss = rr.squared_error(X, Y)
D = np.identity(p)
pen = rr.l1norm(p, lagrange=1.5)
ADMM = admm_problem(loss, pen, ra.astransform(D), 0.5)
ADMM.solve(niter=1000)
coef1 = ADMM.atom_coefs
problem2 = rr.simple_problem(loss, pen)
coef2 = problem2.solve(tol=1.e-12, min_its=500)
np.testing.assert_allclose(coef1, coef2, rtol=1.e-3, atol=1.e-4)
def test_admm_l1_seminorm():
"""
Test ADMM using the l1norm in lagrange form
"""
p = 1000
Y = 10 * np.random.normal(0,1,p)
loss = R.quadratic.shift(-Y, coef=0.5)
sparsity = R.l1norm(p, lagrange=5.)
prob = R.container(loss, sparsity)
solver = R.admm_problem(prob)
solver.fit(debug=False, tol=1e-12)
solution = solver.beta
npt.assert_array_almost_equal(solution, np.maximum(np.fabs(Y) - sparsity.lagrange,0.)*np.sign(Y), 3)
def test_conjugate_l1norm():
'''
this test verifies that numerically computing the conjugate
is essentially the same as using the smooth_conjugate
of the atom
'''
l1 = rr.l1norm(4, lagrange=0.3)
v1=rr.smooth_conjugate(l1, rr.identity_quadratic(0.3,None,None,0))
v2 = rr.conjugate(l1, rr.identity_quadratic(0.3,None,None,0), tol=1.e-12)
w=np.random.standard_normal(4)
u11, u12 = v1.smooth_objective(w)
u21, u22 = v2.smooth_objective(w)
np.testing.assert_approx_equal(u11, u21)
np.testing.assert_allclose(u12, u22, rtol=1.0e-05)
def test_l1_constraint():
"""
Test using the l1norm in bound form
"""
p = 1000
sparsity = R.l1norm(p, bound=5.)
prob = R.simple_problem.nonsmooth(sparsity)
prob.coefs[:] = np.random.standard_normal(1000)
problem = prob
solver = R.FISTA(problem)
vals = solver.fit(tol=1e-8, max_its=500)
solution = solver.composite.coefs
npt.assert_almost_equal(np.fabs(solution).sum(), sparsity.bound, 3)
