def test_ProxElasticNet(self):
"""...Test of ProxElasticNet
"""
coeffs = self.coeffs.copy()
l_enet = 3e-2
ratio = .3
t = 1.7
prox_enet = ProxElasticNet(l_enet, ratio=ratio)
prox_l1 = ProxL1(ratio * l_enet)
prox_l2 = ProxL2Sq((1 - ratio) * l_enet)
self.assertAlmostEqual(prox_enet.value(coeffs),
prox_l1.value(coeffs) + prox_l2.value(coeffs),
delta=1e-15)
out = coeffs.copy()
prox_l1.call(out, t, out)
prox_l2.call(out, t, out)
assert_almost_equal(prox_enet.call(coeffs, step=t), out, decimal=10)
prox_enet = ProxElasticNet(l_enet, ratio=ratio, positive=True)
prox_l1 = ProxL1(ratio * l_enet, positive=True)
prox_l2 = ProxL2Sq((1 - ratio) * l_enet, positive=True)
self.assertAlmostEqual(prox_enet.value(coeffs),
prox_l1.value(coeffs) + prox_l2.value(coeffs),
delta=1e-15)
out = coeffs.copy()
prox_l1.call(out, t, out)
prox_l2.call(out, t, out)
assert_almost_equal(prox_enet.call(coeffs, step=t), out, decimal=10)
def test_prox_multi(self):
"""...Test of ProxMulti
"""
coeffs = self.coeffs.copy()
double_coeffs = np.concatenate([coeffs, coeffs])
half_size = coeffs.shape[0]
full_size = double_coeffs.shape[0]
l_tv = 0.5
t = 1.7
prox_tv = ProxTV(strength=l_tv)
prox_tv_multi = ProxTV(strength=l_tv, range=(0, half_size))
l_enet = 3e-2
ratio = .3
prox_enet = ProxElasticNet(l_enet, ratio=ratio)
prox_enet_multi = ProxElasticNet(l_enet,
ratio=ratio,
range=(half_size, full_size))
prox_multi = ProxMulti((prox_tv_multi, prox_enet_multi))
# Test that the value of the prox is correct
val_multi = prox_multi.value(double_coeffs)
val_correct = prox_enet.value(coeffs) + prox_tv.value(coeffs)
self.assertAlmostEqual(val_multi, val_correct)
# Test that the prox is correct
out1 = prox_tv.call(coeffs, step=t)
out2 = prox_enet.call(coeffs, step=t)
out_correct = np.concatenate([out1, out2])
out_multi = prox_multi.call(double_coeffs, step=t)
np.testing.assert_almost_equal(out_multi, out_correct)
# An example with overlapping coefficients
start1 = 5
end1 = 13
start2 = 10
end2 = 17
prox_tv = ProxTV(strength=l_tv, range=(start1, end1))
prox_enet = ProxElasticNet(strength=l_enet,
ratio=ratio,
range=(start2, end2))
prox_multi = ProxMulti((prox_tv, prox_enet))
val_correct = prox_tv.value(double_coeffs)
val_correct += prox_enet.value(double_coeffs)
val_multi = prox_multi.value(double_coeffs)
self.assertAlmostEqual(val_multi, val_correct)
out_correct = prox_tv.call(double_coeffs)
out_correct = prox_enet.call(out_correct)
out_multi = prox_multi.call(double_coeffs)
np.testing.assert_almost_equal(out_multi, out_correct)
def compare_solver_sdca(self):
"""...Compare SDCA solution with SVRG solution
"""
np.random.seed(12)
n_samples = Test.n_samples
n_features = Test.n_features
for fit_intercept in [True, False]:
y, X, coeffs0, interc0 = TestSolver.generate_logistic_data(
n_features, n_samples)
model = ModelLogReg(fit_intercept=fit_intercept).fit(X, y)
ratio = 0.5
l_enet = 1e-2
# SDCA "elastic-net" formulation is different from elastic-net
# implementation
l_l2_sdca = ratio * l_enet
l_l1_sdca = (1 - ratio) * l_enet
sdca = SDCA(l_l2sq=l_l2_sdca, max_iter=100, verbose=False, tol=0,
seed=Test.sto_seed).set_model(model)
prox_l1 = ProxL1(l_l1_sdca)
sdca.set_prox(prox_l1)
coeffs_sdca = sdca.solve()
# Compare with SVRG
svrg = SVRG(max_iter=100, verbose=False, tol=0,
seed=Test.sto_seed).set_model(model)
prox_enet = ProxElasticNet(l_enet, ratio)
svrg.set_prox(prox_enet)
coeffs_svrg = svrg.solve(step=0.1)
np.testing.assert_allclose(coeffs_sdca, coeffs_svrg)
def test_dense_and_sparse_match(self):
"""...Test in SVRG that dense and sparse code matches in all possible
settings
"""
variance_reductions = ['last', 'rand']
rand_types = ['perm', 'unif']
seed = 123
tol = 0.
max_iter = 50
n_samples = 500
n_features = 20
# Crazy prox examples
proxs = [
ProxTV(strength=1e-2, range=(5, 13), positive=True),
ProxElasticNet(strength=1e-2, ratio=0.9),
ProxEquality(range=(0, n_features)),
ProxL1(strength=1e-3, range=(5, 17)),
ProxL1w(strength=1e-3, weights=np.arange(5, 17, dtype=np.double),
range=(5, 17)),
]
for intercept in [-1, None]:
X, y = self.simu_linreg_data(interc=intercept,
n_features=n_features,
n_samples=n_samples)
fit_intercept = intercept is not None
model_dense, model_spars = self.get_dense_and_sparse_linreg_model(
X, y, fit_intercept=fit_intercept)
step = 1 / model_spars.get_lip_max()
for variance_reduction, rand_type, prox in product(
variance_reductions, rand_types, proxs):
solver_sparse = SVRG(step=step, tol=tol, max_iter=max_iter,
verbose=False,
variance_reduction=variance_reduction,
rand_type=rand_type, seed=seed) \
.set_model(model_spars) \
.set_prox(prox)
solver_dense = SVRG(step=step, tol=tol, max_iter=max_iter,
verbose=False,
variance_reduction=variance_reduction,
rand_type=rand_type, seed=seed) \
.set_model(model_dense) \
.set_prox(prox)
solver_sparse.solve()
solver_dense.solve()
np.testing.assert_array_almost_equal(solver_sparse.solution,
solver_dense.solution, 7)
def test_solver_sdca(self):
"""...Check SDCA solver for a Logistic regression with Ridge
penalization and L1 penalization
"""
solver = SDCA(l_l2sq=1e-5, max_iter=100, verbose=False, tol=0)
self.check_solver(solver,
fit_intercept=False,
model="logreg",
decimal=1)
# Now a specific test with a real prox for SDCA
np.random.seed(12)
n_samples = Test.n_samples
n_features = Test.n_features
for fit_intercept in [True, False]:
y, X, coeffs0, interc0 = TestSolver.generate_logistic_data(
n_features, n_samples)
model = ModelLogReg(fit_intercept=fit_intercept).fit(X, y)
ratio = 0.5
l_enet = 1e-2
# SDCA "elastic-net" formulation is different from elastic-net
# implementation
l_l2_sdca = ratio * l_enet
l_l1_sdca = (1 - ratio) * l_enet
sdca = SDCA(l_l2sq=l_l2_sdca,
max_iter=100,
verbose=False,
tol=0,
seed=Test.sto_seed).set_model(model)
prox_l1 = ProxL1(l_l1_sdca)
sdca.set_prox(prox_l1)
coeffs_sdca = sdca.solve()
# Compare with SVRG
svrg = SVRG(max_iter=100, verbose=False, tol=0,
seed=Test.sto_seed).set_model(model)
prox_enet = ProxElasticNet(l_enet, ratio)
svrg.set_prox(prox_enet)
coeffs_svrg = svrg.solve(step=0.1)
np.testing.assert_allclose(coeffs_sdca, coeffs_svrg)
def test_solver_gfb(self):
"""...Check GFB's solver for a Logistic Regression with ElasticNet
penalization
Notes
-----
Using GFB solver with l1 and l2 penalizations is obviously a bad
idea as ElasticNet prox is meant to do this, but it allows us to
compare with another algorithm.
"""
n_samples = 200
n_features = 10
y, X, w, c = Test.generate_logistic_data(n_features=n_features,
n_samples=n_samples)
strength = 1e-3
ratio = 0.3
prox_elasticnet = ProxElasticNet(strength, ratio)
prox_l1 = ProxL1(strength * ratio)
prox_l2 = ProxL2Sq(strength * (1 - ratio))
# First we get GFB solution with prox l1 and prox l2
gfb = GFB(tol=1e-13, max_iter=1000, verbose=False, step=1)
Test.prepare_solver(gfb, X, y, prox=None)
gfb.set_prox([prox_l1, prox_l2])
gfb_solution = gfb.solve()
# Then we get AGD solution with prox ElasticNet
agd = AGD(tol=1e-13,
max_iter=1000,
verbose=False,
step=0.5,
linesearch=False)
Test.prepare_solver(agd, X, y, prox=prox_elasticnet)
agd_solution = agd.solve()
# Finally we assert that both algorithms lead to the same solution
np.testing.assert_almost_equal(gfb_solution, agd_solution, decimal=1)
from tick.optim.solver import SVRG
from tick.optim.model import ModelLogReg
from tick.optim.prox import ProxElasticNet
from tick.plot import plot_history
n_samples, n_features, = 5000, 50
weights0 = weights_sparse_gauss(n_features, nnz=10)
intercept0 = 0.2
X, y = SimuLogReg(weights=weights0,
intercept=intercept0,
n_samples=n_samples,
seed=123,
verbose=False).simulate()
model = ModelLogReg(fit_intercept=True).fit(X, y)
prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features))
x0 = np.zeros(model.n_coeffs)
optimal_step = 1 / model.get_lip_max()
tested_steps = [optimal_step, 1e-2 * optimal_step, 10 * optimal_step]
solvers = []
solver_labels = []
for step in tested_steps:
svrg = SVRG(max_iter=30, tol=1e-10, verbose=False)
svrg.set_model(model).set_prox(prox)
svrg.solve(step=step)
svrg_bb = SVRG(max_iter=30, tol=1e-10, verbose=False, step_type='bb')
svrg_bb.set_model(model).set_prox(prox)
import numpy as np
import matplotlib.pyplot as plt
from tick.optim.prox import ProxL1, ProxElasticNet, ProxL2Sq, \
ProxPositive, ProxSlope, ProxTV, ProxZero
x = np.random.randn(50)
a, b = x.min() - 1e-1, x.max() + 1e-1
s = 0.4
proxs = [
ProxZero(),
ProxPositive(),
ProxL2Sq(strength=s),
ProxL1(strength=s),
ProxElasticNet(strength=s, ratio=0.5),
ProxSlope(strength=s),
ProxTV(strength=s)
]
fig, _ = plt.subplots(2, 4, figsize=(16, 8), sharey=True, sharex=True)
fig.axes[0].stem(x)
fig.axes[0].set_title("original vector", fontsize=16)
fig.axes[0].set_xlim((-1, 51))
fig.axes[0].set_ylim((a, b))
for i, prox in enumerate(proxs):
fig.axes[i + 1].stem(prox.call(x))
fig.axes[i + 1].set_title(prox.name, fontsize=16)
fig.axes[i + 1].set_xlim((-1, 51))
fig.axes[i + 1].set_ylim((a, b))
penalty_strength = 1e-6
weights = weights_sparse_gauss(n_features, nnz=10)
intercept = 0.2
features = sparse.rand(n_samples, n_features, density=sparsity, format='csr')
simulator = SimuLogReg(weights,
n_samples=n_samples,
features=features,
verbose=False,
intercept=intercept)
features, labels = simulator.simulate()
model = ModelLogReg(fit_intercept=True)
model.fit(features, labels)
prox = ProxElasticNet(penalty_strength, ratio=0.5, range=(0, n_features))
svrg_step = 1. / model.get_lip_max()
test_n_threads = [1, 2, 4]
svrg_list = []
svrg_labels = []
for n_threads in test_n_threads:
svrg = SVRG(step=svrg_step,
seed=seed,
max_iter=30,
verbose=False,
n_threads=n_threads)
svrg.set_model(model).set_prox(prox)
svrg.solve()