def test_ModelLogReg(self):
"""...Numerical consistency check of loss and gradient for Logistic
Regression
"""
np.random.seed(12)
n_samples, n_features = 5000, 10
w0 = np.random.randn(n_features)
c0 = np.random.randn()
# First check with intercept
X, y = SimuLogReg(w0, c0, n_samples=n_samples, verbose=False,
dtype=self.dtype).simulate()
X_spars = csr_matrix(X, dtype=self.dtype)
model = ModelLogReg(fit_intercept=True).fit(X, y)
model_spars = ModelLogReg(fit_intercept=True).fit(X_spars, y)
self.run_test_for_glm(model, model_spars)
self._test_glm_intercept_vs_hardcoded_intercept(model)
# Then check without intercept
X, y = SimuLogReg(w0, None, n_samples=n_samples, verbose=False,
seed=2038, dtype=self.dtype).simulate()
X_spars = csr_matrix(X, dtype=self.dtype)
model = ModelLogReg(fit_intercept=False).fit(X, y)
model_spars = ModelLogReg(fit_intercept=False).fit(X_spars, y)
self.run_test_for_glm(model, model_spars)
self._test_glm_intercept_vs_hardcoded_intercept(model)
# Test for the Lipschitz constants without intercept
self.assertAlmostEqual(model.get_lip_best(), 0.67184209642814952,
places=self.decimal_places)
self.assertAlmostEqual(model.get_lip_mean(), 2.48961431697108,
places=self.decimal_places)
self.assertAlmostEqual(model.get_lip_max(), 13.706542412138093,
places=self.decimal_places)
self.assertAlmostEqual(model_spars.get_lip_mean(),
model.get_lip_mean(),
places=self.decimal_places)
self.assertAlmostEqual(model_spars.get_lip_max(), model.get_lip_max(),
places=self.decimal_places)
# Test for the Lipschitz constants with intercept
model = ModelLogReg(fit_intercept=True).fit(X, y)
model_spars = ModelLogReg(fit_intercept=True).fit(X_spars, y)
self.assertAlmostEqual(model.get_lip_best(), 0.671892096428,
places=self.decimal_places)
self.assertAlmostEqual(model.get_lip_mean(), 2.739614316971082,
places=self.decimal_places)
self.assertAlmostEqual(model.get_lip_max(), 13.956542412138093,
places=self.decimal_places)
self.assertAlmostEqual(model_spars.get_lip_mean(),
model.get_lip_mean(),
places=self.decimal_places)
self.assertAlmostEqual(model_spars.get_lip_max(), model.get_lip_max(),
places=self.decimal_places)
def test_asaga_solver(self):
"""...Check ASAGA solver for a Logistic Regression with Elastic net
penalization
"""
seed = 1398
np.random.seed(seed)
n_samples = 4000
n_features = 30
weights = weights_sparse_gauss(n_features, nnz=3).astype(self.dtype)
intercept = 0.2
penalty_strength = 1e-3
sparsity = 1e-4
features = sparse.rand(n_samples, n_features, density=sparsity,
format='csr', random_state=8).astype(self.dtype)
simulator = SimuLogReg(weights, n_samples=n_samples, features=features,
verbose=False, intercept=intercept,
dtype=self.dtype)
features, labels = simulator.simulate()
model = ModelLogReg(fit_intercept=True)
model.fit(features, labels)
prox = ProxElasticNet(penalty_strength, ratio=0.1, range=(0,
n_features))
solver_step = 1. / model.get_lip_max()
saga = SAGA(step=solver_step, max_iter=100, tol=1e-10, verbose=False,
n_threads=1, record_every=10, seed=seed)
saga.set_model(model).set_prox(prox)
saga.solve()
asaga = SAGA(step=solver_step, max_iter=100, tol=1e-10, verbose=False,
n_threads=2, record_every=10, seed=seed)
asaga.set_model(model).set_prox(prox)
asaga.solve()
np.testing.assert_array_almost_equal(saga.solution, asaga.solution,
decimal=4)
self.assertGreater(np.linalg.norm(saga.solution[:-1]), 0)
n_features = 20000
sparsity = 1e-4
penalty_strength = 1e-5
weights = weights_sparse_gauss(n_features, nnz=1000)
intercept = 0.2
features = sparse.rand(n_samples, n_features, density=sparsity, format='csr')
simulator = SimuLogReg(weights, n_samples=n_samples, features=features,
verbose=False, intercept=intercept)
features, labels = simulator.simulate()
model = ModelLogReg(fit_intercept=True)
model.fit(features, labels)
prox = ProxElasticNet(penalty_strength, ratio=0.5, range=(0, n_features))
svrg_step = 1. / model.get_lip_max()
test_n_threads = [1, 2, 4]
fig, axes = plt.subplots(1, 2, figsize=(8, 4))
for ax, SolverClass in zip(axes, [SVRG, SAGA]):
solver_list = []
solver_labels = []
for n_threads in test_n_threads:
solver = SolverClass(step=svrg_step, seed=seed, max_iter=50,
verbose=False, n_threads=n_threads, tol=0,
record_every=3)
solver.set_model(model).set_prox(prox)
solver.solve()
from tick.solver import SVRG
from tick.linear_model import SimuLogReg, ModelLogReg
from tick.prox import ProxElasticNet
from tick.plot import plot_history
n_samples, n_features, = 5000, 50
weights0 = weights_sparse_gauss(n_features, nnz=10)
intercept0 = 0.2
X, y = SimuLogReg(weights=weights0, intercept=intercept0,
n_samples=n_samples, seed=123, verbose=False).simulate()
model = ModelLogReg(fit_intercept=True).fit(X, y)
prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features))
x0 = np.zeros(model.n_coeffs)
optimal_step = 1 / model.get_lip_max()
tested_steps = [optimal_step, 1e-2 * optimal_step, 10 * optimal_step]
solvers = []
solver_labels = []
for step in tested_steps:
svrg = SVRG(max_iter=30, tol=1e-10, verbose=False)
svrg.set_model(model).set_prox(prox)
svrg.solve(step=step)
svrg_bb = SVRG(max_iter=30, tol=1e-10, verbose=False, step_type='bb')
svrg_bb.set_model(model).set_prox(prox)
svrg_bb.solve(step=step)
solvers += [svrg, svrg_bb]
from tick.prox import ProxElasticNet, ProxL1
from tick.plot import plot_history
n_samples, n_features, = 5000, 50
weights0 = weights_sparse_gauss(n_features, nnz=10)
intercept0 = 0.2
X, y = SimuLogReg(weights=weights0,
intercept=intercept0,
n_samples=n_samples,
seed=123,
verbose=False).simulate()
model = ModelLogReg(fit_intercept=True).fit(X, y)
prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features))
solver_params = {'max_iter': 100, 'tol': 0., 'verbose': False}
x0 = np.zeros(model.n_coeffs)
gd = GD(linesearch=False, **solver_params).set_model(model).set_prox(prox)
gd.solve(x0, step=1 / model.get_lip_best())
agd = AGD(linesearch=False, **solver_params).set_model(model).set_prox(prox)
agd.solve(x0, step=1 / model.get_lip_best())
sgd = SGD(**solver_params).set_model(model).set_prox(prox)
sgd.solve(x0, step=500.)
svrg = SVRG(**solver_params).set_model(model).set_prox(prox)
svrg.solve(x0, step=1 / model.get_lip_max())
plot_history([gd, agd, sgd, svrg], log_scale=True, dist_min=True)
