def _test_solver_astype_consistency(self, create_solver):
# Launch this test only once
if self.dtype != 'float64':
return
prox = ProxL2Sq(0.1)
use_intercept = True
y_64, X_64, coeffs0_64, interc0 = self.generate_logistic_data(
100, 30, 'float64', use_intercept)
model_64 = ModelLogReg(fit_intercept=use_intercept)
model_64.fit(X_64, y_64)
solver_64 = create_solver()
solver_64.set_model(model_64).set_prox(prox)
solution_64 = solver_64.solve()
solver_32 = solver_64.astype('float32')
solution_32 = solver_32.solve()
self.assertEqual(solution_64.dtype, 'float64')
self.assertEqual(solution_32.dtype, 'float32')
np.testing.assert_array_almost_equal(solution_32,
solution_64,
decimal=3)
def fit(self, X, y):
import statick.linear_model.bin.statick_linear_model as statick_linear_model
TMLR.fit(self, X, y)
func = ModelLogReg.CFUNC_RESOLVER(self, "_dao_")
object.__setattr__(self, "_dao",
getattr(statick_linear_model, func)(X, y))
return self
def compare_solver_sdca(self):
"""...Compare SDCA solution with SVRG solution
"""
np.random.seed(12)
n_samples = Test.n_samples
n_features = Test.n_features
for fit_intercept in [True, False]:
y, X, coeffs0, interc0 = TestSolver.generate_logistic_data(
n_features, n_samples)
model = ModelLogReg(fit_intercept=fit_intercept).fit(X, y)
ratio = 0.5
l_enet = 1e-2
# SDCA "elastic-net" formulation is different from elastic-net
# implementation
l_l2_sdca = ratio * l_enet
l_l1_sdca = (1 - ratio) * l_enet
sdca = SDCA(l_l2sq=l_l2_sdca, max_iter=100, verbose=False, tol=0,
seed=Test.sto_seed).set_model(model)
prox_l1 = ProxL1(l_l1_sdca)
sdca.set_prox(prox_l1)
coeffs_sdca = sdca.solve()
# Compare with SVRG
svrg = SVRG(max_iter=100, verbose=False, tol=0,
seed=Test.sto_seed).set_model(model)
prox_enet = ProxElasticNet(l_enet, ratio)
svrg.set_prox(prox_enet)
coeffs_svrg = svrg.solve(step=0.1)
np.testing.assert_allclose(coeffs_sdca, coeffs_svrg)
def test_step_type_setting(self):
"""...Test that SVRG step_type parameter behaves correctly
"""
svrg = SVRG()
coeffs0 = weights_sparse_gauss(20, nnz=5, dtype=self.dtype)
interc0 = None
X, y = SimuLogReg(coeffs0,
interc0,
n_samples=3000,
verbose=False,
seed=123,
dtype=self.dtype).simulate()
model = ModelLogReg().fit(X, y)
svrg.set_model(model)
self.assertEqual(svrg.step_type, 'fixed')
self.assertEqual(svrg._solver.get_step_type(), SVRG_StepType_Fixed)
svrg = SVRG(step_type='bb')
svrg.set_model(model)
self.assertEqual(svrg.step_type, 'bb')
self.assertEqual(svrg._solver.get_step_type(),
SVRG_StepType_BarzilaiBorwein)
svrg.step_type = 'fixed'
self.assertEqual(svrg.step_type, 'fixed')
self.assertEqual(svrg._solver.get_step_type(), SVRG_StepType_Fixed)
svrg.step_type = 'bb'
self.assertEqual(svrg.step_type, 'bb')
self.assertEqual(svrg._solver.get_step_type(),
SVRG_StepType_BarzilaiBorwein)
def prepare_solver(solver, X, y, fit_intercept=True, model="logistic",
prox="l2"):
if model == "logistic":
model = ModelLogReg(fit_intercept=fit_intercept).fit(X, y)
elif model == "poisson":
model = ModelPoisReg(fit_intercept=fit_intercept).fit(X, y)
solver.set_model(model)
if prox == "l2":
l_l2sq = TestSolver.l_l2sq
prox = ProxL2Sq(l_l2sq, (0, model.n_coeffs))
if prox is not None:
solver.set_prox(prox)
def test_asaga_solver(self):
"""...Check ASAGA solver for a Logistic Regression with Elastic net
penalization
"""
seed = 1398
np.random.seed(seed)
n_samples = 4000
n_features = 30
weights = weights_sparse_gauss(n_features, nnz=3).astype(self.dtype)
intercept = 0.2
penalty_strength = 1e-3
sparsity = 1e-4
features = sparse.rand(n_samples, n_features, density=sparsity,
format='csr', random_state=8).astype(self.dtype)
simulator = SimuLogReg(weights, n_samples=n_samples, features=features,
verbose=False, intercept=intercept,
dtype=self.dtype)
features, labels = simulator.simulate()
model = ModelLogReg(fit_intercept=True)
model.fit(features, labels)
prox = ProxElasticNet(penalty_strength, ratio=0.1, range=(0,
n_features))
solver_step = 1. / model.get_lip_max()
saga = SAGA(step=solver_step, max_iter=100, tol=1e-10, verbose=False,
n_threads=1, record_every=10, seed=seed)
saga.set_model(model).set_prox(prox)
saga.solve()
asaga = SAGA(step=solver_step, max_iter=100, tol=1e-10, verbose=False,
n_threads=2, record_every=10, seed=seed)
asaga.set_model(model).set_prox(prox)
asaga.solve()
np.testing.assert_array_almost_equal(saga.solution, asaga.solution,
decimal=4)
self.assertGreater(np.linalg.norm(saga.solution[:-1]), 0)
def test_variance_reduction_setting(self):
"""...SolverTest SAGA variance_reduction parameter is correctly set"""
svrg = SAGA()
coeffs0 = weights_sparse_gauss(20, nnz=5, dtype=self.dtype)
interc0 = None
X, y = SimuLogReg(coeffs0,
interc0,
n_samples=3000,
verbose=False,
seed=123,
dtype=self.dtype).simulate()
model = ModelLogReg().fit(X, y)
svrg.set_model(model)
svrg.astype(self.dtype)
self.assertEqual(svrg.variance_reduction, 'last')
self.assertEqual(svrg._solver.get_variance_reduction(),
SAGA_VarianceReductionMethod_Last)
svrg = SAGA(variance_reduction='rand')
svrg.set_model(model)
svrg.astype(self.dtype)
self.assertEqual(svrg.variance_reduction, 'rand')
self.assertEqual(svrg._solver.get_variance_reduction(),
SAGA_VarianceReductionMethod_Random)
svrg.variance_reduction = 'avg'
self.assertEqual(svrg.variance_reduction, 'avg')
self.assertEqual(svrg._solver.get_variance_reduction(),
SAGA_VarianceReductionMethod_Average)
svrg.variance_reduction = 'rand'
self.assertEqual(svrg.variance_reduction, 'rand')
self.assertEqual(svrg._solver.get_variance_reduction(),
SAGA_VarianceReductionMethod_Random)
svrg.variance_reduction = 'last'
self.assertEqual(svrg.variance_reduction, 'last')
self.assertEqual(svrg._solver.get_variance_reduction(),
SAGA_VarianceReductionMethod_Last)
with self.assertRaises(ValueError):
svrg.variance_reduction = 'wrong_name'
def test_solver_bfgs(self):
"""...Check BFGS solver for Logistic Regression with Ridge
penalization
"""
# It is the reference solver used in other unittests so we check that
# it's actually close to the true parameter of the simulated dataset
np.random.seed(12)
n_samples = 3000
n_features = 10
coeffs0 = weights_sparse_gauss(n_features, nnz=5)
interc0 = 2.
X, y = SimuLogReg(coeffs0, interc0, n_samples=n_samples,
verbose=False).simulate()
model = ModelLogReg(fit_intercept=True).fit(X, y)
prox = ProxL2Sq(strength=1e-6)
solver = BFGS(max_iter=100, print_every=1, verbose=False,
tol=1e-6).set_model(model).set_prox(prox)
coeffs = solver.solve()
err = Test.evaluate_model(coeffs, coeffs0, interc0)
self.assertAlmostEqual(err, 0., delta=5e-1)
def create_model(model_type, n_samples, n_features, with_intercept=True):
weights = np.random.randn(n_features)
intercept = None
if with_intercept:
intercept = np.random.normal()
if model_type == 'Poisson':
# we need to rescale features to avoid overflows
weights /= n_features
if intercept is not None:
intercept /= n_features
if model_type == 'Linear':
simulator = SimuLinReg(weights,
intercept=intercept,
n_samples=n_samples,
verbose=False)
elif model_type == 'Logistic':
simulator = SimuLogReg(weights,
intercept=intercept,
n_samples=n_samples,
verbose=False)
elif model_type == 'Poisson':
simulator = SimuPoisReg(weights,
intercept=intercept,
n_samples=n_samples,
verbose=False)
labels, features = simulator.simulate()
if model_type == 'Linear':
model = ModelLinReg(fit_intercept=with_intercept)
elif model_type == 'Logistic':
model = ModelLogReg(fit_intercept=with_intercept)
elif model_type == 'Poisson':
model = ModelPoisReg(fit_intercept=with_intercept)
model.fit(labels, features)
return model
def test_ModelLogReg(self):
"""...Numerical consistency check of loss and gradient for Logistic
Regression
"""
np.random.seed(12)
n_samples, n_features = 5000, 10
w0 = np.random.randn(n_features)
c0 = np.random.randn()
# First check with intercept
X, y = SimuLogReg(w0, c0, n_samples=n_samples,
verbose=False).simulate()
X_spars = csr_matrix(X)
model = ModelLogReg(fit_intercept=True).fit(X, y)
model_spars = ModelLogReg(fit_intercept=True).fit(X_spars, y)
self.run_test_for_glm(model, model_spars, 1e-5, 1e-4)
self._test_glm_intercept_vs_hardcoded_intercept(model)
# Then check without intercept
X, y = SimuLogReg(w0,
None,
n_samples=n_samples,
verbose=False,
seed=2038).simulate()
X_spars = csr_matrix(X)
model = ModelLogReg(fit_intercept=False).fit(X, y)
model_spars = ModelLogReg(fit_intercept=False).fit(X_spars, y)
self.run_test_for_glm(model, model_spars, 1e-5, 1e-4)
self._test_glm_intercept_vs_hardcoded_intercept(model)
# Test for the Lipschitz constants without intercept
self.assertAlmostEqual(model.get_lip_best(), 0.67184209642814952)
self.assertAlmostEqual(model.get_lip_mean(), 2.48961431697108)
self.assertAlmostEqual(model.get_lip_max(), 13.706542412138093)
self.assertAlmostEqual(model_spars.get_lip_mean(),
model.get_lip_mean())
self.assertAlmostEqual(model_spars.get_lip_max(), model.get_lip_max())
# Test for the Lipschitz constants with intercept
model = ModelLogReg(fit_intercept=True).fit(X, y)
model_spars = ModelLogReg(fit_intercept=True).fit(X_spars, y)
self.assertAlmostEqual(model.get_lip_best(), 0.671892096428)
self.assertAlmostEqual(model.get_lip_mean(), 2.739614316971082)
self.assertAlmostEqual(model.get_lip_max(), 13.956542412138093)
self.assertAlmostEqual(model_spars.get_lip_mean(),
model.get_lip_mean())
self.assertAlmostEqual(model_spars.get_lip_max(), model.get_lip_max())
np.random.seed(seed)
n_samples = 40000
n_features = 20000
sparsity = 1e-4
penalty_strength = 1e-5
weights = weights_sparse_gauss(n_features, nnz=1000)
intercept = 0.2
features = sparse.rand(n_samples, n_features, density=sparsity, format='csr')
simulator = SimuLogReg(weights, n_samples=n_samples, features=features,
verbose=False, intercept=intercept)
features, labels = simulator.simulate()
model = ModelLogReg(fit_intercept=True)
model.fit(features, labels)
prox = ProxElasticNet(penalty_strength, ratio=0.5, range=(0, n_features))
svrg_step = 1. / model.get_lip_max()
test_n_threads = [1, 2, 4]
fig, axes = plt.subplots(1, 2, figsize=(8, 4))
for ax, SolverClass in zip(axes, [SVRG, SAGA]):
solver_list = []
solver_labels = []
for n_threads in test_n_threads:
solver = SolverClass(step=svrg_step, seed=seed, max_iter=50,
verbose=False, n_threads=n_threads, tol=0,
import matplotlib.pyplot as plt
from cycler import cycler
from tick.simulation import weights_sparse_gauss
from tick.solver import SVRG
from tick.linear_model import SimuLogReg, ModelLogReg
from tick.prox import ProxElasticNet
from tick.plot import plot_history
n_samples, n_features, = 5000, 50
weights0 = weights_sparse_gauss(n_features, nnz=10)
intercept0 = 0.2
X, y = SimuLogReg(weights=weights0, intercept=intercept0,
n_samples=n_samples, seed=123, verbose=False).simulate()
model = ModelLogReg(fit_intercept=True).fit(X, y)
prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features))
x0 = np.zeros(model.n_coeffs)
optimal_step = 1 / model.get_lip_max()
tested_steps = [optimal_step, 1e-2 * optimal_step, 10 * optimal_step]
solvers = []
solver_labels = []
for step in tested_steps:
svrg = SVRG(max_iter=30, tol=1e-10, verbose=False)
svrg.set_model(model).set_prox(prox)
svrg.solve(step=step)
svrg_bb = SVRG(max_iter=30, tol=1e-10, verbose=False, step_type='bb')
#!/usr/bin/python3
# expect tick first on PYTHONPATH
from tick.array.build.array import tick_double_sparse2d_from_file, tick_double_array_from_file
from tick.prox import ProxL2Sq; from tick.solver import SAGA; from tick.linear_model import ModelLogReg
X = tick_double_sparse2d_from_file("url.features.cereal")
n_samples = X.shape[0]; n_features = X.shape[1]
y = tick_double_array_from_file ("url.labels.cereal")
model = ModelLogReg(fit_intercept=False).fit(X, y)
prox = ProxL2Sq((1. / n_samples) + 1e-10, range=(0, n_features))
asaga = SAGA(step=0.00257480411965, max_iter=200, tol=1e-10, verbose=False,
n_threads=8, log_every_n_epochs=10)
asaga.set_model(model).set_prox(prox)
asaga.solve()
asaga.print_history()
def test_variance_reduction_setting(self):
"""...Test that SVRG variance_reduction parameter behaves correctly
"""
svrg = SVRG()
coeffs0 = weights_sparse_gauss(20, nnz=5, dtype=self.dtype)
interc0 = None
X, y = SimuLogReg(coeffs0,
interc0,
n_samples=3000,
verbose=False,
seed=123,
dtype=self.dtype).simulate()
model = ModelLogReg().fit(X, y)
svrg.set_model(model)
self.assertEqual(svrg.variance_reduction, 'last')
self.assertEqual(svrg._solver.get_variance_reduction(),
SVRG_VarianceReductionMethod_Last)
svrg = SVRG(variance_reduction='rand')
svrg.set_model(model)
self.assertEqual(svrg.variance_reduction, 'rand')
self.assertEqual(svrg._solver.get_variance_reduction(),
SVRG_VarianceReductionMethod_Random)
svrg.variance_reduction = 'avg'
self.assertEqual(svrg.variance_reduction, 'avg')
self.assertEqual(svrg._solver.get_variance_reduction(),
SVRG_VarianceReductionMethod_Average)
svrg.variance_reduction = 'rand'
self.assertEqual(svrg.variance_reduction, 'rand')
self.assertEqual(svrg._solver.get_variance_reduction(),
SVRG_VarianceReductionMethod_Random)
svrg.variance_reduction = 'last'
self.assertEqual(svrg.variance_reduction, 'last')
self.assertEqual(svrg._solver.get_variance_reduction(),
SVRG_VarianceReductionMethod_Last)
msg = '^variance_reduction should be one of "avg, last, rand", ' \
'got "stuff"$'
with self.assertRaisesRegex(ValueError, msg):
svrg = SVRG(variance_reduction='stuff')
svrg.set_model(model)
with self.assertRaisesRegex(ValueError, msg):
svrg.variance_reduction = 'stuff'
X, y = self.simu_linreg_data(dtype=self.dtype)
model_dense, model_spars = self.get_dense_and_sparse_linreg_model(
X, y, dtype=self.dtype)
try:
svrg.set_model(model_dense)
svrg.variance_reduction = 'avg'
svrg.variance_reduction = 'last'
svrg.variance_reduction = 'rand'
svrg.set_model(model_spars)
svrg.variance_reduction = 'last'
svrg.variance_reduction = 'rand'
except Exception:
self.fail('Setting variance_reduction in these cases should have '
'been ok')
msg = "'avg' variance reduction cannot be used with sparse datasets"
with catch_warnings(record=True) as w:
simplefilter('always')
svrg.set_model(model_spars)
svrg.variance_reduction = 'avg'
self.assertEqual(len(w), 1)
self.assertTrue(issubclass(w[0].category, UserWarning))
self.assertEqual(str(w[0].message), msg)
from tick.linear_model import ModelLogReg, SimuLogReg
from tick.simulation import weights_sparse_gauss
from tick.solver import GD, AGD, SGD, SVRG, SDCA
from tick.prox import ProxElasticNet, ProxL1
from tick.plot import plot_history
n_samples, n_features, = 5000, 50
weights0 = weights_sparse_gauss(n_features, nnz=10)
intercept0 = 0.2
X, y = SimuLogReg(weights=weights0,
intercept=intercept0,
n_samples=n_samples,
seed=123,
verbose=False).simulate()
model = ModelLogReg(fit_intercept=True).fit(X, y)
prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features))
solver_params = {'max_iter': 100, 'tol': 0., 'verbose': False}
x0 = np.zeros(model.n_coeffs)
gd = GD(linesearch=False, **solver_params).set_model(model).set_prox(prox)
gd.solve(x0, step=1 / model.get_lip_best())
agd = AGD(linesearch=False, **solver_params).set_model(model).set_prox(prox)
agd.solve(x0, step=1 / model.get_lip_best())
sgd = SGD(**solver_params).set_model(model).set_prox(prox)
sgd.solve(x0, step=500.)
svrg = SVRG(**solver_params).set_model(model).set_prox(prox)
def __init__(self):
TMLR.__init__(self)
self._model = None
object.__setattr__(self, "_MANGLING", "log_reg")
object.__setattr__(self, "_dao", None)
def check_solver(self,
solver,
fit_intercept=True,
model='logreg',
decimal=1):
"""Check solver instance finds same parameters as scipy BFGS
Parameters
----------
solver : `Solver`
Instance of a solver to be tested
fit_intercept : `bool`, default=True
Model uses intercept is `True`
model : 'linreg' | 'logreg' | 'poisreg', default='logreg'
Name of the model used to test the solver
decimal : `int`, default=1
Number of decimals required for the test
"""
# Set seed for data simulation
np.random.seed(12)
n_samples = TestSolver.n_samples
n_features = TestSolver.n_features
coeffs0 = weights_sparse_gauss(n_features, nnz=5)
if fit_intercept:
interc0 = 2.
else:
interc0 = None
if model == 'linreg':
X, y = SimuLinReg(coeffs0,
interc0,
n_samples=n_samples,
verbose=False,
seed=123).simulate()
model = ModelLinReg(fit_intercept=fit_intercept).fit(X, y)
elif model == 'logreg':
X, y = SimuLogReg(coeffs0,
interc0,
n_samples=n_samples,
verbose=False,
seed=123).simulate()
model = ModelLogReg(fit_intercept=fit_intercept).fit(X, y)
elif model == 'poisreg':
X, y = SimuPoisReg(coeffs0,
interc0,
n_samples=n_samples,
verbose=False,
seed=123).simulate()
# Rescale features to avoid overflows in Poisson simulations
X /= np.linalg.norm(X, axis=1).reshape(n_samples, 1)
model = ModelPoisReg(fit_intercept=fit_intercept).fit(X, y)
else:
raise ValueError("``model`` must be either 'linreg', 'logreg' or"
" 'poisreg'")
solver.set_model(model)
strength = 1e-2
prox = ProxL2Sq(strength, (0, model.n_features))
if type(solver) is not SDCA:
solver.set_prox(prox)
else:
solver.set_prox(ProxZero())
solver.l_l2sq = strength
coeffs_solver = solver.solve()
# Compare with BFGS
bfgs = BFGS(max_iter=100,
verbose=False).set_model(model).set_prox(prox)
coeffs_bfgs = bfgs.solve()
np.testing.assert_almost_equal(coeffs_solver,
coeffs_bfgs,
decimal=decimal)
# We ensure that reached coeffs are not equal to zero
self.assertGreater(norm(coeffs_solver), 0)
self.assertAlmostEqual(solver.objective(coeffs_bfgs),
solver.objective(coeffs_solver),
delta=1e-2)
import numpy as np
import matplotlib.pyplot as plt
from tick.linear_model import ModelLogReg, SimuLogReg
from tick.simulation import weights_sparse_gauss
n_samples, n_features = 2000, 50
weights0 = weights_sparse_gauss(n_weights=n_features, nnz=10)
intercept0 = 1.
X, y = SimuLogReg(weights0,
intercept=intercept0,
seed=123,
n_samples=n_samples,
verbose=False).simulate()
model = ModelLogReg(fit_intercept=True).fit(X, y)
coeffs0 = np.concatenate([weights0, [intercept0]])
_, ax = plt.subplots(1, 2, sharey=True, figsize=(9, 3))
ax[0].stem(model.grad(coeffs0))
ax[0].set_title(r"$\nabla f(\mathrm{coeffs0})$", fontsize=16)
ax[1].stem(model.grad(np.ones(model.n_coeffs)))
ax[1].set_title(r"$\nabla f(\mathrm{coeffs1})$", fontsize=16)
plt.show()