def test_ModelLogReg(self): """...Numerical consistency check of loss and gradient for Logistic Regression """ np.random.seed(12) n_samples, n_features = 5000, 10 w0 = np.random.randn(n_features) c0 = np.random.randn() # First check with intercept X, y = SimuLogReg(w0, c0, n_samples=n_samples, verbose=False, dtype=self.dtype).simulate() X_spars = csr_matrix(X, dtype=self.dtype) model = ModelLogReg(fit_intercept=True).fit(X, y) model_spars = ModelLogReg(fit_intercept=True).fit(X_spars, y) self.run_test_for_glm(model, model_spars) self._test_glm_intercept_vs_hardcoded_intercept(model) # Then check without intercept X, y = SimuLogReg(w0, None, n_samples=n_samples, verbose=False, seed=2038, dtype=self.dtype).simulate() X_spars = csr_matrix(X, dtype=self.dtype) model = ModelLogReg(fit_intercept=False).fit(X, y) model_spars = ModelLogReg(fit_intercept=False).fit(X_spars, y) self.run_test_for_glm(model, model_spars) self._test_glm_intercept_vs_hardcoded_intercept(model) # Test for the Lipschitz constants without intercept self.assertAlmostEqual(model.get_lip_best(), 0.67184209642814952, places=self.decimal_places) self.assertAlmostEqual(model.get_lip_mean(), 2.48961431697108, places=self.decimal_places) self.assertAlmostEqual(model.get_lip_max(), 13.706542412138093, places=self.decimal_places) self.assertAlmostEqual(model_spars.get_lip_mean(), model.get_lip_mean(), places=self.decimal_places) self.assertAlmostEqual(model_spars.get_lip_max(), model.get_lip_max(), places=self.decimal_places) # Test for the Lipschitz constants with intercept model = ModelLogReg(fit_intercept=True).fit(X, y) model_spars = ModelLogReg(fit_intercept=True).fit(X_spars, y) self.assertAlmostEqual(model.get_lip_best(), 0.671892096428, places=self.decimal_places) self.assertAlmostEqual(model.get_lip_mean(), 2.739614316971082, places=self.decimal_places) self.assertAlmostEqual(model.get_lip_max(), 13.956542412138093, places=self.decimal_places) self.assertAlmostEqual(model_spars.get_lip_mean(), model.get_lip_mean(), places=self.decimal_places) self.assertAlmostEqual(model_spars.get_lip_max(), model.get_lip_max(), places=self.decimal_places)
def test_asaga_solver(self): """...Check ASAGA solver for a Logistic Regression with Elastic net penalization """ seed = 1398 np.random.seed(seed) n_samples = 4000 n_features = 30 weights = weights_sparse_gauss(n_features, nnz=3).astype(self.dtype) intercept = 0.2 penalty_strength = 1e-3 sparsity = 1e-4 features = sparse.rand(n_samples, n_features, density=sparsity, format='csr', random_state=8).astype(self.dtype) simulator = SimuLogReg(weights, n_samples=n_samples, features=features, verbose=False, intercept=intercept, dtype=self.dtype) features, labels = simulator.simulate() model = ModelLogReg(fit_intercept=True) model.fit(features, labels) prox = ProxElasticNet(penalty_strength, ratio=0.1, range=(0, n_features)) solver_step = 1. / model.get_lip_max() saga = SAGA(step=solver_step, max_iter=100, tol=1e-10, verbose=False, n_threads=1, record_every=10, seed=seed) saga.set_model(model).set_prox(prox) saga.solve() asaga = SAGA(step=solver_step, max_iter=100, tol=1e-10, verbose=False, n_threads=2, record_every=10, seed=seed) asaga.set_model(model).set_prox(prox) asaga.solve() np.testing.assert_array_almost_equal(saga.solution, asaga.solution, decimal=4) self.assertGreater(np.linalg.norm(saga.solution[:-1]), 0)
n_features = 20000 sparsity = 1e-4 penalty_strength = 1e-5 weights = weights_sparse_gauss(n_features, nnz=1000) intercept = 0.2 features = sparse.rand(n_samples, n_features, density=sparsity, format='csr') simulator = SimuLogReg(weights, n_samples=n_samples, features=features, verbose=False, intercept=intercept) features, labels = simulator.simulate() model = ModelLogReg(fit_intercept=True) model.fit(features, labels) prox = ProxElasticNet(penalty_strength, ratio=0.5, range=(0, n_features)) svrg_step = 1. / model.get_lip_max() test_n_threads = [1, 2, 4] fig, axes = plt.subplots(1, 2, figsize=(8, 4)) for ax, SolverClass in zip(axes, [SVRG, SAGA]): solver_list = [] solver_labels = [] for n_threads in test_n_threads: solver = SolverClass(step=svrg_step, seed=seed, max_iter=50, verbose=False, n_threads=n_threads, tol=0, record_every=3) solver.set_model(model).set_prox(prox) solver.solve()
from tick.solver import SVRG from tick.linear_model import SimuLogReg, ModelLogReg from tick.prox import ProxElasticNet from tick.plot import plot_history n_samples, n_features, = 5000, 50 weights0 = weights_sparse_gauss(n_features, nnz=10) intercept0 = 0.2 X, y = SimuLogReg(weights=weights0, intercept=intercept0, n_samples=n_samples, seed=123, verbose=False).simulate() model = ModelLogReg(fit_intercept=True).fit(X, y) prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features)) x0 = np.zeros(model.n_coeffs) optimal_step = 1 / model.get_lip_max() tested_steps = [optimal_step, 1e-2 * optimal_step, 10 * optimal_step] solvers = [] solver_labels = [] for step in tested_steps: svrg = SVRG(max_iter=30, tol=1e-10, verbose=False) svrg.set_model(model).set_prox(prox) svrg.solve(step=step) svrg_bb = SVRG(max_iter=30, tol=1e-10, verbose=False, step_type='bb') svrg_bb.set_model(model).set_prox(prox) svrg_bb.solve(step=step) solvers += [svrg, svrg_bb]
from tick.prox import ProxElasticNet, ProxL1 from tick.plot import plot_history n_samples, n_features, = 5000, 50 weights0 = weights_sparse_gauss(n_features, nnz=10) intercept0 = 0.2 X, y = SimuLogReg(weights=weights0, intercept=intercept0, n_samples=n_samples, seed=123, verbose=False).simulate() model = ModelLogReg(fit_intercept=True).fit(X, y) prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features)) solver_params = {'max_iter': 100, 'tol': 0., 'verbose': False} x0 = np.zeros(model.n_coeffs) gd = GD(linesearch=False, **solver_params).set_model(model).set_prox(prox) gd.solve(x0, step=1 / model.get_lip_best()) agd = AGD(linesearch=False, **solver_params).set_model(model).set_prox(prox) agd.solve(x0, step=1 / model.get_lip_best()) sgd = SGD(**solver_params).set_model(model).set_prox(prox) sgd.solve(x0, step=500.) svrg = SVRG(**solver_params).set_model(model).set_prox(prox) svrg.solve(x0, step=1 / model.get_lip_max()) plot_history([gd, agd, sgd, svrg], log_scale=True, dist_min=True)