def test_ProxElasticNet(self): """...Test of ProxElasticNet """ coeffs = self.coeffs.copy() l_enet = 3e-2 ratio = .3 t = 1.7 prox_enet = ProxElasticNet(l_enet, ratio=ratio) prox_l1 = ProxL1(ratio * l_enet) prox_l2 = ProxL2Sq((1 - ratio) * l_enet) self.assertAlmostEqual( prox_enet.value(coeffs), prox_l1.value(coeffs) + prox_l2.value(coeffs), delta=1e-15) out = coeffs.copy() prox_l1.call(out, t, out) prox_l2.call(out, t, out) assert_almost_equal(prox_enet.call(coeffs, step=t), out, decimal=10) prox_enet = ProxElasticNet(l_enet, ratio=ratio, positive=True) prox_l1 = ProxL1(ratio * l_enet, positive=True) prox_l2 = ProxL2Sq((1 - ratio) * l_enet, positive=True) self.assertAlmostEqual( prox_enet.value(coeffs), prox_l1.value(coeffs) + prox_l2.value(coeffs), delta=1e-15) out = coeffs.copy() prox_l1.call(out, t, out) prox_l2.call(out, t, out) assert_almost_equal(prox_enet.call(coeffs, step=t), out, decimal=10)
def test_ProxL2Sq(self): """...Test of ProxL2Sq """ coeffs = self.coeffs.copy().astype(self.dtype) l_l2sq = 3e-2 t = 1.7 prox = ProxL2Sq(l_l2sq).astype(self.dtype) out = coeffs.copy() out *= 1. / (1. + t * l_l2sq) self.assertAlmostEqual(prox.value(coeffs), 0.5 * l_l2sq * norm(coeffs)**2., delta=self.delta) assert_almost_equal(prox.call(coeffs, step=t), out, decimal=self.decimal_places) prox = ProxL2Sq(l_l2sq, (3, 8)).astype(self.dtype) out = coeffs.copy() out[3:8] *= 1. / (1. + t * l_l2sq) self.assertAlmostEqual(prox.value(coeffs), 0.5 * l_l2sq * norm(coeffs[3:8])**2., delta=self.delta) assert_almost_equal(prox.call(coeffs, step=t), out, decimal=self.decimal_places) prox = ProxL2Sq(l_l2sq, (3, 8), positive=True).astype(self.dtype) out = coeffs.copy() out[3:8] *= 1. / (1. + t * l_l2sq) idx = out[3:8] < 0 out[3:8][idx] = 0 self.assertAlmostEqual(prox.value(coeffs), 0.5 * l_l2sq * norm(coeffs[3:8])**2., delta=self.delta) assert_almost_equal(prox.call(coeffs, step=t), out, decimal=self.decimal_places) prox = ProxL2Sq(l_l2sq, (3, 8)).astype(self.dtype) out = coeffs.copy() t = np.linspace(1, 10, 5).astype(self.dtype) out[3:8] *= 1. / (1. + t * l_l2sq) self.assertAlmostEqual(prox.value(coeffs), 0.5 * l_l2sq * norm(coeffs[3:8])**2., delta=self.delta) assert_almost_equal(prox.call(coeffs, t), out, decimal=self.decimal_places)
def _test_solver_astype_consistency(self, create_solver): # Launch this test only once if self.dtype != 'float64': return prox = ProxL2Sq(0.1) use_intercept = True y_64, X_64, coeffs0_64, interc0 = self.generate_logistic_data( 100, 30, 'float64', use_intercept) model_64 = ModelLogReg(fit_intercept=use_intercept) model_64.fit(X_64, y_64) solver_64 = create_solver() solver_64.set_model(model_64).set_prox(prox) solution_64 = solver_64.solve() solver_32 = solver_64.astype('float32') solution_32 = solver_32.solve() self.assertEqual(solution_64.dtype, 'float64') self.assertEqual(solution_32.dtype, 'float32') np.testing.assert_array_almost_equal(solution_32, solution_64, decimal=3)
def test_sdca_identity_poisreg(self): """...Test SDCA on specific case of Poisson regression with indentity link """ l_l2sq = 1e-3 n_samples = 10000 n_features = 3 np.random.seed(123) weight0 = np.random.rand(n_features) features = np.random.rand(n_samples, n_features) for intercept in [None, 0.45]: if intercept is None: fit_intercept = False else: fit_intercept = True simu = SimuPoisReg(weight0, intercept=intercept, features=features, n_samples=n_samples, link='identity', verbose=False) features, labels = simu.simulate() model = ModelPoisReg(fit_intercept=fit_intercept, link='identity') model.fit(features, labels) sdca = SDCA(l_l2sq=l_l2sq, max_iter=100, verbose=False, tol=1e-14, seed=Test.sto_seed) sdca.set_model(model).set_prox(ProxZero()) start_dual = np.sqrt(sdca._rand_max * l_l2sq) start_dual = start_dual * np.ones(sdca._rand_max) sdca.solve(start_dual) # Check that duality gap is 0 self.assertAlmostEqual( sdca.objective(sdca.solution), sdca.dual_objective(sdca.dual_solution)) # Check that original vector is approximatively retrieved if fit_intercept: original_coeffs = np.hstack((weight0, intercept)) else: original_coeffs = weight0 np.testing.assert_array_almost_equal(original_coeffs, sdca.solution, decimal=1) # Ensure that we solve the same problem as other solvers svrg = SVRG(max_iter=100, verbose=False, tol=1e-14, seed=Test.sto_seed) svrg.set_model(model).set_prox(ProxL2Sq(l_l2sq)) svrg.solve(0.5 * np.ones(model.n_coeffs), step=1e-2) np.testing.assert_array_almost_equal(svrg.solution, sdca.solution, decimal=4)
def run_solvers(model, l_l2sq): try: svrg_step = 1. / model.get_lip_max() except AttributeError: svrg_step = 1e-3 try: gd_step = 1. / model.get_lip_best() except AttributeError: gd_step = 1e-1 bfgs = BFGS(verbose=False, tol=1e-13) bfgs.set_model(model).set_prox(ProxL2Sq(l_l2sq)) bfgs.solve() bfgs.history.set_minimizer(bfgs.solution) bfgs.history.set_minimum(bfgs.objective(bfgs.solution)) bfgs.solve() svrg = SVRG(step=svrg_step, verbose=False, tol=1e-10, seed=seed) svrg.set_model(model).set_prox(ProxL2Sq(l_l2sq)) svrg.history.set_minimizer(bfgs.solution) svrg.history.set_minimum(bfgs.objective(bfgs.solution)) svrg.solve() sdca = SDCA(l_l2sq, verbose=False, seed=seed, tol=1e-10) sdca.set_model(model).set_prox(ProxZero()) sdca.history.set_minimizer(bfgs.solution) sdca.history.set_minimum(bfgs.objective(bfgs.solution)) sdca.solve() gd = GD(verbose=False, tol=1e-10, step=gd_step, linesearch=False) gd.set_model(model).set_prox(ProxL2Sq(l_l2sq)) gd.history.set_minimizer(bfgs.solution) gd.history.set_minimum(bfgs.objective(bfgs.solution)) gd.solve() agd = AGD(verbose=False, tol=1e-10, step=gd_step, linesearch=False) agd.set_model(model).set_prox(ProxL2Sq(l_l2sq)) agd.history.set_minimizer(bfgs.solution) agd.history.set_minimum(bfgs.objective(bfgs.solution)) agd.solve() return bfgs, svrg, sdca, gd, agd
def prepare_solver(solver, X, y, fit_intercept=True, model="logistic", prox="l2"): if model == "logistic": model = ModelLogReg(fit_intercept=fit_intercept).fit(X, y) elif model == "poisson": model = ModelPoisReg(fit_intercept=fit_intercept).fit(X, y) solver.set_model(model) if prox == "l2": l_l2sq = TestSolver.l_l2sq prox = ProxL2Sq(l_l2sq, (0, model.n_coeffs)) if prox is not None: solver.set_prox(prox)
def test_solver_bfgs(self): """...Check BFGS solver for Logistic Regression with Ridge penalization """ # It is the reference solver used in other unittests so we check that # it's actually close to the true parameter of the simulated dataset np.random.seed(12) n_samples = 3000 n_features = 10 coeffs0 = weights_sparse_gauss(n_features, nnz=5) interc0 = 2. X, y = SimuLogReg(coeffs0, interc0, n_samples=n_samples, verbose=False).simulate() model = ModelLogReg(fit_intercept=True).fit(X, y) prox = ProxL2Sq(strength=1e-6) solver = BFGS(max_iter=100, print_every=1, verbose=False, tol=1e-6).set_model(model).set_prox(prox) coeffs = solver.solve() err = Test.evaluate_model(coeffs, coeffs0, interc0) self.assertAlmostEqual(err, 0., delta=5e-1)
def test_set_model_and_set_prox(self): np.random.seed(12) n_samples = TestSolver.n_samples n_features = TestSolver.n_features weights0 = weights_sparse_gauss(n_features, nnz=5) interc0 = 2. model = ModelLinReg() msg = '^Passed object ModelLinReg has not been fitted. You must call' \ ' ``fit`` on it before passing it to ``set_model``$' with self.assertRaisesRegex(ValueError, msg): for solver_class in self.solvers: if solver_class is SDCA: solver = solver_class(l_l2sq=1e-1) else: solver = solver_class() solver.set_model(model) X, y = SimuLinReg(weights0, interc0, n_samples=n_samples, verbose=False, seed=123, dtype=self.dtype).simulate() prox = ProxL2Sq(strength=1e-1) msg = '^Passed object of class ProxL2Sq is not a Model class$' with self.assertRaisesRegex(ValueError, msg): for solver_class in self.solvers: if solver_class is SDCA: solver = solver_class(l_l2sq=1e-1) else: solver = solver_class() solver.set_model(prox) model.fit(X, y) msg = '^Passed object of class ModelLinReg is not a Prox class$' with self.assertRaisesRegex(ValueError, msg): for solver_class in self.solvers: if solver_class is SDCA: solver = solver_class(l_l2sq=1e-1) else: solver = solver_class() solver.set_model(model).set_prox(model)
def test_solver_scpg(self): """...Check Self-concordant proximal gradient solver for a Hawkes model with ridge penalization """ beta = 3 betas = beta * np.ones((2, 2)) alphas = np.zeros((2, 2)) alphas[0, 0] = 1 alphas[0, 1] = 2 alphas[1, 1] = 3 mus = np.arange(1, 3) / 3 hawkes = SimuHawkesExpKernels(adjacency=alphas, decays=betas, baseline=mus, seed=1231, end_time=20000, verbose=False) hawkes.adjust_spectral_radius(0.8) alphas = hawkes.adjacency hawkes.simulate() timestamps = hawkes.timestamps model = ModelHawkesExpKernLogLik(beta).fit(timestamps) prox = ProxL2Sq(1e-7, positive=True) pg = SCPG(max_iter=2000, tol=1e-10, verbose=False, step=1e-5).set_model(model).set_prox(prox) pg.solve(np.ones(model.n_coeffs)) original_coeffs = np.hstack((mus, alphas.reshape(4))) np.testing.assert_array_almost_equal(pg.solution, original_coeffs, decimal=2)
def test_solver_gfb(self): """...Check GFB's solver for a Logistic Regression with ElasticNet penalization Notes ----- Using GFB solver with l1 and l2 penalizations is obviously a bad idea as ElasticNet prox is meant to do this, but it allows us to compare with another algorithm. """ n_samples = 200 n_features = 10 y, X, w, c = TestSolver.generate_logistic_data(n_features=n_features, n_samples=n_samples, dtype=self.dtype) strength = 1e-3 ratio = 0.3 prox_elasticnet = ProxElasticNet(strength, ratio).astype(self.dtype) prox_l1 = ProxL1(strength * ratio).astype(self.dtype) prox_l2 = ProxL2Sq(strength * (1 - ratio)).astype(self.dtype) # First we get GFB solution with prox l1 and prox l2 gfb = GFB(tol=1e-13, max_iter=1000, verbose=False, step=1) TestSolver.prepare_solver(gfb, X, y, prox=None) gfb.set_prox([prox_l1, prox_l2]) gfb_solution = gfb.solve() # Then we get AGD solution with prox ElasticNet agd = AGD(tol=1e-13, max_iter=1000, verbose=False, step=0.5, linesearch=False) TestSolver.prepare_solver(agd, X, y, prox=prox_elasticnet) agd_solution = agd.solve() # Finally we assert that both algorithms lead to the same solution np.testing.assert_almost_equal(gfb_solution, agd_solution, decimal=1)
def __init__(self, decay, n_threads=1, approx_type='smooth', decay_neg=100.0, gamma=1000.0, epsilon=1e-3, hawkes_penalty='l1', hawkes_base_C=1e3, hawkes_adj_C=1e3, noise_penalty='l2', noise_C=1e4, solver='sgd', n_chunks=10, max_iter=1000, tol=1e-5, step_z=1e-2, step_theta=1e-2, verbose=True, print_every=100, record_every=100): self.decay = decay if solver not in self._available_solvers: raise ValueError("``solver`` must be one of [%s], got %s" % (', '.join(self._available_solvers), solver)) self.solver = solver self.n_chunks = n_chunks self.max_iter = max_iter self.tol = tol self._n_iter_done = 0 if callable(step_z): self.step_z = step_z else: self.step_z = lambda t: step_z if callable(step_theta): self.step_theta = step_theta else: self.step_theta = lambda t: step_theta assert noise_penalty == 'l2' assert hawkes_penalty == 'l1' self.noise_C = noise_C self.hawkes_base_C = hawkes_base_C self.hawkes_adj_C = hawkes_adj_C self.prox_noise = ProxL2Sq(strength=1 / self.noise_C) self.prox_base = ProxL2Sq(strength=1 / self.hawkes_base_C) self.prox_adj = ProxL1(strength=1 / self.hawkes_adj_C) self.model_obj = ModelHawkesExpKernCondLogLikSyncNoise( decay=decay, n_threads=n_threads, approx_type=approx_type, decay_neg=decay_neg, gamma=gamma, epsilon=epsilon) self._fitted = False self._verbose = verbose self._print_every = print_every self._record_every = record_every self._init_monitor()
def check_solver(self, solver, fit_intercept=True, model='logreg', decimal=1): """Check solver instance finds same parameters as scipy BFGS Parameters ---------- solver : `Solver` Instance of a solver to be tested fit_intercept : `bool`, default=True Model uses intercept is `True` model : 'linreg' | 'logreg' | 'poisreg', default='logreg' Name of the model used to test the solver decimal : `int`, default=1 Number of decimals required for the test """ # Set seed for data simulation np.random.seed(12) n_samples = TestSolver.n_samples n_features = TestSolver.n_features coeffs0 = weights_sparse_gauss(n_features, nnz=5) if fit_intercept: interc0 = 2. else: interc0 = None if model == 'linreg': X, y = SimuLinReg(coeffs0, interc0, n_samples=n_samples, verbose=False, seed=123).simulate() model = ModelLinReg(fit_intercept=fit_intercept).fit(X, y) elif model == 'logreg': X, y = SimuLogReg(coeffs0, interc0, n_samples=n_samples, verbose=False, seed=123).simulate() model = ModelLogReg(fit_intercept=fit_intercept).fit(X, y) elif model == 'poisreg': X, y = SimuPoisReg(coeffs0, interc0, n_samples=n_samples, verbose=False, seed=123).simulate() # Rescale features to avoid overflows in Poisson simulations X /= np.linalg.norm(X, axis=1).reshape(n_samples, 1) model = ModelPoisReg(fit_intercept=fit_intercept).fit(X, y) else: raise ValueError("``model`` must be either 'linreg', 'logreg' or" " 'poisreg'") solver.set_model(model) strength = 1e-2 prox = ProxL2Sq(strength, (0, model.n_features)) if type(solver) is not SDCA: solver.set_prox(prox) else: solver.set_prox(ProxZero()) solver.l_l2sq = strength coeffs_solver = solver.solve() # Compare with BFGS bfgs = BFGS(max_iter=100, verbose=False).set_model(model).set_prox(prox) coeffs_bfgs = bfgs.solve() np.testing.assert_almost_equal(coeffs_solver, coeffs_bfgs, decimal=decimal) # We ensure that reached coeffs are not equal to zero self.assertGreater(norm(coeffs_solver), 0) self.assertAlmostEqual(solver.objective(coeffs_bfgs), solver.objective(coeffs_solver), delta=1e-2)
def __init__(self, **kwargs): PROX.__init__(self, **kwargs) TPL2SQ.__init__(self, **kwargs) object.__setattr__(self, "_MANGLING", "l2sq")
""" import numpy as np import matplotlib.pyplot as plt from tick.prox import ProxL1, ProxElasticNet, ProxL2Sq, \ ProxPositive, ProxSlope, ProxTV, ProxZero, ProxBinarsity, ProxGroupL1, \ ProxEquality, ProxL1w np.random.seed(12) x = np.random.randn(50) a, b = x.min() - 1e-1, x.max() + 1e-1 s = 0.4 proxs = [ ProxZero(), ProxPositive(), ProxL2Sq(strength=s), ProxL1(strength=s), ProxElasticNet(strength=s, ratio=0.5), ProxSlope(strength=s), ProxTV(strength=s), ProxEquality(range=(25, 40)), ProxL1w(strength=s, weights=0.1 * np.arange(50, dtype=np.double)), ProxGroupL1(strength=2 * s, blocks_start=np.arange(0, 50, 10), blocks_length=10 * np.ones((5, ))), ProxBinarsity(strength=s, blocks_start=np.arange(0, 50, 10), blocks_length=10 * np.ones((5, ))) ] fig, _ = plt.subplots(3, 4, figsize=(16, 12), sharey=True, sharex=True)
#!/usr/bin/python3 # expect tick first on PYTHONPATH from tick.array.build.array import tick_double_sparse2d_from_file, tick_double_array_from_file from tick.prox import ProxL2Sq; from tick.solver import SAGA; from tick.linear_model import ModelLogReg X = tick_double_sparse2d_from_file("url.features.cereal") n_samples = X.shape[0]; n_features = X.shape[1] y = tick_double_array_from_file ("url.labels.cereal") model = ModelLogReg(fit_intercept=False).fit(X, y) prox = ProxL2Sq((1. / n_samples) + 1e-10, range=(0, n_features)) asaga = SAGA(step=0.00257480411965, max_iter=200, tol=1e-10, verbose=False, n_threads=8, log_every_n_epochs=10) asaga.set_model(model).set_prox(prox) asaga.solve() asaga.print_history()