def test_sdca_identity_poisreg(self): """...Test SDCA on specific case of Poisson regression with indentity link """ l_l2sq = 1e-3 n_samples = 10000 n_features = 3 np.random.seed(123) weight0 = np.random.rand(n_features) features = np.random.rand(n_samples, n_features) for intercept in [None, 0.45]: if intercept is None: fit_intercept = False else: fit_intercept = True simu = SimuPoisReg(weight0, intercept=intercept, features=features, n_samples=n_samples, link='identity', verbose=False) features, labels = simu.simulate() model = ModelPoisReg(fit_intercept=fit_intercept, link='identity') model.fit(features, labels) sdca = SDCA(l_l2sq=l_l2sq, max_iter=100, verbose=False, tol=1e-14, seed=Test.sto_seed) sdca.set_model(model).set_prox(ProxZero()) start_dual = np.sqrt(sdca._rand_max * l_l2sq) start_dual = start_dual * np.ones(sdca._rand_max) sdca.solve(start_dual) # Check that duality gap is 0 self.assertAlmostEqual(sdca.objective(sdca.solution), sdca.dual_objective(sdca.dual_solution)) # Check that original vector is approximatively retrieved if fit_intercept: original_coeffs = np.hstack((weight0, intercept)) else: original_coeffs = weight0 np.testing.assert_array_almost_equal(original_coeffs, sdca.solution, decimal=1) # Ensure that we solve the same problem as other solvers svrg = SVRG(max_iter=100, verbose=False, tol=1e-14, seed=Test.sto_seed) svrg.set_model(model).set_prox(ProxL2Sq(l_l2sq)) svrg.solve(0.5 * np.ones(model.n_coeffs), step=1e-2) np.testing.assert_array_almost_equal(svrg.solution, sdca.solution, decimal=4)
def test_set_model(self): """...Test SVRG set_model """ X, y = self.simu_linreg_data() _, model_spars = self.get_dense_and_sparse_linreg_model(X, y) svrg = SVRG(variance_reduction='avg') msg = "'avg' variance reduction cannot be used with sparse datasets. " \ "Please change `variance_reduction` before passing sparse data." with catch_warnings(record=True) as w: simplefilter('always') svrg.set_model(model_spars) self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[0].category, UserWarning)) self.assertEqual(str(w[0].message), msg)
def test_convergence_with_lags(self): """Test longitudinal multinomial model convergence.""" n_intervals = 10 n_lags = 3 n_samples = 5000 n_features = 3 sim = SimuSCCS(n_samples, n_intervals, n_features, n_lags, None, True, "short", seed=42, verbose=False) X, y, censoring, coeffs = sim.simulate() X = LongitudinalFeaturesLagger(n_lags=n_lags) \ .fit_transform(X, censoring) model = ModelSCCS(n_intervals=n_intervals, n_lags=n_lags).fit(X, y, censoring) solver = SVRG(max_iter=15, verbose=False) solver.set_model(model).set_prox(ProxZero()) coeffs_svrg = solver.solve(step=1/model.get_lip_max()) np.testing.assert_almost_equal(coeffs, coeffs_svrg, decimal=1)
def run_solvers(model, l_l2sq): try: svrg_step = 1. / model.get_lip_max() except AttributeError: svrg_step = 1e-3 try: gd_step = 1. / model.get_lip_best() except AttributeError: gd_step = 1e-1 bfgs = BFGS(verbose=False, tol=1e-13) bfgs.set_model(model).set_prox(ProxL2Sq(l_l2sq)) bfgs.solve() bfgs.history.set_minimizer(bfgs.solution) bfgs.history.set_minimum(bfgs.objective(bfgs.solution)) bfgs.solve() svrg = SVRG(step=svrg_step, verbose=False, tol=1e-10, seed=seed) svrg.set_model(model).set_prox(ProxL2Sq(l_l2sq)) svrg.history.set_minimizer(bfgs.solution) svrg.history.set_minimum(bfgs.objective(bfgs.solution)) svrg.solve() sdca = SDCA(l_l2sq, verbose=False, seed=seed, tol=1e-10) sdca.set_model(model).set_prox(ProxZero()) sdca.history.set_minimizer(bfgs.solution) sdca.history.set_minimum(bfgs.objective(bfgs.solution)) sdca.solve() gd = GD(verbose=False, tol=1e-10, step=gd_step, linesearch=False) gd.set_model(model).set_prox(ProxL2Sq(l_l2sq)) gd.history.set_minimizer(bfgs.solution) gd.history.set_minimum(bfgs.objective(bfgs.solution)) gd.solve() agd = AGD(verbose=False, tol=1e-10, step=gd_step, linesearch=False) agd.set_model(model).set_prox(ProxL2Sq(l_l2sq)) agd.history.set_minimizer(bfgs.solution) agd.history.set_minimum(bfgs.objective(bfgs.solution)) agd.solve() return bfgs, svrg, sdca, gd, agd
def test_variance_reduction_setting(self): """...Test that SVRG variance_reduction parameter behaves correctly """ svrg = SVRG() self.assertEqual(svrg.variance_reduction, 'last') self.assertEqual(svrg._solver.get_variance_reduction(), _SVRG.VarianceReductionMethod_Last) svrg = SVRG(variance_reduction='rand') self.assertEqual(svrg.variance_reduction, 'rand') self.assertEqual(svrg._solver.get_variance_reduction(), _SVRG.VarianceReductionMethod_Random) svrg.variance_reduction = 'avg' self.assertEqual(svrg.variance_reduction, 'avg') self.assertEqual(svrg._solver.get_variance_reduction(), _SVRG.VarianceReductionMethod_Average) svrg.variance_reduction = 'rand' self.assertEqual(svrg.variance_reduction, 'rand') self.assertEqual(svrg._solver.get_variance_reduction(), _SVRG.VarianceReductionMethod_Random) svrg.variance_reduction = 'last' self.assertEqual(svrg.variance_reduction, 'last') self.assertEqual(svrg._solver.get_variance_reduction(), _SVRG.VarianceReductionMethod_Last) msg = '^variance_reduction should be one of "avg, last, rand", ' \ 'got "stuff"$' with self.assertRaisesRegex(ValueError, msg): svrg = SVRG(variance_reduction='stuff') with self.assertRaisesRegex(ValueError, msg): svrg.variance_reduction = 'stuff' X, y = self.simu_linreg_data() model_dense, model_spars = self.get_dense_and_sparse_linreg_model(X, y) try: svrg.set_model(model_dense) svrg.variance_reduction = 'avg' svrg.variance_reduction = 'last' svrg.variance_reduction = 'rand' svrg.set_model(model_spars) svrg.variance_reduction = 'last' svrg.variance_reduction = 'rand' except Exception: self.fail('Setting variance_reduction in these cases should have ' 'been ok') msg = "'avg' variance reduction cannot be used with sparse datasets" with catch_warnings(record=True) as w: simplefilter('always') svrg.set_model(model_spars) svrg.variance_reduction = 'avg' self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[0].category, UserWarning)) self.assertEqual(str(w[0].message), msg)
seed=123, verbose=False).simulate() model = ModelLogReg(fit_intercept=True).fit(X, y) prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features)) x0 = np.zeros(model.n_coeffs) optimal_step = 1 / model.get_lip_max() tested_steps = [optimal_step, 1e-2 * optimal_step, 10 * optimal_step] solvers = [] solver_labels = [] for step in tested_steps: svrg = SVRG(max_iter=30, tol=1e-10, verbose=False) svrg.set_model(model).set_prox(prox) svrg.solve(step=step) svrg_bb = SVRG(max_iter=30, tol=1e-10, verbose=False, step_type='bb') svrg_bb.set_model(model).set_prox(prox) svrg_bb.solve(step=step) solvers += [svrg, svrg_bb] optimal_factor = step / optimal_step if optimal_factor != 1: solver_labels += [ 'SVRG {:.2g} * optimal step'.format(optimal_factor), 'SVRG BB {:.2g} * optimal step'.format(optimal_factor) ] else: