def test_step_type_setting(self): """...Test that SVRG step_type parameter behaves correctly """ svrg = SVRG() coeffs0 = weights_sparse_gauss(20, nnz=5, dtype=self.dtype) interc0 = None X, y = SimuLogReg(coeffs0, interc0, n_samples=3000, verbose=False, seed=123, dtype=self.dtype).simulate() model = ModelLogReg().fit(X, y) svrg.set_model(model) self.assertEqual(svrg.step_type, 'fixed') self.assertEqual(svrg._solver.get_step_type(), SVRG_StepType_Fixed) svrg = SVRG(step_type='bb') svrg.set_model(model) self.assertEqual(svrg.step_type, 'bb') self.assertEqual(svrg._solver.get_step_type(), SVRG_StepType_BarzilaiBorwein) svrg.step_type = 'fixed' self.assertEqual(svrg.step_type, 'fixed') self.assertEqual(svrg._solver.get_step_type(), SVRG_StepType_Fixed) svrg.step_type = 'bb' self.assertEqual(svrg.step_type, 'bb') self.assertEqual(svrg._solver.get_step_type(), SVRG_StepType_BarzilaiBorwein)
def test_sdca_identity_poisreg(self): """...Test SDCA on specific case of Poisson regression with indentity link """ l_l2sq = 1e-3 n_samples = 10000 n_features = 3 np.random.seed(123) weight0 = np.random.rand(n_features) features = np.random.rand(n_samples, n_features) for intercept in [None, 0.45]: if intercept is None: fit_intercept = False else: fit_intercept = True simu = SimuPoisReg(weight0, intercept=intercept, features=features, n_samples=n_samples, link='identity', verbose=False) features, labels = simu.simulate() model = ModelPoisReg(fit_intercept=fit_intercept, link='identity') model.fit(features, labels) sdca = SDCA(l_l2sq=l_l2sq, max_iter=100, verbose=False, tol=1e-14, seed=Test.sto_seed) sdca.set_model(model).set_prox(ProxZero()) start_dual = np.sqrt(sdca._rand_max * l_l2sq) start_dual = start_dual * np.ones(sdca._rand_max) sdca.solve(start_dual) # Check that duality gap is 0 self.assertAlmostEqual( sdca.objective(sdca.solution), sdca.dual_objective(sdca.dual_solution)) # Check that original vector is approximatively retrieved if fit_intercept: original_coeffs = np.hstack((weight0, intercept)) else: original_coeffs = weight0 np.testing.assert_array_almost_equal(original_coeffs, sdca.solution, decimal=1) # Ensure that we solve the same problem as other solvers svrg = SVRG(max_iter=100, verbose=False, tol=1e-14, seed=Test.sto_seed) svrg.set_model(model).set_prox(ProxL2Sq(l_l2sq)) svrg.solve(0.5 * np.ones(model.n_coeffs), step=1e-2) np.testing.assert_array_almost_equal(svrg.solution, sdca.solution, decimal=4)
def test_dense_and_sparse_match(self): """...Test in SVRG that dense and sparse code matches in all possible settings """ variance_reductions = ['last', 'rand'] rand_types = ['perm', 'unif'] seed = 123 tol = 0. max_iter = 50 n_samples = 500 n_features = 20 # Crazy prox examples proxs = [ ProxTV(strength=1e-2, range=(5, 13), positive=True).astype(self.dtype), ProxElasticNet(strength=1e-2, ratio=0.9).astype(self.dtype), ProxEquality(range=(0, n_features)).astype(self.dtype), ProxL1(strength=1e-3, range=(5, 17)).astype(self.dtype), ProxL1w(strength=1e-3, weights=np.arange(5, 17, dtype=np.double), range=(5, 17)).astype(self.dtype), ] for intercept in [-1, None]: X, y = self.simu_linreg_data(dtype=self.dtype, interc=intercept, n_features=n_features, n_samples=n_samples) fit_intercept = intercept is not None model_dense, model_spars = self.get_dense_and_sparse_linreg_model( X, y, dtype=self.dtype, fit_intercept=fit_intercept) step = 1 / model_spars.get_lip_max() for variance_reduction, rand_type, prox in product( variance_reductions, rand_types, proxs): solver_sparse = SVRG(step=step, tol=tol, max_iter=max_iter, verbose=False, variance_reduction=variance_reduction, rand_type=rand_type, seed=seed) solver_sparse.set_model(model_spars).set_prox(prox) solver_dense = SVRG(step=step, tol=tol, max_iter=max_iter, verbose=False, variance_reduction=variance_reduction, rand_type=rand_type, seed=seed) solver_dense.set_model(model_dense).set_prox(prox) solver_sparse.solve() solver_dense.solve() places = 7 if self.dtype is "float32": places = 3 np.testing.assert_array_almost_equal(solver_sparse.solution, solver_dense.solution, decimal=places)
def test_set_model(self): """...Test SVRG set_model """ X, y = self.simu_linreg_data() _, model_spars = self.get_dense_and_sparse_linreg_model(X, y) svrg = SVRG(variance_reduction='avg') msg = "'avg' variance reduction cannot be used with sparse datasets. " \ "Please change `variance_reduction` before passing sparse data." with catch_warnings(record=True) as w: simplefilter('always') svrg.set_model(model_spars) self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[0].category, UserWarning)) self.assertEqual(str(w[0].message), msg)
def test_convergence_with_lags(self): """Test longitudinal multinomial model convergence.""" n_intervals = 10 n_lags = 3 n_samples = 1500 n_features = 3 sim = SimuSCCS(n_samples, n_intervals, n_features, n_lags, None, True, "short", seed=42, verbose=False) X, y, censoring, coeffs = sim.simulate() X = LongitudinalFeaturesLagger(n_lags=n_lags) \ .fit_transform(X, censoring) model = ModelSCCS(n_intervals=n_intervals, n_lags=n_lags).fit(X, y, censoring) solver = SVRG(max_iter=15, verbose=False) solver.set_model(model).set_prox(ProxZero()) coeffs_svrg = solver.solve(step=1 / model.get_lip_max()) np.testing.assert_almost_equal(coeffs, coeffs_svrg, decimal=1)
def run_solvers(model, l_l2sq): try: svrg_step = 1. / model.get_lip_max() except AttributeError: svrg_step = 1e-3 try: gd_step = 1. / model.get_lip_best() except AttributeError: gd_step = 1e-1 bfgs = BFGS(verbose=False, tol=1e-13) bfgs.set_model(model).set_prox(ProxL2Sq(l_l2sq)) bfgs.solve() bfgs.history.set_minimizer(bfgs.solution) bfgs.history.set_minimum(bfgs.objective(bfgs.solution)) bfgs.solve() svrg = SVRG(step=svrg_step, verbose=False, tol=1e-10, seed=seed) svrg.set_model(model).set_prox(ProxL2Sq(l_l2sq)) svrg.history.set_minimizer(bfgs.solution) svrg.history.set_minimum(bfgs.objective(bfgs.solution)) svrg.solve() sdca = SDCA(l_l2sq, verbose=False, seed=seed, tol=1e-10) sdca.set_model(model).set_prox(ProxZero()) sdca.history.set_minimizer(bfgs.solution) sdca.history.set_minimum(bfgs.objective(bfgs.solution)) sdca.solve() gd = GD(verbose=False, tol=1e-10, step=gd_step, linesearch=False) gd.set_model(model).set_prox(ProxL2Sq(l_l2sq)) gd.history.set_minimizer(bfgs.solution) gd.history.set_minimum(bfgs.objective(bfgs.solution)) gd.solve() agd = AGD(verbose=False, tol=1e-10, step=gd_step, linesearch=False) agd.set_model(model).set_prox(ProxL2Sq(l_l2sq)) agd.history.set_minimizer(bfgs.solution) agd.history.set_minimum(bfgs.objective(bfgs.solution)) agd.solve() return bfgs, svrg, sdca, gd, agd
def test_convergence_with_lags(self): """Test longitudinal multinomial model convergence.""" n_intervals = 10 n_samples = 800 n_features = 2 n_lags = np.repeat(2, n_features).astype(dtype="uint64") sim = SimuSCCS(n_samples, n_intervals, n_features, n_lags, None, "multiple_exposures", seed=42) _, X, y, censoring, coeffs = sim.simulate() coeffs = np.hstack(coeffs) X, _, _ = LongitudinalFeaturesLagger(n_lags=n_lags) \ .fit_transform(X, censoring) model = ModelSCCS(n_intervals=n_intervals, n_lags=n_lags).fit(X, y, censoring) solver = SVRG(max_iter=15, verbose=False) solver.set_model(model).set_prox(ProxZero()) coeffs_svrg = solver.solve(step=1 / model.get_lip_max()) np.testing.assert_almost_equal(coeffs, coeffs_svrg, decimal=1)
def test_variance_reduction_setting(self): """...Test that SVRG variance_reduction parameter behaves correctly """ svrg = SVRG() self.assertEqual(svrg.variance_reduction, 'last') self.assertEqual(svrg._solver.get_variance_reduction(), SVRG_VarianceReductionMethod_Last) svrg = SVRG(variance_reduction='rand') self.assertEqual(svrg.variance_reduction, 'rand') self.assertEqual(svrg._solver.get_variance_reduction(), SVRG_VarianceReductionMethod_Random) svrg.variance_reduction = 'avg' self.assertEqual(svrg.variance_reduction, 'avg') self.assertEqual(svrg._solver.get_variance_reduction(), SVRG_VarianceReductionMethod_Average) svrg.variance_reduction = 'rand' self.assertEqual(svrg.variance_reduction, 'rand') self.assertEqual(svrg._solver.get_variance_reduction(), SVRG_VarianceReductionMethod_Random) svrg.variance_reduction = 'last' self.assertEqual(svrg.variance_reduction, 'last') self.assertEqual(svrg._solver.get_variance_reduction(), SVRG_VarianceReductionMethod_Last) msg = '^variance_reduction should be one of "avg, last, rand", ' \ 'got "stuff"$' with self.assertRaisesRegex(ValueError, msg): svrg = SVRG(variance_reduction='stuff') with self.assertRaisesRegex(ValueError, msg): svrg.variance_reduction = 'stuff' X, y = self.simu_linreg_data() model_dense, model_spars = self.get_dense_and_sparse_linreg_model(X, y) try: svrg.set_model(model_dense) svrg.variance_reduction = 'avg' svrg.variance_reduction = 'last' svrg.variance_reduction = 'rand' svrg.set_model(model_spars) svrg.variance_reduction = 'last' svrg.variance_reduction = 'rand' except Exception: self.fail('Setting variance_reduction in these cases should have ' 'been ok') msg = "'avg' variance reduction cannot be used with sparse datasets" with catch_warnings(record=True) as w: simplefilter('always') svrg.set_model(model_spars) svrg.variance_reduction = 'avg' self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[0].category, UserWarning)) self.assertEqual(str(w[0].message), msg)
X, y = SimuLogReg(weights=weights0, intercept=intercept0, n_samples=n_samples, seed=123, verbose=False).simulate() model = ModelLogReg(fit_intercept=True).fit(X, y) prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features)) x0 = np.zeros(model.n_coeffs) optimal_step = 1 / model.get_lip_max() tested_steps = [optimal_step, 1e-2 * optimal_step, 10 * optimal_step] solvers = [] solver_labels = [] for step in tested_steps: svrg = SVRG(max_iter=30, tol=1e-10, verbose=False) svrg.set_model(model).set_prox(prox) svrg.solve(step=step) svrg_bb = SVRG(max_iter=30, tol=1e-10, verbose=False, step_type='bb') svrg_bb.set_model(model).set_prox(prox) svrg_bb.solve(step=step) solvers += [svrg, svrg_bb] optimal_factor = step / optimal_step if optimal_factor != 1: solver_labels += ['SVRG {:.2g} * optimal step'.format(optimal_factor), 'SVRG BB {:.2g} * optimal step'.format(optimal_factor)] else: solver_labels += ['SVRG optimal step'.format(optimal_factor), 'SVRG BB optimal step'.format(optimal_factor)]