def compare_solver_sdca(self):
"""...Compare SDCA solution with SVRG solution
"""
np.random.seed(12)
n_samples = Test.n_samples
n_features = Test.n_features
for fit_intercept in [True, False]:
y, X, coeffs0, interc0 = TestSolver.generate_logistic_data(
n_features, n_samples)
model = ModelLogReg(fit_intercept=fit_intercept).fit(X, y)
ratio = 0.5
l_enet = 1e-2
# SDCA "elastic-net" formulation is different from elastic-net
# implementation
l_l2_sdca = ratio * l_enet
l_l1_sdca = (1 - ratio) * l_enet
sdca = SDCA(l_l2sq=l_l2_sdca, max_iter=100, verbose=False, tol=0,
seed=Test.sto_seed).set_model(model)
prox_l1 = ProxL1(l_l1_sdca)
sdca.set_prox(prox_l1)
coeffs_sdca = sdca.solve()
# Compare with SVRG
svrg = SVRG(max_iter=100, verbose=False, tol=0,
seed=Test.sto_seed).set_model(model)
prox_enet = ProxElasticNet(l_enet, ratio)
svrg.set_prox(prox_enet)
coeffs_svrg = svrg.solve(step=0.1)
np.testing.assert_allclose(coeffs_sdca, coeffs_svrg)
def test_solver_sdca(self):
"""...Check SDCA solver for a Logistic regression with Ridge
penalization and L1 penalization
"""
solver = SDCA(l_l2sq=1e-5, max_iter=100, verbose=False, tol=0)
self.check_solver(solver, fit_intercept=False, model="logreg",
decimal=1)
def test_sdca_identity_poisreg(self):
"""...Test SDCA on specific case of Poisson regression with
indentity link
"""
l_l2sq = 1e-3
n_samples = 10000
n_features = 3
np.random.seed(123)
weight0 = np.random.rand(n_features)
features = np.random.rand(n_samples, n_features)
for intercept in [None, 0.45]:
if intercept is None:
fit_intercept = False
else:
fit_intercept = True
simu = SimuPoisReg(weight0, intercept=intercept, features=features,
n_samples=n_samples, link='identity',
verbose=False)
features, labels = simu.simulate()
model = ModelPoisReg(fit_intercept=fit_intercept, link='identity')
model.fit(features, labels)
sdca = SDCA(l_l2sq=l_l2sq, max_iter=100, verbose=False, tol=1e-14,
seed=Test.sto_seed)
sdca.set_model(model).set_prox(ProxZero())
start_dual = np.sqrt(sdca._rand_max * l_l2sq)
start_dual = start_dual * np.ones(sdca._rand_max)
sdca.solve(start_dual)
# Check that duality gap is 0
self.assertAlmostEqual(
sdca.objective(sdca.solution),
sdca.dual_objective(sdca.dual_solution))
# Check that original vector is approximatively retrieved
if fit_intercept:
original_coeffs = np.hstack((weight0, intercept))
else:
original_coeffs = weight0
np.testing.assert_array_almost_equal(original_coeffs,
sdca.solution, decimal=1)
# Ensure that we solve the same problem as other solvers
svrg = SVRG(max_iter=100, verbose=False, tol=1e-14,
seed=Test.sto_seed)
svrg.set_model(model).set_prox(ProxL2Sq(l_l2sq))
svrg.solve(0.5 * np.ones(model.n_coeffs), step=1e-2)
np.testing.assert_array_almost_equal(svrg.solution, sdca.solution,
decimal=4)
def run_solvers(model, l_l2sq):
try:
svrg_step = 1. / model.get_lip_max()
except AttributeError:
svrg_step = 1e-3
try:
gd_step = 1. / model.get_lip_best()
except AttributeError:
gd_step = 1e-1
bfgs = BFGS(verbose=False, tol=1e-13)
bfgs.set_model(model).set_prox(ProxL2Sq(l_l2sq))
bfgs.solve()
bfgs.history.set_minimizer(bfgs.solution)
bfgs.history.set_minimum(bfgs.objective(bfgs.solution))
bfgs.solve()
svrg = SVRG(step=svrg_step, verbose=False, tol=1e-10, seed=seed)
svrg.set_model(model).set_prox(ProxL2Sq(l_l2sq))
svrg.history.set_minimizer(bfgs.solution)
svrg.history.set_minimum(bfgs.objective(bfgs.solution))
svrg.solve()
sdca = SDCA(l_l2sq, verbose=False, seed=seed, tol=1e-10)
sdca.set_model(model).set_prox(ProxZero())
sdca.history.set_minimizer(bfgs.solution)
sdca.history.set_minimum(bfgs.objective(bfgs.solution))
sdca.solve()
gd = GD(verbose=False, tol=1e-10, step=gd_step, linesearch=False)
gd.set_model(model).set_prox(ProxL2Sq(l_l2sq))
gd.history.set_minimizer(bfgs.solution)
gd.history.set_minimum(bfgs.objective(bfgs.solution))
gd.solve()
agd = AGD(verbose=False, tol=1e-10, step=gd_step, linesearch=False)
agd.set_model(model).set_prox(ProxL2Sq(l_l2sq))
agd.history.set_minimizer(bfgs.solution)
agd.history.set_minimum(bfgs.objective(bfgs.solution))
agd.solve()
return bfgs, svrg, sdca, gd, agd
def test_serializing_solvers(self):
"""...Test serialization of solvers
"""
ratio = 0.5
l_enet = 1e-2
sd = ratio * l_enet
solvers = [
AdaGrad(step=1e-3, max_iter=100, verbose=False, tol=0),
SGD(step=1e-3, max_iter=100, verbose=False, tol=0),
SDCA(l_l2sq=sd, max_iter=100, verbose=False, tol=0),
SAGA(step=1e-3, max_iter=100, verbose=False, tol=0),
SVRG(step=1e-3, max_iter=100, verbose=False, tol=0)
]
model_map = {
ModelLinReg: SimuLinReg,
ModelLogReg: SimuLogReg,
ModelPoisReg: SimuPoisReg,
ModelHinge: SimuLogReg,
ModelQuadraticHinge: SimuLogReg,
ModelSmoothedHinge: SimuLogReg,
ModelAbsoluteRegression: SimuLinReg,
ModelEpsilonInsensitive: SimuLinReg,
ModelHuber: SimuLinReg,
ModelLinRegWithIntercepts: SimuLinReg,
ModelModifiedHuber: SimuLogReg
}
for solver in solvers:
for mod in model_map:
np.random.seed(12)
n_samples, n_features = 100, 5
w0 = np.random.randn(n_features)
intercept0 = 50 * weights_sparse_gauss(n_weights=n_samples,
nnz=30)
c0 = None
X, y = SimuLinReg(w0,
c0,
n_samples=n_samples,
verbose=False,
seed=2038).simulate()
if mod == ModelLinRegWithIntercepts:
y += intercept0
model = mod(fit_intercept=False).fit(X, y)
prox = ProxL1(2.)
solver.set_model(model)
solver.set_prox(prox)
pickled = pickle.loads(pickle.dumps(solver))
self.assertTrue(solver._solver.compare(pickled._solver))
self.assertTrue(
solver.model._model.compare(pickled.model._model))
self.assertTrue(solver.prox._prox.compare(pickled.prox._prox))
if mod == ModelLinRegWithIntercepts:
test_vector = np.hstack((X[0], np.ones(n_samples)))
self.assertEqual(model.loss(test_vector),
solver.model.loss(test_vector))
else:
self.assertEqual(model.loss(X[0]), solver.model.loss(X[0]))
def create_solver():
return SDCA(max_iter=1, verbose=False, l_l2sq=1e-3,
seed=TestSolver.sto_seed)
def create_solver():
return SDCA(l_l2sq=0.1,
max_iter=100,
verbose=False,
seed=TestSolver.sto_seed)