def test_plot_history_solver_log_scale(self):
"""...Test plot_history rendering on a log scale
"""
fig = plot_history([self.solver1, self.solver2],
show=False,
dist_min=True,
log_scale=True)
ax = fig.axes[0]
self.assertEqual(ax.yaxis.get_scale(), 'log')
def test_plot_history_solver(self):
"""...Test plot_history rendering given a list of solvers
"""
labels = ['solver 1', 'solver 2']
fig = plot_history([self.solver1, self.solver2],
show=False,
labels=labels)
ax = fig.axes[0]
ax_n_iter1, ax_obj1 = ax.lines[0].get_xydata().T
np.testing.assert_array_equal(ax_n_iter1, self.n_iter1)
np.testing.assert_array_equal(ax_obj1, self.obj1)
self.assertEqual(ax.lines[0].get_label(), labels[0])
ax_n_iter2, ax_obj2 = ax.lines[1].get_xydata().T
np.testing.assert_array_equal(ax_n_iter2, self.n_iter2)
np.testing.assert_array_equal(ax_obj2, self.obj2)
self.assertEqual(ax.lines[1].get_label(), labels[1])
def test_plot_history_learner(self):
"""...Test plot_history rendering given a list of learners
"""
learner1 = LogisticRegression(solver='svrg')
learner1._solver_obj._set('history', self.solver1.history)
learner2 = LogisticRegression(solver='agd')
learner2._solver_obj._set('history', self.solver2.history)
fig = plot_history([learner1, learner2], show=False)
ax = fig.axes[0]
ax_n_iter1, ax_obj1 = ax.lines[0].get_xydata().T
np.testing.assert_array_equal(ax_n_iter1, self.n_iter1)
np.testing.assert_array_equal(ax_obj1, self.obj1)
self.assertEqual(ax.lines[0].get_label(), 'SVRG')
ax_n_iter2, ax_obj2 = ax.lines[1].get_xydata().T
np.testing.assert_array_equal(ax_n_iter2, self.n_iter2)
np.testing.assert_array_equal(ax_obj2, self.obj2)
self.assertEqual(ax.lines[1].get_label(), 'AGD')
def test_plot_history_solver_dist_min(self):
"""...Test plot_history rendering with dist_min argument
"""
fig = plot_history([self.solver1, self.solver2],
show=False,
dist_min=True)
ax = fig.axes[0]
min_obj = min(min(self.obj1), min(self.obj2))
ax_n_iter1, ax_obj1 = ax.lines[0].get_xydata().T
np.testing.assert_array_equal(ax_n_iter1, self.n_iter1)
np.testing.assert_array_equal(ax_obj1, np.array(self.obj1) - min_obj)
self.assertEqual(ax.lines[0].get_label(), 'GD')
ax_n_iter2, ax_obj2 = ax.lines[1].get_xydata().T
np.testing.assert_array_equal(ax_n_iter2, self.n_iter2)
np.testing.assert_array_equal(ax_obj2, np.array(self.obj2) - min_obj)
self.assertEqual(ax.lines[1].get_label(), 'AGD')
for step in tested_steps:
svrg = SVRG(max_iter=30, tol=1e-10, verbose=False)
svrg.set_model(model).set_prox(prox)
svrg.solve(step=step)
svrg_bb = SVRG(max_iter=30, tol=1e-10, verbose=False, step_type='bb')
svrg_bb.set_model(model).set_prox(prox)
svrg_bb.solve(step=step)
solvers += [svrg, svrg_bb]
optimal_factor = step / optimal_step
if optimal_factor != 1:
solver_labels += [
'SVRG {:.2g} * optimal step'.format(optimal_factor),
'SVRG BB {:.2g} * optimal step'.format(optimal_factor)
]
else:
solver_labels += [
'SVRG optimal step'.format(optimal_factor),
'SVRG BB optimal step'.format(optimal_factor)
]
# To easily differentiate fixed steps from Barzilai Borwein steps SVRG solvers
plt.rc('axes', prop_cycle=(cycler('linestyle', ['-', '--'])))
plot_history(solvers=solvers,
labels=solver_labels,
log_scale=True,
dist_min=True)
from tick.optim.solver import GD, AGD, SGD, SVRG, SDCA
from tick.optim.model import ModelLogReg
from tick.optim.prox import ProxElasticNet, ProxL1
from tick.plot import plot_history
n_samples, n_features, = 5000, 50
weights0 = weights_sparse_gauss(n_features, nnz=10)
intercept0 = 0.2
X, y = SimuLogReg(weights=weights0, intercept=intercept0,
n_samples=n_samples, seed=123, verbose=False).simulate()
model = ModelLogReg(fit_intercept=True).fit(X, y)
prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features))
solver_params = {'max_iter': 100, 'tol': 0., 'verbose': False}
x0 = np.zeros(model.n_coeffs)
gd = GD(linesearch=False, **solver_params).set_model(model).set_prox(prox)
gd.solve(x0, step=1 / model.get_lip_best())
agd = AGD(linesearch=False, **solver_params).set_model(model).set_prox(prox)
agd.solve(x0, step=1 / model.get_lip_best())
sgd = SGD(**solver_params).set_model(model).set_prox(prox)
sgd.solve(x0, step=500.)
svrg = SVRG(**solver_params).set_model(model).set_prox(prox)
svrg.solve(x0, step=1 / model.get_lip_max())
plot_history([gd, agd, sgd, svrg], log_scale=True, dist_min=True)
max_iter=max_iter,
record_every=2,
random_state=seed)
learner_32.fit(X_32, y_32)
# For a fair comparison, we access private attributes to compute both
# objective with float 64 precision
learner_32._solver_obj.history.values['obj'] = [
learner_64._solver_obj.objective(coeffs.astype('float64'))
for coeffs in learner_32._solver_obj.history.values['x']
]
fig, axes = plt.subplots(1, 2, figsize=(8, 4), sharey=True)
plot_history([learner_32, learner_64],
x='n_iter',
labels=['float 32', 'float 64'],
dist_min=True,
log_scale=True,
ax=axes[0])
plot_history([learner_32, learner_64],
x='time',
labels=['float 32', 'float 64'],
dist_min=True,
log_scale=True,
ax=axes[1])
axes[0].set_ylabel(r'$\frac{f(w^t) - f(w^*)}{f(w^*)}$')
axes[0].set_xlabel('n epochs')
axes[1].set_ylabel('')
axes[1].set_xlabel('time (s)')
plt.show()
svrg_step = 1. / model.get_lip_max()
test_n_threads = [1, 2, 4]
fig, axes = plt.subplots(1, 2, figsize=(8, 4))
for ax, SolverClass in zip(axes, [SVRG, SAGA]):
solver_list = []
solver_labels = []
for n_threads in test_n_threads:
solver = SolverClass(step=svrg_step, seed=seed, max_iter=50,
verbose=False, n_threads=n_threads, tol=0,
record_every=3)
solver.set_model(model).set_prox(prox)
solver.solve()
solver_list += [solver]
if n_threads == 1:
solver_labels += [solver.name]
else:
solver_labels += ['A{} {}'.format(solver.name, n_threads)]
plot_history(solver_list, x="time", dist_min=True, log_scale=True,
labels=solver_labels, ax=ax)
ax.set_ylabel('log distance to optimal objective', fontsize=14)
fig.tight_layout()
plt.show()
fig, axes = plt.subplots(len(model_types),
len(l_l2sqs),
figsize=(4 * len(l_l2sqs), 3 * len(model_types)),
sharey=True,
sharex=True)
n_samples = 1000
n_features = 20
for (model_type, l_l2sq), ax in zip(product(model_types, l_l2sqs),
axes.ravel()):
model = create_model(model_type, n_samples, n_features)
bfgs, svrg, sdca, gd, agd = run_solvers(model, l_l2sq)
plot_history([bfgs, svrg, sdca, gd, agd],
ax=ax,
dist_min=True,
log_scale=True)
ax.legend_.remove()
ax.set_xlabel('')
ax.set_ylim([1e-9, 1])
for l_l2sq, ax in zip(l_l2sqs, axes[0]):
ax.set_title('$\lambda = %.2g$' % l_l2sq)
for model_type, ax in zip(model_types, axes):
ax[0].set_ylabel('%s regression' % model_type, fontsize=17)
for ax in axes[-1]:
ax.set_xlabel('epochs')
axes[-1][1].legend(loc=9, bbox_to_anchor=(0.5, -0.2), ncol=5)
import matplotlib.pyplot as plt
from tick.simulation import SimuPoisReg, weights_sparse_gauss
from tick.inference import PoissonRegression
from tick.plot import plot_history
n_samples = 50000
n_features = 100
np.random.seed(123)
weight0 = weights_sparse_gauss(n_features, nnz=int(n_features-1)) / 20.
intercept0 = -0.1
X, y = SimuPoisReg(weight0, intercept0, n_samples=n_samples,
verbose=False, seed=123).simulate()
opts = {'verbose': False, 'record_every': 1, 'tol': 1e-8, 'max_iter': 40}
poisson_regressions = [
PoissonRegression(solver='gd', **opts),
PoissonRegression(solver='agd', **opts),
PoissonRegression(solver='svrg', random_state=1234, **opts),
PoissonRegression(solver='bfgs', **opts)
]
for poisson_regression in poisson_regressions:
poisson_regression.fit(X, y)
plot_history(poisson_regressions, log_scale=True, dist_min=True)
plt.title('Solvers comparison for Poisson regression', fontsize=16)
plt.tight_layout()
model = ModelLogReg(fit_intercept=True)
model.fit(features, labels)
prox = ProxElasticNet(penalty_strength, ratio=0.5, range=(0, n_features))
svrg_step = 1. / model.get_lip_max()
test_n_threads = [1, 2, 4]
svrg_list = []
svrg_labels = []
for n_threads in test_n_threads:
svrg = SVRG(step=svrg_step, seed=seed, max_iter=30, verbose=False,
n_threads=n_threads)
svrg.set_model(model).set_prox(prox)
svrg.solve()
svrg_list += [svrg]
if n_threads == 1:
svrg_labels += ['SVRG']
else:
svrg_labels += ['ASVRG {}'.format(n_threads)]
plot_history(svrg_list, x="time", dist_min=True, log_scale=True,
labels=svrg_labels, show=False)
plt.ylim([3e-3, 0.3])
plt.ylabel('log distance to optimal objective', fontsize=14)
plt.tight_layout()
plt.show()
