def test_plot_history_solver_log_scale(self): """...Test plot_history rendering on a log scale """ fig = plot_history([self.solver1, self.solver2], show=False, dist_min=True, log_scale=True) ax = fig.axes[0] self.assertEqual(ax.yaxis.get_scale(), 'log')
def test_plot_history_solver(self): """...Test plot_history rendering given a list of solvers """ labels = ['solver 1', 'solver 2'] fig = plot_history([self.solver1, self.solver2], show=False, labels=labels) ax = fig.axes[0] ax_n_iter1, ax_obj1 = ax.lines[0].get_xydata().T np.testing.assert_array_equal(ax_n_iter1, self.n_iter1) np.testing.assert_array_equal(ax_obj1, self.obj1) self.assertEqual(ax.lines[0].get_label(), labels[0]) ax_n_iter2, ax_obj2 = ax.lines[1].get_xydata().T np.testing.assert_array_equal(ax_n_iter2, self.n_iter2) np.testing.assert_array_equal(ax_obj2, self.obj2) self.assertEqual(ax.lines[1].get_label(), labels[1])
def test_plot_history_learner(self): """...Test plot_history rendering given a list of learners """ learner1 = LogisticRegression(solver='svrg') learner1._solver_obj._set('history', self.solver1.history) learner2 = LogisticRegression(solver='agd') learner2._solver_obj._set('history', self.solver2.history) fig = plot_history([learner1, learner2], show=False) ax = fig.axes[0] ax_n_iter1, ax_obj1 = ax.lines[0].get_xydata().T np.testing.assert_array_equal(ax_n_iter1, self.n_iter1) np.testing.assert_array_equal(ax_obj1, self.obj1) self.assertEqual(ax.lines[0].get_label(), 'SVRG') ax_n_iter2, ax_obj2 = ax.lines[1].get_xydata().T np.testing.assert_array_equal(ax_n_iter2, self.n_iter2) np.testing.assert_array_equal(ax_obj2, self.obj2) self.assertEqual(ax.lines[1].get_label(), 'AGD')
def test_plot_history_solver_dist_min(self): """...Test plot_history rendering with dist_min argument """ fig = plot_history([self.solver1, self.solver2], show=False, dist_min=True) ax = fig.axes[0] min_obj = min(min(self.obj1), min(self.obj2)) ax_n_iter1, ax_obj1 = ax.lines[0].get_xydata().T np.testing.assert_array_equal(ax_n_iter1, self.n_iter1) np.testing.assert_array_equal(ax_obj1, np.array(self.obj1) - min_obj) self.assertEqual(ax.lines[0].get_label(), 'GD') ax_n_iter2, ax_obj2 = ax.lines[1].get_xydata().T np.testing.assert_array_equal(ax_n_iter2, self.n_iter2) np.testing.assert_array_equal(ax_obj2, np.array(self.obj2) - min_obj) self.assertEqual(ax.lines[1].get_label(), 'AGD')
for step in tested_steps: svrg = SVRG(max_iter=30, tol=1e-10, verbose=False) svrg.set_model(model).set_prox(prox) svrg.solve(step=step) svrg_bb = SVRG(max_iter=30, tol=1e-10, verbose=False, step_type='bb') svrg_bb.set_model(model).set_prox(prox) svrg_bb.solve(step=step) solvers += [svrg, svrg_bb] optimal_factor = step / optimal_step if optimal_factor != 1: solver_labels += [ 'SVRG {:.2g} * optimal step'.format(optimal_factor), 'SVRG BB {:.2g} * optimal step'.format(optimal_factor) ] else: solver_labels += [ 'SVRG optimal step'.format(optimal_factor), 'SVRG BB optimal step'.format(optimal_factor) ] # To easily differentiate fixed steps from Barzilai Borwein steps SVRG solvers plt.rc('axes', prop_cycle=(cycler('linestyle', ['-', '--']))) plot_history(solvers=solvers, labels=solver_labels, log_scale=True, dist_min=True)
from tick.optim.solver import GD, AGD, SGD, SVRG, SDCA from tick.optim.model import ModelLogReg from tick.optim.prox import ProxElasticNet, ProxL1 from tick.plot import plot_history n_samples, n_features, = 5000, 50 weights0 = weights_sparse_gauss(n_features, nnz=10) intercept0 = 0.2 X, y = SimuLogReg(weights=weights0, intercept=intercept0, n_samples=n_samples, seed=123, verbose=False).simulate() model = ModelLogReg(fit_intercept=True).fit(X, y) prox = ProxElasticNet(strength=1e-3, ratio=0.5, range=(0, n_features)) solver_params = {'max_iter': 100, 'tol': 0., 'verbose': False} x0 = np.zeros(model.n_coeffs) gd = GD(linesearch=False, **solver_params).set_model(model).set_prox(prox) gd.solve(x0, step=1 / model.get_lip_best()) agd = AGD(linesearch=False, **solver_params).set_model(model).set_prox(prox) agd.solve(x0, step=1 / model.get_lip_best()) sgd = SGD(**solver_params).set_model(model).set_prox(prox) sgd.solve(x0, step=500.) svrg = SVRG(**solver_params).set_model(model).set_prox(prox) svrg.solve(x0, step=1 / model.get_lip_max()) plot_history([gd, agd, sgd, svrg], log_scale=True, dist_min=True)
max_iter=max_iter, record_every=2, random_state=seed) learner_32.fit(X_32, y_32) # For a fair comparison, we access private attributes to compute both # objective with float 64 precision learner_32._solver_obj.history.values['obj'] = [ learner_64._solver_obj.objective(coeffs.astype('float64')) for coeffs in learner_32._solver_obj.history.values['x'] ] fig, axes = plt.subplots(1, 2, figsize=(8, 4), sharey=True) plot_history([learner_32, learner_64], x='n_iter', labels=['float 32', 'float 64'], dist_min=True, log_scale=True, ax=axes[0]) plot_history([learner_32, learner_64], x='time', labels=['float 32', 'float 64'], dist_min=True, log_scale=True, ax=axes[1]) axes[0].set_ylabel(r'$\frac{f(w^t) - f(w^*)}{f(w^*)}$') axes[0].set_xlabel('n epochs') axes[1].set_ylabel('') axes[1].set_xlabel('time (s)') plt.show()
svrg_step = 1. / model.get_lip_max() test_n_threads = [1, 2, 4] fig, axes = plt.subplots(1, 2, figsize=(8, 4)) for ax, SolverClass in zip(axes, [SVRG, SAGA]): solver_list = [] solver_labels = [] for n_threads in test_n_threads: solver = SolverClass(step=svrg_step, seed=seed, max_iter=50, verbose=False, n_threads=n_threads, tol=0, record_every=3) solver.set_model(model).set_prox(prox) solver.solve() solver_list += [solver] if n_threads == 1: solver_labels += [solver.name] else: solver_labels += ['A{} {}'.format(solver.name, n_threads)] plot_history(solver_list, x="time", dist_min=True, log_scale=True, labels=solver_labels, ax=ax) ax.set_ylabel('log distance to optimal objective', fontsize=14) fig.tight_layout() plt.show()
fig, axes = plt.subplots(len(model_types), len(l_l2sqs), figsize=(4 * len(l_l2sqs), 3 * len(model_types)), sharey=True, sharex=True) n_samples = 1000 n_features = 20 for (model_type, l_l2sq), ax in zip(product(model_types, l_l2sqs), axes.ravel()): model = create_model(model_type, n_samples, n_features) bfgs, svrg, sdca, gd, agd = run_solvers(model, l_l2sq) plot_history([bfgs, svrg, sdca, gd, agd], ax=ax, dist_min=True, log_scale=True) ax.legend_.remove() ax.set_xlabel('') ax.set_ylim([1e-9, 1]) for l_l2sq, ax in zip(l_l2sqs, axes[0]): ax.set_title('$\lambda = %.2g$' % l_l2sq) for model_type, ax in zip(model_types, axes): ax[0].set_ylabel('%s regression' % model_type, fontsize=17) for ax in axes[-1]: ax.set_xlabel('epochs') axes[-1][1].legend(loc=9, bbox_to_anchor=(0.5, -0.2), ncol=5)
import matplotlib.pyplot as plt from tick.simulation import SimuPoisReg, weights_sparse_gauss from tick.inference import PoissonRegression from tick.plot import plot_history n_samples = 50000 n_features = 100 np.random.seed(123) weight0 = weights_sparse_gauss(n_features, nnz=int(n_features-1)) / 20. intercept0 = -0.1 X, y = SimuPoisReg(weight0, intercept0, n_samples=n_samples, verbose=False, seed=123).simulate() opts = {'verbose': False, 'record_every': 1, 'tol': 1e-8, 'max_iter': 40} poisson_regressions = [ PoissonRegression(solver='gd', **opts), PoissonRegression(solver='agd', **opts), PoissonRegression(solver='svrg', random_state=1234, **opts), PoissonRegression(solver='bfgs', **opts) ] for poisson_regression in poisson_regressions: poisson_regression.fit(X, y) plot_history(poisson_regressions, log_scale=True, dist_min=True) plt.title('Solvers comparison for Poisson regression', fontsize=16) plt.tight_layout()
model = ModelLogReg(fit_intercept=True) model.fit(features, labels) prox = ProxElasticNet(penalty_strength, ratio=0.5, range=(0, n_features)) svrg_step = 1. / model.get_lip_max() test_n_threads = [1, 2, 4] svrg_list = [] svrg_labels = [] for n_threads in test_n_threads: svrg = SVRG(step=svrg_step, seed=seed, max_iter=30, verbose=False, n_threads=n_threads) svrg.set_model(model).set_prox(prox) svrg.solve() svrg_list += [svrg] if n_threads == 1: svrg_labels += ['SVRG'] else: svrg_labels += ['ASVRG {}'.format(n_threads)] plot_history(svrg_list, x="time", dist_min=True, log_scale=True, labels=svrg_labels, show=False) plt.ylim([3e-3, 0.3]) plt.ylabel('log distance to optimal objective', fontsize=14) plt.tight_layout() plt.show()