def test_display(self): """Check if something is printed when `display` is True.""" with Capturing() as output: hessian_free_newton(self.oracle, self.x0, display=True) self.assertTrue( len(output) > 0, 'You should print the progress when `display` is True.')
def test_line_search_options(self): """Check if argument `line_search_options` is supported.""" hessian_free_newton(self.oracle, self.x0, line_search_options={ 'method': 'Wolfe', 'c1': 1e-4, 'c2': 0.9 })
def test_trace(self): """Check if the history is returned correctly when `trace` is True.""" x_min, message, hist = hessian_free_newton(self.oracle, self.x0, trace=True) self.assertEqual(len(hist['grad_norm']), len(hist['func'])) self.assertEqual(len(hist['time']), len(hist['func'])) self.assertEqual(len(hist['x']), len(hist['func'])) x_min, message, hist = hessian_free_newton(self.oracle, self.x0, trace=False) self.assertTrue(hist is None)
def test_quality(self): x_min, message, _ = hessian_free_newton(self.oracle, self.x0, tolerance=1e-10) f_min = self.oracle.func(x_min) g_k_norm = norm(self.A.dot(x_min) - self.b, 2) self.assertLessEqual(abs(g_k_norm), 1e-3) self.assertLessEqual(abs(f_min - self.f_star), 1e-3)
def test_default(self): """Check if everything works correctly with default parameters.""" with Capturing() as output: x_min, message, _ = hessian_free_newton(self.oracle, self.x0) assert_equal(message, 'success') self.assertTrue(len(output) == 0, 'You should not print anything by default.')
def experiment_3(): np.random.seed(31415) data_path = "data" datasets = ["w8a", "gisette_scale", "real-sim", "news20", "rcv1_train"] for dataset in datasets: print("___________________________") logging.info(f"{dataset} is in process...") A, b = load_svmlight_file(os.path.join(data_path, dataset)) print(A.shape, 1 - A.size / (A.shape[0] * A.shape[1])) m = b.size oracle = create_log_reg_oracle(A, b, 1.0 / m, "optimized") x_0 = np.zeros((A.shape[1],)) x_opt1, message, history1 = gradient_descent(oracle, x_0, trace=True) logging.info("GD ended") x_opt2, message, history2 = hessian_free_newton(oracle, x_0, trace=True) logging.info("HFN ended") x_opt3, message, history3 = lbfgs(oracle, x_0, trace=True) logging.info("L-BFGS ended") os.makedirs("report/pics/3", exist_ok=True) plt.figure() plt.plot(history1['func'], label='GD') plt.plot(history2['func'], label='HFN') plt.plot(history3['func'], label='L-BFGS') print(f"GD iterations={len(history1['func'])}, time={history1['time'][-1]}") print(f"HFN iterations={len(history2['func'])}, time={history2['time'][-1]}") print(f"L-BFGS iterations={len(history3['func'])}, time={history3['time'][-1]}") plt.xlabel('Номер итерации') plt.ylabel('Значение функции потерь') plt.legend() plt.grid() plt.savefig(f"report/pics/3/logreg_loss_value_vs_iter_{dataset}.pdf", bbox_inches='tight') plt.figure() plt.plot(history1['time'], history1['func'], label='GD') plt.plot(history2['time'], history2['func'], label='HFN') plt.plot(history3['time'], history3['func'], label='L-BFGS') plt.xlabel('Время от начала эксперимента в секундах') plt.ylabel('Значение функции потерь') plt.legend() plt.grid() plt.savefig(f"report/pics/3/logreg_loss_value_vs_time_{dataset}.pdf", bbox_inches='tight') plt.figure() plt.plot(history1['time'], (history1['grad_norm'] / history1['grad_norm'][0]) ** 2, label='GD') plt.plot(history2['time'], (history2['grad_norm'] / history2['grad_norm'][0]) ** 2, label='HFN') plt.plot(history3['time'], (history3['grad_norm'] / history3['grad_norm'][0]) ** 2, label='L-BFGS') plt.yscale('log') plt.xlabel('Время от начала эксперимента в секундах') plt.ylabel('Относительный квадрат нормы градиента') plt.legend() plt.grid() plt.savefig(f"report/pics/3/logreg_grad_norm_vs_time_{dataset}.pdf", bbox_inches='tight')
def test_quality(self): x_min, message, _ = hessian_free_newton(self.oracle, self.x0, tolerance=1e-5) f_min = self.oracle.func(x_min) g_k_norm_sqr = norm(self.A.dot(x_min) - self.b, 2)**2 g_0_norm_sqr = norm(self.A.dot(self.x0) - self.b, 2)**2 self.assertLessEqual(g_k_norm_sqr, 1e-5 * g_0_norm_sqr) self.assertLessEqual(abs(f_min - self.f_star), 1e-5 * g_0_norm_sqr)
def test_history(self): x0 = -np.array([1.3, 2.7]) x_min, message, history = hessian_free_newton(self.oracle, x0, trace=True, line_search_options={'method': 'Constant', 'c': 1.0}, tolerance=1e-6) func_steps = [25.635, -9.5] grad_norm_steps = [11.629703349613008, 0.0] time_steps = [0.0] * 2 # Dummy values x_steps = [np.array([-1.3, -2.7]), np.array([1., 3.])] true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps) check_equal_histories(history, true_history)
def test_hfn_history(self): x0 = -np.array([1.3, 2.7]) x_min, message, history = hessian_free_newton(self.oracle, x0, trace=True, line_search_options={'method': 'Best'}, tolerance=1e-6) func_steps = [25.635, -9.5] grad_norm_steps = [11.629703349613008, 0.0] time_steps = [0.0] * 2 # Dummy values x_steps = [np.array([-1.3, -2.7]), np.array([1., 3.])] true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps) self.check_equal_histories(history, true_history) self.assertEqual(message, "success") assert_array_almost_equal(x_min, np.array([1.0, 3.0]), decimal=3)
def check_lyoha(): B = np.array(range(1, 10)).reshape(3, 3) # A = [email protected] A = B.dot(B.T) b = np.array(range(1, 4)) oracle = create_log_reg_oracle(A, b, 1.0, oracle_type='optimized') x0 = np.zeros(3) x_expected = np.array([0.01081755, 0.02428744, 0.03775733]) hist_expected = {'func': [0.69314718055994518, 0.060072133470449901, 0.020431219493905504], 'time': [0.0] * 3, 'grad_norm': [176.70314088889307, 3.295883082719103, 1.101366262174557]} x_star, msg, history = hessian_free_newton(oracle, x0, trace=True, line_search_options={'method': 'Wolfe'}, tolerance=1e-4) oracle.hess(np.zeros(3)) oracle.hess(np.array([0.01952667, 0.04648169, 0.07343672])) pass
def test_max_iter(self): """Check if argument `max_iter` is supported.""" hessian_free_newton(self.oracle, self.x0, max_iter=15)
def test_tol(self): """Check if argument `tol` is supported.""" hessian_free_newton(self.oracle, self.x0, tolerance=1e-6)