def test_display(self):
"""Check if something is printed when `display` is True."""
with Capturing() as output:
hessian_free_newton(self.oracle, self.x0, display=True)
self.assertTrue(
len(output) > 0,
'You should print the progress when `display` is True.')
def test_line_search_options(self):
"""Check if argument `line_search_options` is supported."""
hessian_free_newton(self.oracle,
self.x0,
line_search_options={
'method': 'Wolfe',
'c1': 1e-4,
'c2': 0.9
})
def test_trace(self):
"""Check if the history is returned correctly when `trace` is True."""
x_min, message, hist = hessian_free_newton(self.oracle, self.x0, trace=True)
self.assertEqual(len(hist['grad_norm']), len(hist['func']))
self.assertEqual(len(hist['time']), len(hist['func']))
self.assertEqual(len(hist['x']), len(hist['func']))
x_min, message, hist = hessian_free_newton(self.oracle, self.x0, trace=False)
self.assertTrue(hist is None)
def test_quality(self):
x_min, message, _ = hessian_free_newton(self.oracle, self.x0, tolerance=1e-10)
f_min = self.oracle.func(x_min)
g_k_norm = norm(self.A.dot(x_min) - self.b, 2)
self.assertLessEqual(abs(g_k_norm), 1e-3)
self.assertLessEqual(abs(f_min - self.f_star), 1e-3)
def test_default(self):
"""Check if everything works correctly with default parameters."""
with Capturing() as output:
x_min, message, _ = hessian_free_newton(self.oracle, self.x0)
assert_equal(message, 'success')
self.assertTrue(len(output) == 0, 'You should not print anything by default.')
def experiment_3():
np.random.seed(31415)
data_path = "data"
datasets = ["w8a", "gisette_scale", "real-sim", "news20", "rcv1_train"]
for dataset in datasets:
print("___________________________")
logging.info(f"{dataset} is in process...")
A, b = load_svmlight_file(os.path.join(data_path, dataset))
print(A.shape, 1 - A.size / (A.shape[0] * A.shape[1]))
m = b.size
oracle = create_log_reg_oracle(A, b, 1.0 / m, "optimized")
x_0 = np.zeros((A.shape[1],))
x_opt1, message, history1 = gradient_descent(oracle, x_0, trace=True)
logging.info("GD ended")
x_opt2, message, history2 = hessian_free_newton(oracle, x_0, trace=True)
logging.info("HFN ended")
x_opt3, message, history3 = lbfgs(oracle, x_0, trace=True)
logging.info("L-BFGS ended")
os.makedirs("report/pics/3", exist_ok=True)
plt.figure()
plt.plot(history1['func'], label='GD')
plt.plot(history2['func'], label='HFN')
plt.plot(history3['func'], label='L-BFGS')
print(f"GD iterations={len(history1['func'])}, time={history1['time'][-1]}")
print(f"HFN iterations={len(history2['func'])}, time={history2['time'][-1]}")
print(f"L-BFGS iterations={len(history3['func'])}, time={history3['time'][-1]}")
plt.xlabel('Номер итерации')
plt.ylabel('Значение функции потерь')
plt.legend()
plt.grid()
plt.savefig(f"report/pics/3/logreg_loss_value_vs_iter_{dataset}.pdf", bbox_inches='tight')
plt.figure()
plt.plot(history1['time'], history1['func'], label='GD')
plt.plot(history2['time'], history2['func'], label='HFN')
plt.plot(history3['time'], history3['func'], label='L-BFGS')
plt.xlabel('Время от начала эксперимента в секундах')
plt.ylabel('Значение функции потерь')
plt.legend()
plt.grid()
plt.savefig(f"report/pics/3/logreg_loss_value_vs_time_{dataset}.pdf", bbox_inches='tight')
plt.figure()
plt.plot(history1['time'], (history1['grad_norm'] / history1['grad_norm'][0]) ** 2, label='GD')
plt.plot(history2['time'], (history2['grad_norm'] / history2['grad_norm'][0]) ** 2, label='HFN')
plt.plot(history3['time'], (history3['grad_norm'] / history3['grad_norm'][0]) ** 2, label='L-BFGS')
plt.yscale('log')
plt.xlabel('Время от начала эксперимента в секундах')
plt.ylabel('Относительный квадрат нормы градиента')
plt.legend()
plt.grid()
plt.savefig(f"report/pics/3/logreg_grad_norm_vs_time_{dataset}.pdf", bbox_inches='tight')
def test_quality(self):
x_min, message, _ = hessian_free_newton(self.oracle, self.x0, tolerance=1e-5)
f_min = self.oracle.func(x_min)
g_k_norm_sqr = norm(self.A.dot(x_min) - self.b, 2)**2
g_0_norm_sqr = norm(self.A.dot(self.x0) - self.b, 2)**2
self.assertLessEqual(g_k_norm_sqr, 1e-5 * g_0_norm_sqr)
self.assertLessEqual(abs(f_min - self.f_star), 1e-5 * g_0_norm_sqr)
def test_history(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = hessian_free_newton(self.oracle, x0, trace=True, line_search_options={'method': 'Constant', 'c': 1.0},
tolerance=1e-6)
func_steps = [25.635, -9.5]
grad_norm_steps = [11.629703349613008, 0.0]
time_steps = [0.0] * 2 # Dummy values
x_steps = [np.array([-1.3, -2.7]),
np.array([1., 3.])]
true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps)
check_equal_histories(history, true_history)
def test_hfn_history(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = hessian_free_newton(self.oracle, x0, trace=True, line_search_options={'method': 'Best'}, tolerance=1e-6)
func_steps = [25.635, -9.5]
grad_norm_steps = [11.629703349613008, 0.0]
time_steps = [0.0] * 2 # Dummy values
x_steps = [np.array([-1.3, -2.7]),
np.array([1., 3.])]
true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps)
self.check_equal_histories(history, true_history)
self.assertEqual(message, "success")
assert_array_almost_equal(x_min, np.array([1.0, 3.0]), decimal=3)
def check_lyoha():
B = np.array(range(1, 10)).reshape(3, 3)
# A = [email protected]
A = B.dot(B.T)
b = np.array(range(1, 4))
oracle = create_log_reg_oracle(A, b, 1.0, oracle_type='optimized')
x0 = np.zeros(3)
x_expected = np.array([0.01081755, 0.02428744, 0.03775733])
hist_expected = {'func': [0.69314718055994518, 0.060072133470449901, 0.020431219493905504],
'time': [0.0] * 3,
'grad_norm': [176.70314088889307, 3.295883082719103, 1.101366262174557]}
x_star, msg, history = hessian_free_newton(oracle, x0, trace=True,
line_search_options={'method': 'Wolfe'}, tolerance=1e-4)
oracle.hess(np.zeros(3))
oracle.hess(np.array([0.01952667, 0.04648169, 0.07343672]))
pass
def test_max_iter(self):
"""Check if argument `max_iter` is supported."""
hessian_free_newton(self.oracle, self.x0, max_iter=15)
def test_tol(self):
"""Check if argument `tol` is supported."""
hessian_free_newton(self.oracle, self.x0, tolerance=1e-6)
