def test_display(self):
"""Check if something is printed when `display` is True."""
with Capturing() as output:
lbfgs(self.oracle, self.x0, display=True)
self.assertTrue(
len(output) > 0,
'You should print the progress when `display` is True.')
def test_line_search_options(self):
"""Check if argument `line_search_options` is supported."""
lbfgs(self.oracle,
self.x0,
line_search_options={
'method': 'Wolfe',
'c1': 1e-4,
'c2': 0.9
})
def test_trace(self):
"""Check if the history is returned correctly when `trace` is True."""
x_min, message, hist = lbfgs(self.oracle, self.x0, trace=True)
self.assertEqual(len(hist['grad_norm']), len(hist['func']))
self.assertEqual(len(hist['time']), len(hist['func']))
self.assertEqual(len(hist['x']), len(hist['func']))
x_min, message, hist = lbfgs(self.oracle, self.x0, trace=False)
self.assertTrue(hist is None)
def second_experiment():
data_path = 'data/gisette_scale'
result_path = lambda x: 'experiment_2/grad_norm-vs-{}'.format(x)
A, b = load_svmlight_file(data_path)
m, n = A.shape
oracle = oracles.create_log_reg_oracle(A, b, 1 / m)
results = []
ls = [0, 1, 5, 10, 50, 100]
for l in ls:
_, _, history = optimization.lbfgs(oracle,
np.zeros(n),
memory_size=l,
trace=True)
print('lbfgs with l = {} finished'.format(l))
grad_norm = np.array(history['grad_norm'])
grad_norm /= grad_norm[0]
grad_norm = np.power(grad_norm, 2)
grad_norm = np.log(grad_norm)
results.append((l, grad_norm, history['time']))
def plotting(flag):
plt.figure(figsize=(12, 8))
for l, grad_norm, times in results:
x = list(range(len(grad_norm))) if flag == 'iterations' else times
plt.plot(x, grad_norm, label='history size = {}'.format(l))
plt.xlabel('iterations' if flag == 'iterations' else 'seconds')
plt.ylabel(r'$\log\left(grad\_norm\right)$')
plt.legend()
plt.grid()
plt.savefig(result_path(flag))
plotting('seconds')
plotting('iterations')
def test_quality(self):
x_min, message, _ = lbfgs(self.oracle, self.x0, tolerance=1e-10)
f_min = self.oracle.func(x_min)
g_k_norm = norm(self.A.dot(x_min) - self.b, 2)
self.assertLessEqual(abs(g_k_norm), 1e-3)
self.assertLessEqual(abs(f_min - self.f_star), 1e-3)
def test_default(self):
"""Check if everything works correctly with default parameters."""
with Capturing() as output:
x_min, message, _ = lbfgs(self.oracle, self.x0)
assert_equal(message, 'success')
self.assertEqual(len(output), 0, 'You should not print anything by default.')
def test_history(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = lbfgs(self.oracle,
x0,
trace=True,
memory_size=10,
line_search_options={
'method': 'Constant',
'c': 1.0
},
tolerance=1e-6)
func_steps = [
25.635000000000005, 22.99, -9.3476294733722725,
-9.4641732176886055, -9.5
]
grad_norm_steps = [
11.629703349613008, 11.4, 0.55751193505619512, 0.26830541958992876,
0.0
]
time_steps = [0.0] * 5 # Dummy values
x_steps = [
np.array([-1.3, -2.7]),
np.array([1.0, 8.7]),
np.array([0.45349973, 3.05512941]),
np.array([0.73294321, 3.01292737]),
np.array([0.99999642, 2.99998814])
]
true_history = dict(grad_norm=grad_norm_steps,
time=time_steps,
x=x_steps,
func=func_steps)
check_equal_histories(history, true_history)
def experiment_3():
np.random.seed(31415)
data_path = "data"
datasets = ["w8a", "gisette_scale", "real-sim", "news20", "rcv1_train"]
for dataset in datasets:
print("___________________________")
logging.info(f"{dataset} is in process...")
A, b = load_svmlight_file(os.path.join(data_path, dataset))
print(A.shape, 1 - A.size / (A.shape[0] * A.shape[1]))
m = b.size
oracle = create_log_reg_oracle(A, b, 1.0 / m, "optimized")
x_0 = np.zeros((A.shape[1],))
x_opt1, message, history1 = gradient_descent(oracle, x_0, trace=True)
logging.info("GD ended")
x_opt2, message, history2 = hessian_free_newton(oracle, x_0, trace=True)
logging.info("HFN ended")
x_opt3, message, history3 = lbfgs(oracle, x_0, trace=True)
logging.info("L-BFGS ended")
os.makedirs("report/pics/3", exist_ok=True)
plt.figure()
plt.plot(history1['func'], label='GD')
plt.plot(history2['func'], label='HFN')
plt.plot(history3['func'], label='L-BFGS')
print(f"GD iterations={len(history1['func'])}, time={history1['time'][-1]}")
print(f"HFN iterations={len(history2['func'])}, time={history2['time'][-1]}")
print(f"L-BFGS iterations={len(history3['func'])}, time={history3['time'][-1]}")
plt.xlabel('Номер итерации')
plt.ylabel('Значение функции потерь')
plt.legend()
plt.grid()
plt.savefig(f"report/pics/3/logreg_loss_value_vs_iter_{dataset}.pdf", bbox_inches='tight')
plt.figure()
plt.plot(history1['time'], history1['func'], label='GD')
plt.plot(history2['time'], history2['func'], label='HFN')
plt.plot(history3['time'], history3['func'], label='L-BFGS')
plt.xlabel('Время от начала эксперимента в секундах')
plt.ylabel('Значение функции потерь')
plt.legend()
plt.grid()
plt.savefig(f"report/pics/3/logreg_loss_value_vs_time_{dataset}.pdf", bbox_inches='tight')
plt.figure()
plt.plot(history1['time'], (history1['grad_norm'] / history1['grad_norm'][0]) ** 2, label='GD')
plt.plot(history2['time'], (history2['grad_norm'] / history2['grad_norm'][0]) ** 2, label='HFN')
plt.plot(history3['time'], (history3['grad_norm'] / history3['grad_norm'][0]) ** 2, label='L-BFGS')
plt.yscale('log')
plt.xlabel('Время от начала эксперимента в секундах')
plt.ylabel('Относительный квадрат нормы градиента')
plt.legend()
plt.grid()
plt.savefig(f"report/pics/3/logreg_grad_norm_vs_time_{dataset}.pdf", bbox_inches='tight')
def test_quality(self):
x_min, message, _ = lbfgs(self.oracle, self.x0, tolerance=1e-5)
f_min = self.oracle.func(x_min)
g_k_norm_sqr = norm(self.A.dot(x_min) - self.b, 2)**2
g_0_norm_sqr = norm(self.A.dot(self.x0) - self.b, 2)**2
self.assertLessEqual(g_k_norm_sqr, 1e-5 * g_0_norm_sqr)
self.assertLessEqual(abs(f_min - self.f_star), 1e-5 * g_0_norm_sqr)
def test_history_best(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = lbfgs(self.oracle, x0, trace=True, memory_size=10, line_search_options={'method': 'Best'}, tolerance=1e-6)
func_steps = [25.635000000000005,
-8.8519395950378961,
-9.5]
grad_norm_steps = [11.629703349613008,
1.1497712070693149,
0]
time_steps = [0.0] * 3 # Dummy values
x_steps = [np.array([-1.3, -2.7]),
np.array([-0.12706157, 3.11369481]),
np.array([1., 3.])]
true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps)
check_equal_histories(history, true_history)
def experiment_2():
np.random.seed(31415)
A, b = load_svmlight_file('data/gisette_scale')
m = b.size
oracle = create_log_reg_oracle(A, b, 1.0 / m, "optimized")
x_0 = np.zeros((A.shape[1],))
fig1, ax1 = plt.subplots()
fig2, ax2 = plt.subplots()
ax1.set_yscale('log')
ax1.set_xlabel('Номер итерации')
ax1.set_ylabel('Относительный квадрат нормы градиента')
ax1.grid()
ax2.set_yscale('log')
ax2.set_xlabel('Время от начала эксперимента')
ax2.set_ylabel('Относительный квадрат нормы градиента')
ax2.grid()
for memory_size in tqdm([0, 1, 5, 10, 50, 100]):
x_opt, message, history = lbfgs(oracle, x_0, trace=True, memory_size=memory_size)
relative_grad_norms = (history['grad_norm'] / history['grad_norm'][0]) ** 2
iter_times = history['time']
iters = range(len(history['time']))
if len(relative_grad_norms) > 400:
relative_grad_norms = [elem for (i, elem) in enumerate(relative_grad_norms) if i % 2 == 0]
iter_times = [elem for (i, elem) in enumerate(iter_times) if i % 2 == 0]
iters = range(0, len(history['time']), 2)
ax1.plot(iters, relative_grad_norms, label=f"l={memory_size}")
ax2.plot(iter_times, relative_grad_norms, label=f"l={memory_size}")
print(f"При l={memory_size} до сходимости потребовалось "
f"{len(history['time'])} итераций и {history['time'][-1]} секунд")
ax1.legend()
ax2.legend()
os.makedirs("report/pics/2", exist_ok=True)
fig1.savefig("report/pics/2/lbfgs_grad_norm_vs_iter.pdf", bbox_inches='tight')
fig2.savefig("report/pics/2/lbfgs_grad_norm_vs_time.pdf", bbox_inches='tight')
def test_memory_size(self):
"""Check if argument `memory_size` is supported."""
lbfgs(self.oracle, self.x0, memory_size=1)
def test_max_iter(self):
"""Check if argument `max_iter` is supported."""
lbfgs(self.oracle, self.x0, max_iter=15)
def test_tol(self):
"""Check if argument `tol` is supported."""
lbfgs(self.oracle, self.x0, tolerance=1e-6)