def run_trajectory(x_0, A, b=np.zeros((2, )), tag=""):
oracle = QuadraticOracle(A, b)
func = oracle.func
line_search_options = [{
'method': 'Constant',
'c': 0.1
}, {
'method': 'Constant',
'c': 1.0
}, {
'method': 'Armijo'
}, {
'method': 'Wolfe'
}]
Path("pics/" + tag).mkdir(parents=True, exist_ok=True)
for options in line_search_options:
x_opt, message, history = gradient_descent(oracle,
x_0,
trace=True,
line_search_options=options)
plt.figure()
plot_levels(func)
plot_trajectory(func, history['x'])
filename = "pics/{2}/{0}{1}.png".format(
options['method'],
"_" + str(options['c']) if 'c' in options else "", tag)
print("{}: {}".format(options['method'], len(history['x'])))
plt.savefig(filename)
class TestBestLineSearch(unittest.TestCase):
# Define a simple quadratic function for testing
A = np.array([[1, 0], [0, 2]])
b = np.array([1, 6])
x0 = np.array([0, 0])
# no need for `extra` for this simple function
oracle = QuadraticOracle(A, b)
def test_line_search(self):
ls_tool = LineSearchTool(method='Best')
x_k = np.array([2.0, 2.0])
d_k = np.array([-1.0, 1.0])
alpha_test = ls_tool.line_search(self.oracle, x_k, d_k)
alpha_real = 1.0
self.assertAlmostEqual(alpha_real, alpha_test)
x_k = np.array([2.0, 2.0])
d_k = np.array([-1.0, 0.0])
alpha_test = ls_tool.line_search(self.oracle, x_k, d_k)
alpha_real = 1.0
self.assertAlmostEqual(alpha_real, alpha_test)
x_k = np.array([10.0, 10.0])
d_k = np.array([-1.0, -1.0])
alpha_test = ls_tool.line_search(self.oracle, x_k, d_k)
alpha_real = 7.666666666666667
self.assertAlmostEqual(alpha_real, alpha_test)
class TestHFN(unittest.TestCase):
# Define a simple quadratic function for testing
A = np.array([[1, 0], [0, 2]])
b = np.array([1, 6])
f_star = -9.5
x0 = np.array([0, 0])
# no need for `extra` for this simple function
oracle = QuadraticOracle(A, b)
# For this func |nabla f(x)| < tol ensures |f(x) - f(x^*)| < tol^2
def test_default(self):
"""Check if everything works correctly with default parameters."""
with Capturing() as output:
x_min, message, _ = hessian_free_newton(self.oracle, self.x0)
assert_equal(message, 'success')
self.assertTrue(len(output) == 0, 'You should not print anything by default.')
def test_tolerance(self):
"""Check if argument `tolerance` is supported."""
hessian_free_newton(self.oracle, self.x0, tolerance=1e-5)
def test_max_iter(self):
"""Check if argument `max_iter` is supported."""
hessian_free_newton(self.oracle, self.x0, max_iter=15)
def test_line_search_options(self):
"""Check if argument `line_search_options` is supported."""
hessian_free_newton(self.oracle, self.x0, line_search_options={'method': 'Wolfe', 'c1': 1e-4, 'c2': 0.9})
def test_display(self):
"""Check if something is printed when `display` is True."""
with Capturing() as output:
hessian_free_newton(self.oracle, self.x0, display=True)
self.assertTrue(len(output) > 0, 'You should print the progress when `display` is True.')
def test_quality(self):
x_min, message, _ = hessian_free_newton(self.oracle, self.x0, tolerance=1e-5)
f_min = self.oracle.func(x_min)
g_k_norm_sqr = norm(self.A.dot(x_min) - self.b, 2)**2
g_0_norm_sqr = norm(self.A.dot(self.x0) - self.b, 2)**2
self.assertLessEqual(g_k_norm_sqr, 1e-5 * g_0_norm_sqr)
self.assertLessEqual(abs(f_min - self.f_star), 1e-5 * g_0_norm_sqr)
def test_history(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = hessian_free_newton(self.oracle, x0, trace=True, line_search_options={'method': 'Constant', 'c': 1.0},
tolerance=1e-6)
func_steps = [25.635, -9.5]
grad_norm_steps = [11.629703349613008, 0.0]
time_steps = [0.0] * 2 # Dummy values
x_steps = [np.array([-1.3, -2.7]),
np.array([1., 3.])]
true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps)
check_equal_histories(history, true_history)
def experiment_2():
np.random.seed(31415)
kappas = np.linspace(1., 1000., 50)
plt.figure()
colors = ['red', 'green', 'blue', 'orange']
labels = ['n = 2', 'n = 10', 'n = 100', 'n = 1000']
repeat_times = 100
runs = pd.DataFrame()
for i, n in enumerate([2, 10, 100, 1000]):
for j in range(repeat_times):
iter_num_arr = []
for kappa in kappas:
D = np.random.uniform(1, kappa, n)
D[0] = 1
D[-1] = kappa
A = diags(D)
b = np.random.uniform(-1, 1, (n, ))
x_0 = np.zeros((n, ))
oracle = QuadraticOracle(A, b)
x_opt, message, history = gradient_descent(oracle,
x_0,
trace=True,
max_iter=10**6)
iter_num = len(history['func'])
iter_num_arr.append(iter_num)
runs['run_%s' % j] = iter_num_arr
means = runs.mean(axis=1)
stds = runs.std(axis=1)
plt.plot(kappas, means, color=colors[i], label=labels[i], alpha=1.0)
plt.fill_between(kappas,
means - stds,
means + stds,
color=colors[i],
alpha=0.3)
plt.plot(kappas,
means - stds,
color=colors[i],
alpha=0.7,
linewidth=0.1)
plt.plot(kappas,
means + stds,
color=colors[i],
alpha=0.7,
linewidth=0.1)
plt.legend()
plt.grid()
plt.xlabel('Число обуcловленности')
plt.ylabel('Число итераций до сходимости')
plt.savefig("pics/iterations_vs_condition_number.pdf")
class TestHessVec(unittest.TestCase):
# Define a simple quadratic function for testing
A = np.array([[1, 0], [0, 2]])
b = np.array([1, 6])
x0 = np.array([0, 0])
# no need for `extra` for this simple function
oracle = QuadraticOracle(A, b)
def test_hess_vec(self):
# f(x, y) = x^3 + y^2
func = lambda x: x[0] ** 3 + x[1] ** 2
x = np.array([2.0, 3.0])
v = np.array([1.0, 0.1])
hess_vec_test = hess_vec_finite_diff(func, x, v, eps=1e-5)
hess_vec_real = np.array([12, 0.2])
assert_array_almost_equal(hess_vec_real, hess_vec_test, decimal=3)
v = np.array([1.0, -0.1])
hess_vec_test = hess_vec_finite_diff(func, x, v, eps=1e-5)
hess_vec_real = np.array([12, -0.2])
assert_array_almost_equal(hess_vec_real, hess_vec_test, decimal=3)
class TestLBFGS(unittest.TestCase):
# Define a simple quadratic function for testing
A = np.array([[1, 0], [0, 2]])
b = np.array([1, 6])
oracle = QuadraticOracle(A, b)
f_star = -9.5
x0 = np.array([0, 0])
# For this func |nabla f(x)| < tol ensures |f(x) - f(x^*)| < tol^2
def check_equal_histories(self, history1, history2, atol=1e-3):
if history1 is None or history2 is None:
self.assertEqual(history1, history2)
return
self.assertTrue('func' in history1 and 'func' in history2)
self.assertTrue(np.allclose(history1['func'], history2['func'], atol=atol))
self.assertTrue('grad_norm' in history1 and 'grad_norm' in history2)
self.assertTrue(np.allclose(history1['grad_norm'], history2['grad_norm'], atol=atol))
self.assertTrue('time' in history1 and 'time' in history2)
self.assertTrue(len(history1['time']), len(history2['time']))
self.assertTrue('x' in history1, 'x' in history2)
if 'x' in history1:
self.assertTrue(np.allclose(history1['x'], history2['x'], atol=atol))
def test_default(self):
"""Check if everything works correctly with default parameters."""
with Capturing() as output:
x_min, message, _ = lbfgs(self.oracle, self.x0)
assert_equal(message, 'success')
self.assertEqual(len(output), 0, 'You should not print anything by default.')
def test_tol(self):
"""Check if argument `tol` is supported."""
lbfgs(self.oracle, self.x0, tolerance=1e-6)
def test_max_iter(self):
"""Check if argument `max_iter` is supported."""
lbfgs(self.oracle, self.x0, max_iter=15)
def test_memory_size(self):
"""Check if argument `memory_size` is supported."""
lbfgs(self.oracle, self.x0, memory_size=1)
def test_line_search_options(self):
"""Check if argument `line_search_options` is supported."""
lbfgs(self.oracle, self.x0, line_search_options={'method': 'Wolfe', 'c1': 1e-4, 'c2': 0.9})
def test_disp(self):
"""Check if something is printed when `disp` is True."""
with Capturing() as output:
lbfgs(self.oracle, self.x0, display=True)
self.assertTrue(len(output) > 0, 'You should print the progress when `display` is True.')
def test_trace(self):
"""Check if the history is returned correctly when `trace` is True."""
x_min, message, hist = lbfgs(self.oracle, self.x0, trace=True)
self.assertEqual(len(hist['grad_norm']), len(hist['func']))
self.assertEqual(len(hist['time']), len(hist['func']))
self.assertEqual(len(hist['x']), len(hist['func']))
x_min, message, hist = lbfgs(self.oracle, self.x0, trace=False)
self.assertTrue(hist is None)
def test_quality(self):
x_min, message, _ = lbfgs(self.oracle, self.x0, tolerance=1e-10)
f_min = self.oracle.func(x_min)
g_k_norm = norm(self.A.dot(x_min) - self.b, 2)
self.assertLessEqual(abs(g_k_norm), 1e-3)
self.assertLessEqual(abs(f_min - self.f_star), 1e-3)
def test_histories(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = lbfgs(self.oracle, x0, trace=True, memory_size=10, line_search_options={'method': 'Constant', 'c': 1.0},
tolerance=1e-6)
func_steps = [25.635000000000005,
22.99,
-9.3476294733722725,
-9.4641732176886055,
-9.5]
grad_norm_steps = [11.629703349613008,
11.4,
0.55751193505619512,
0.26830541958992876,
0.0]
time_steps = [0.0] * 5 # Dummy values
x_steps = [np.array([-1.3, -2.7]),
np.array([1.0, 8.7]),
np.array([0.45349973, 3.05512941]),
np.array([0.73294321, 3.01292737]),
np.array([0.99999642, 2.99998814])]
true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps)
self.check_equal_histories(history, true_history)
self.assertEqual(message, "success")
assert_array_almost_equal(x_min, np.array([1.0, 3.0]), decimal=3)
class TestBonus(unittest.TestCase):
# Define a simple quadratic function for testing
A = np.array([[1, 0], [0, 2]])
b = np.array([1, 6])
x0 = np.array([0, 0])
# no need for `extra` for this simple function
oracle = QuadraticOracle(A, b)
def check_equal_histories(self, history1, history2, atol=1e-3):
if history1 is None or history2 is None:
self.assertEqual(history1, history2)
return
self.assertTrue('func' in history1 and 'func' in history2)
self.assertTrue(np.allclose(history1['func'], history2['func'], atol=atol))
self.assertTrue('grad_norm' in history1 and 'grad_norm' in history2)
self.assertTrue(np.allclose(history1['grad_norm'], history2['grad_norm'], atol=atol))
self.assertTrue('time' in history1 and 'time' in history2)
self.assertTrue(len(history1['time']), len(history2['time']))
self.assertTrue('x' in history1, 'x' in history2)
if 'x' in history1:
self.assertTrue(np.allclose(history1['x'], history2['x'], atol=atol))
def test_line_search(self):
ls_tool = LineSearchTool(method='Best')
x_k = np.array([2.0, 2.0])
d_k = np.array([-1.0, 1.0])
alpha_test = ls_tool.line_search(self.oracle, x_k, d_k)
alpha_real = 1.0
self.assertAlmostEqual(alpha_real, alpha_test)
x_k = np.array([2.0, 2.0])
d_k = np.array([-1.0, 0.0])
alpha_test = ls_tool.line_search(self.oracle, x_k, d_k)
alpha_real = 1.0
self.assertAlmostEqual(alpha_real, alpha_test)
x_k = np.array([10.0, 10.0])
d_k = np.array([-1.0, -1.0])
alpha_test = ls_tool.line_search(self.oracle, x_k, d_k)
alpha_real = 7.666666666666667
self.assertAlmostEqual(alpha_real, alpha_test)
def test_lbfgs_history(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = lbfgs(self.oracle, x0, trace=True, memory_size=10, line_search_options={'method': 'Best'}, tolerance=1e-6)
func_steps = [25.635000000000005,
-8.8519395950378961,
-9.5]
grad_norm_steps = [11.629703349613008,
1.1497712070693149,
0]
time_steps = [0.0] * 3 # Dummy values
x_steps = [np.array([-1.3, -2.7]),
np.array([-0.12706157, 3.11369481]),
np.array([1., 3.])]
true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps)
self.check_equal_histories(history, true_history)
self.assertEqual(message, "success")
assert_array_almost_equal(x_min, np.array([1.0, 3.0]), decimal=3)
def test_hfn_history(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = hessian_free_newton(self.oracle, x0, trace=True, line_search_options={'method': 'Best'}, tolerance=1e-6)
func_steps = [25.635, -9.5]
grad_norm_steps = [11.629703349613008, 0.0]
time_steps = [0.0] * 2 # Dummy values
x_steps = [np.array([-1.3, -2.7]),
np.array([1., 3.])]
true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps)
self.check_equal_histories(history, true_history)
self.assertEqual(message, "success")
assert_array_almost_equal(x_min, np.array([1.0, 3.0]), decimal=3)
class TestHFN(unittest.TestCase):
# Define a simple quadratic function for testing
A = np.array([[1, 0], [0, 2]])
b = np.array([1, 6])
f_star = -9.5
x0 = np.array([0, 0])
# no need for `extra` for this simple function
oracle = QuadraticOracle(A, b)
# For this func |nabla f(x)| < tol ensures |f(x) - f(x^*)| < tol^2
def check_equal_histories(self, history1, history2, atol=1e-3):
if history1 is None or history2 is None:
self.assertEqual(history1, history2)
return
self.assertTrue('func' in history1 and 'func' in history2)
self.assertTrue(np.allclose(history1['func'], history2['func'], atol=atol))
self.assertTrue('grad_norm' in history1 and 'grad_norm' in history2)
self.assertTrue(np.allclose(history1['grad_norm'], history2['grad_norm'], atol=atol))
self.assertTrue('time' in history1 and 'time' in history2)
self.assertTrue(len(history1['time']), len(history2['time']))
self.assertTrue('x' in history1, 'x' in history2)
if 'x' in history1:
self.assertTrue(np.allclose(history1['x'], history2['x'], atol=atol))
def test_default(self):
"""Check if everything works correctly with default parameters."""
with Capturing() as output:
x_min, message, _ = hessian_free_newton(self.oracle, self.x0)
assert_equal(message, 'success')
self.assertTrue(len(output) == 0, 'You should not print anything by default.')
def test_tol(self):
"""Check if argument `tol` is supported."""
hessian_free_newton(self.oracle, self.x0, tolerance=1e-6)
def test_max_iter(self):
"""Check if argument `max_iter` is supported."""
hessian_free_newton(self.oracle, self.x0, max_iter=15)
def test_line_search_options(self):
"""Check if argument `line_search_options` is supported."""
hessian_free_newton(self.oracle, self.x0, line_search_options={'method': 'Wolfe', 'c1': 1e-4, 'c2': 0.9})
def test_disp(self):
"""Check if something is printed when `disp` is True."""
with Capturing() as output:
hessian_free_newton(self.oracle, self.x0, display=True)
self.assertTrue(len(output) > 0, 'You should print the progress when `display` is True.')
def test_trace(self):
"""Check if the history is returned correctly when `trace` is True."""
x_min, message, hist = hessian_free_newton(self.oracle, self.x0, trace=True)
self.assertEqual(len(hist['grad_norm']), len(hist['func']))
self.assertEqual(len(hist['time']), len(hist['func']))
self.assertEqual(len(hist['x']), len(hist['func']))
x_min, message, hist = hessian_free_newton(self.oracle, self.x0, trace=False)
self.assertTrue(hist is None)
def test_quality(self):
x_min, message, _ = hessian_free_newton(self.oracle, self.x0, tolerance=1e-10)
f_min = self.oracle.func(x_min)
g_k_norm = norm(self.A.dot(x_min) - self.b, 2)
self.assertLessEqual(abs(g_k_norm), 1e-3)
self.assertLessEqual(abs(f_min - self.f_star), 1e-3)
def test_histories(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = hessian_free_newton(self.oracle, x0, trace=True, line_search_options={'method': 'Constant', 'c': 1.0},
tolerance=1e-6)
func_steps = [25.635, -9.5]
grad_norm_steps = [11.629703349613008, 0.0]
time_steps = [0.0] * 2 # Dummy values
x_steps = [np.array([-1.3, -2.7]),
np.array([1., 3.])]
true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps)
self.check_equal_histories(history, true_history)
self.assertEqual(message, "success")
assert_array_almost_equal(x_min, np.array([1.0, 3.0]), decimal=3)
class TestLBFGS(unittest.TestCase):
# Define a simple quadratic function for testing
A = np.array([[1, 0], [0, 2]])
b = np.array([1, 6])
oracle = QuadraticOracle(A, b)
f_star = -9.5
x0 = np.array([0, 0])
# For this func |nabla f(x)| < tol ensures |f(x) - f(x^*)| < tol^2
def test_default(self):
"""Check if everything works correctly with default parameters."""
opt = LBFGS(self.oracle, self.x0)
opt.run()
def test_tolerance(self):
"""Check if argument `tolerance` is supported."""
LBFGS(self.oracle, self.x0, tolerance=1e-5)
def test_max_iter(self):
"""Check if argument `max_iter` is supported."""
opt = LBFGS(self.oracle, self.x0)
opt.run(max_iter=0)
def test_memory_size(self):
"""Check if argument `memory_size` is supported."""
LBFGS(self.oracle, self.x0, memory_size=1)
def test_line_search_options(self):
"""Check if argument `line_search_options` is supported."""
LBFGS(self.oracle,
self.x0,
line_search_options={
'method': 'Wolfe',
'c1': 1e-4,
'c2': 0.9
})
def test_quality(self):
opt = LBFGS(self.oracle, self.x0, tolerance=1e-5)
opt.run(5)
x_min = opt.hist['x_star']
# x_min, message, _ = LBFGS(self.oracle, self.x0, tolerance=1e-5)
f_min = self.oracle.func(x_min)
g_k_norm_sqr = norm(self.A.dot(x_min) - self.b, 2)**2
g_0_norm_sqr = norm(self.A.dot(self.x0) - self.b, 2)**2
self.assertLessEqual(g_k_norm_sqr, 1e-5 * g_0_norm_sqr)
self.assertLessEqual(abs(f_min - self.f_star), 1e-5 * g_0_norm_sqr)
def test_history(self):
x0 = -np.array([1.3, 2.7])
opt = LBFGS(self.oracle,
x0,
memory_size=10,
line_search_options={
'method': 'Constant',
'c': 1.0
},
tolerance=1e-6)
opt.run(10)
x_min = opt.hist['x_star']
func_steps = [
25.635000000000005, 22.99, -9.3476294733722725,
-9.4641732176886055, -9.5
]
grad_norm_steps = [
11.629703349613008, 11.4, 0.55751193505619512, 0.26830541958992876,
0.0
]
time_steps = [0.0] * 5 # Dummy values
x_steps = [
np.array([-1.3, -2.7]),
np.array([1.0, 8.7]),
np.array([0.45349973, 3.05512941]),
np.array([0.73294321, 3.01292737]),
np.array([0.99999642, 2.99998814])
]
true_history = dict(grad_norm=grad_norm_steps,
time=time_steps,
x=x_steps,
func=func_steps)
check_equal_histories(opt.hist, true_history)
class TestBFGS(unittest.TestCase):
# Define a simple quadratic function for testing
A = np.array([[1, 0], [0, 2]], dtype=np.float64)
b = np.array([1, 6], dtype=np.float64)
oracle = QuadraticOracle(A, b)
f_star = -9.5
x0 = np.array([0, 0], dtype=np.float64)
# For this func |nabla f(x)| < tol ensures |f(x) - f(x^*)| < tol^2
def test_default(self):
"""Check if everything works correctly with default parameters."""
opt = BFGS(self.oracle, self.x0)
opt.run()
def test_tolerance(self):
"""Check if argument `tolerance` is supported."""
BFGS(self.oracle, self.x0, tolerance=1e-5)
def test_max_iter(self):
"""Check if argument `max_iter` is supported."""
opt = BFGS(self.oracle, self.x0)
opt.run(max_iter=0)
def test_line_search_options(self):
"""Check if argument `line_search_options` is supported."""
BFGS(self.oracle,
self.x0,
line_search_options={
'method': 'Wolfe',
'c1': 1e-4,
'c2': 0.9
})
def test_quality(self):
opt = BFGS(self.oracle, self.x0, tolerance=1e-5)
opt.run(5)
x_min = opt.hist['x_star']
# x_min, message, _ = LBFGS(self.oracle, self.x0, tolerance=1e-5)
f_min = self.oracle.func(x_min)
g_k_norm_sqr = norm(self.A.dot(x_min) - self.b, 2)**2
g_0_norm_sqr = norm(self.A.dot(self.x0) - self.b, 2)**2
self.assertLessEqual(g_k_norm_sqr, 1e-5 * g_0_norm_sqr)
self.assertLessEqual(abs(f_min - self.f_star), 1e-5 * g_0_norm_sqr)
def test_history(self):
x0 = -np.array([1.3, 2.7])
opt = BFGS(self.oracle,
x0,
line_search_options={
'method': 'Constant',
'c': 1.0
},
tolerance=1e-6)
opt.run(10)
x_min = opt.hist['x_star']
func_steps = [25.635000000000005, 22.99, -9.48707349065929, -9.5]
grad_norm_steps = [11.629703349613008, 11.4, 0.22738961577617722, 0.0]
time_steps = [0.0] * 4 # Dummy values
x_steps = [
np.array([-1.3, -2.7]),
np.array([1.0, 8.7]),
np.array([1.0, 2.88630519]),
np.array([1., 3.])
]
true_history = dict(grad_norm=grad_norm_steps,
time=time_steps,
x=x_steps,
func=func_steps)
check_equal_histories(opt.hist, true_history)
class TestLBFGS(unittest.TestCase):
# Define a simple quadratic function for testing
A = np.array([[1, 0], [0, 2]])
b = np.array([1, 6])
oracle = QuadraticOracle(A, b)
f_star = -9.5
x0 = np.array([0, 0])
# For this func |nabla f(x)| < tol ensures |f(x) - f(x^*)| < tol^2
def test_default(self):
"""Check if everything works correctly with default parameters."""
with Capturing() as output:
x_min, message, _ = lbfgs(self.oracle, self.x0)
assert_equal(message, 'success')
self.assertEqual(len(output), 0,
'You should not print anything by default.')
def test_tolerance(self):
"""Check if argument `tolerance` is supported."""
lbfgs(self.oracle, self.x0, tolerance=1e-5)
def test_max_iter(self):
"""Check if argument `max_iter` is supported."""
lbfgs(self.oracle, self.x0, max_iter=15)
def test_memory_size(self):
"""Check if argument `memory_size` is supported."""
lbfgs(self.oracle, self.x0, memory_size=1)
def test_line_search_options(self):
"""Check if argument `line_search_options` is supported."""
lbfgs(self.oracle,
self.x0,
line_search_options={
'method': 'Wolfe',
'c1': 1e-4,
'c2': 0.9
})
def test_display(self):
"""Check if something is printed when `display` is True."""
with Capturing() as output:
lbfgs(self.oracle, self.x0, display=True)
self.assertTrue(
len(output) > 0,
'You should print the progress when `display` is True.')
def test_quality(self):
x_min, message, _ = lbfgs(self.oracle, self.x0, tolerance=1e-5)
f_min = self.oracle.func(x_min)
g_k_norm_sqr = norm(self.A.dot(x_min) - self.b, 2)**2
g_0_norm_sqr = norm(self.A.dot(self.x0) - self.b, 2)**2
self.assertLessEqual(g_k_norm_sqr, 1e-5 * g_0_norm_sqr)
self.assertLessEqual(abs(f_min - self.f_star), 1e-5 * g_0_norm_sqr)
def test_history(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = lbfgs(self.oracle,
x0,
trace=True,
memory_size=10,
line_search_options={
'method': 'Constant',
'c': 1.0
},
tolerance=1e-6)
func_steps = [
25.635000000000005, 22.99, -9.3476294733722725,
-9.4641732176886055, -9.5
]
grad_norm_steps = [
11.629703349613008, 11.4, 0.55751193505619512, 0.26830541958992876,
0.0
]
time_steps = [0.0] * 5 # Dummy values
x_steps = [
np.array([-1.3, -2.7]),
np.array([1.0, 8.7]),
np.array([0.45349973, 3.05512941]),
np.array([0.73294321, 3.01292737]),
np.array([0.99999642, 2.99998814])
]
true_history = dict(grad_norm=grad_norm_steps,
time=time_steps,
x=x_steps,
func=func_steps)
check_equal_histories(history, true_history)
@unittest.skipUnless(test_bonus, 'Skipping bonus test...')
def test_history_best(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = lbfgs(self.oracle,
x0,
trace=True,
memory_size=10,
line_search_options={'method': 'Best'},
tolerance=1e-6)
func_steps = [25.635000000000005, -8.8519395950378961, -9.5]
grad_norm_steps = [11.629703349613008, 1.1497712070693149, 0]
time_steps = [0.0] * 3 # Dummy values
x_steps = [
np.array([-1.3, -2.7]),
np.array([-0.12706157, 3.11369481]),
np.array([1., 3.])
]
true_history = dict(grad_norm=grad_norm_steps,
time=time_steps,
x=x_steps,
func=func_steps)
check_equal_histories(history, true_history)
def experiment_5_and_6(algo='gd'):
np.random.seed(31415)
m, n = 2000, 1000
A = np.random.randn(m, n)
b = np.sign(np.random.randn(m))
regcoef = 1 / m
logreg_oracle = create_log_reg_oracle(A,
b,
regcoef,
oracle_type='optimized')
line_search_options = [
{
'method': 'Constant',
'c': 1.0
},
{
'method': 'Constant',
'c': 0.95
},
{
'method': 'Constant',
'c': 0.9
},
{
'method': 'Constant',
'c': 0.85
},
{
'method': 'Armijo',
'c1': 1e-8
},
{
'method': 'Armijo',
'c1': 1e-6
},
{
'method': 'Armijo',
'c1': 1e-4
},
{
'method': 'Armijo',
'c1': 1e-1
},
{
'method': 'Wolfe',
'c2': 1.5
},
{
'method': 'Wolfe',
'c2': 0.9
},
{
'method': 'Wolfe',
'c2': 0.1
},
{
'method': 'Wolfe',
'c2': 0.01
},
]
colors = ['#e66101', '#fdb863', '#b2abd2', '#5e3c99']
styles = {
'Constant': {
'linestyle': '--',
'dashes': (2, 5),
'linewidth': 2
},
'Armijo': {
'linestyle': '--',
'dashes': (5, 2)
},
'Wolfe': {
'linestyle': 'solid'
},
}
x_0_list = [None] * 3
x_0_list[0] = np.zeros((n, ))
x_0_list[1] = np.random.uniform(-1, 1, (n, ))
x_0_list[2] = np.ones((n, ))
for k, x_0 in enumerate(x_0_list):
plt.figure(figsize=(12, 9))
for i, options in tqdm(enumerate(line_search_options)):
if algo == 'GD':
x_opt, message, history = gradient_descent(
logreg_oracle,
x_0,
trace=True,
line_search_options=options)
else:
x_opt, message, history = newton(logreg_oracle,
x_0,
trace=True,
line_search_options=options)
args = list(options.keys()) + list(options.values())
label = "{} ({}={}, {}={})".format(algo, args[0], args[2], args[1],
args[3])
values = 2 * np.log(
(history['grad_norm'] / history['grad_norm'][0]))
method = args[2]
plt.plot(values + np.random.randn(values.size) * 0.05,
color=colors[i % len(colors)],
label=label,
alpha=0.7,
**styles[method])
plt.xlabel('Номер итерации')
plt.ylabel('Логарифм относительного квадрата нормы градиента')
plt.legend(loc='upper right')
plt.grid()
Path("pics/logreg_{}_linear_search_strategies".format(algo)).mkdir(
parents=True, exist_ok=True)
plt.savefig(
"pics/logreg_{}_linear_search_strategies/x_0_{}.png".format(
algo, k))
np.random.seed(31415)
n = 2000
C = ortho_group.rvs(n)
A = C.T @ np.diag(np.random.uniform(1, 20, (n, ))) @ C
b = np.random.randn(n)
x_0 = np.zeros((n, ))
quadratic_oracle = QuadraticOracle(A, b)
x_opt = np.linalg.solve(A, b)
f_opt = quadratic_oracle.func(x_opt)
line_search_options = [
{
'method': 'Constant',
'c': 0.09
},
{
'method': 'Constant',
'c': 0.085
},
{
'method': 'Constant',
'c': 0.08
},
{
'method': 'Constant',
'c': 0.075
},
{
'method': 'Armijo',
'c1': 1e-10
},
{
'method': 'Armijo',
'c1': 1e-7
},
{
'method': 'Armijo',
'c1': 1e-4
},
{
'method': 'Armijo',
'c1': 1e-1
},
{
'method': 'Wolfe',
'c2': 1.5
},
{
'method': 'Wolfe',
'c2': 0.9
},
{
'method': 'Wolfe',
'c2': 0.1
},
{
'method': 'Wolfe',
'c2': 0.01
},
]
x_0_list = [None] * 3
x_0_list[0] = np.zeros((n, ))
x_0_list[1] = np.random.uniform(-1, 1, (n, ))
x_0_list[2] = x_opt + np.random.randn(n, ) * 0.2
for k, x_0 in enumerate(x_0_list):
plt.figure(figsize=(12, 9))
for i, options in tqdm(enumerate(line_search_options)):
if algo == 'GD':
x_opt, message, history = gradient_descent(
quadratic_oracle,
x_0,
trace=True,
line_search_options=options)
else:
x_opt, message, history = newton(quadratic_oracle,
x_0,
trace=True,
line_search_options=options)
args = list(options.keys()) + list(options.values())
label = "{} ({}={}, {}={})".format(algo, args[0], args[2], args[1],
args[3])
values = np.log(np.abs((history['func'] - f_opt) / f_opt) + 1e-16)
method = args[2]
plt.plot(values + np.random.randn(values.size) * 0.05,
color=colors[i % len(colors)],
label=label,
alpha=0.7,
**styles[method])
plt.xlabel('Номер итерации')
plt.ylabel('Логарифм относительной невязки')
plt.legend(loc='upper right')
plt.grid()
Path("pics/quadratic_{}_linear_search_strategies".format(algo)).mkdir(
parents=True, exist_ok=True)
plt.savefig(
"pics/quadratic_{}_linear_search_strategies/x_0_{}.png".format(
algo, k))
class TestNCG(unittest.TestCase):
# Define a simple quadratic function for testing
A = np.array([[1, 0], [0, 2]])
b = np.array([1, 6])
oracle = QuadraticOracle(A, b)
f_star = -9.5
x0 = np.array([0, 0])
# For this func |nabla f(x)| < tol ensures |f(x) - f(x^*)| < tol^2
def test_default(self):
"""Check if everything works correctly with default parameters."""
with Capturing() as output:
x_min, message, _ = nonlinear_conjugate_gradients(self.oracle, self.x0)
assert_equal(message, 'success')
self.assertEqual(len(output), 0, 'You should not print anything by default.')
def test_tolerance(self):
"""Check if argument `tolerance` is supported."""
nonlinear_conjugate_gradients(self.oracle, self.x0, tolerance=1e-5)
def test_max_iter(self):
"""Check if argument `max_iter` is supported."""
nonlinear_conjugate_gradients(self.oracle, self.x0, max_iter=15)
def test_line_search_options(self):
"""Check if argument `line_search_options` is supported."""
nonlinear_conjugate_gradients(self.oracle, self.x0, line_search_options={'method': 'Wolfe', 'c1': 0.01, 'c2': 0.01})
def test_display(self):
"""Check if something is printed when `display` is True."""
with Capturing() as output:
nonlinear_conjugate_gradients(self.oracle, self.x0, display=True)
self.assertTrue(len(output) > 0, 'You should print the progress when `display` is True.')
def test_quality(self):
x_min, message, _ = nonlinear_conjugate_gradients(self.oracle, self.x0, tolerance=1e-5)
f_min = self.oracle.func(x_min)
g_k_norm_sqr = norm(self.A.dot(x_min) - self.b, 2)**2
g_0_norm_sqr = norm(self.A.dot(self.x0) - self.b, 2)**2
self.assertLessEqual(g_k_norm_sqr, 1e-5 * g_0_norm_sqr)
self.assertLessEqual(abs(f_min - self.f_star), 1e-5 * g_0_norm_sqr)
def test_history(self):
x0 = -np.array([1.3, 2.7])
x_min, message, history = nonlinear_conjugate_gradients(self.oracle, x0, trace=True,
line_search_options={'method': 'Constant', 'c': 0.6},
tolerance=1e-3)
func_steps = [25.635000000000005,
-7.777199999999997,
-7.8463333368073584,
-9.439531896665148,
-9.41251357036308,
-9.4977531453388693]
grad_norm_steps = [11.629703349613008,
2.4586174977006889,
2.5711348318091276,
0.48633577943739081,
0.58855118960609565,
0.092585266951596912]
time_steps = [0.0] * 6 # Dummy values
x_steps = [np.array([-1.3, -2.7]),
np.array([0.08, 4.14]),
np.array([0.93729171, 4.28518501]),
np.array([1.07314317, 2.75959796]),
np.array([1.05960886, 2.7072376]),
np.array([1.02038104, 3.04515707])]
true_history = dict(grad_norm=grad_norm_steps, time=time_steps, x=x_steps, func=func_steps)
check_equal_histories(history, true_history)