def test_QuadraticOracle():
# Quadratic function:
# f(x) = 1/2 x^T x - [1, 2, 3]^T x
A = np.eye(3)
b = np.array([1, 2, 3])
quadratic = oracles.QuadraticOracle(A, b)
# Check at point x = [0, 0, 0]
x = np.zeros(3)
assert_almost_equal(quadratic.func(x), 0.0)
ok_(np.allclose(quadratic.grad(x), -b))
ok_(np.allclose(quadratic.hess(x), A))
ok_(isinstance(quadratic.grad(x), np.ndarray))
ok_(isinstance(quadratic.hess(x), np.ndarray))
# Check at point x = [1, 1, 1]
x = np.ones(3)
assert_almost_equal(quadratic.func(x), -4.5)
ok_(np.allclose(quadratic.grad(x), x - b))
ok_(np.allclose(quadratic.hess(x), A))
ok_(isinstance(quadratic.grad(x), np.ndarray))
ok_(isinstance(quadratic.hess(x), np.ndarray))
# Check func_direction and grad_direction oracles at
# x = [1, 1, 1], d = [-1, -1, -1], alpha = 0.5 and 1.0
x = np.ones(3)
d = -np.ones(3)
assert_almost_equal(quadratic.func_directional(x, d, alpha=0.5), -2.625)
assert_almost_equal(quadratic.grad_directional(x, d, alpha=0.5), 4.5)
assert_almost_equal(quadratic.func_directional(x, d, alpha=1.0), 0.0)
assert_almost_equal(quadratic.grad_directional(x, d, alpha=1.0), 6.0)
def first_experiment():
A_good = np.array([[1, 0.2], [0.2, 1.2]])
A_bad = np.array([[0.1, 0.1], [0.1, 1]])
for i, A in enumerate([A_good, A_bad]):
cond = np.linalg.cond(A)
oracle = oracles.QuadraticOracle(A, np.zeros(2))
np.random.seed(i * 42)
x_0_s = [np.random.uniform(-5, 5, size=2) for _ in range(3)]
for j in range(3):
x_0 = x_0_s[j]
for method in ['Wolfe', 'Armijo', 'Constant']:
_, _, history = optimization.gradient_descent(
oracle,
x_0,
line_search_options={
'method': method,
'alpha_0': 100
},
trace=True)
plot_trajectory_2d.plt.figure()
plot_trajectory_2d.plot_levels(oracle.func)
name = 'experiment_1/{}-{}-{}'.format(round(cond, 3), method,
j)
plot_trajectory_2d.plot_trajectory(oracle.func,
history['x'],
save=name)
print('cond = {}, j = {}, method = {}, steps = {}'.format(
round(cond, 3), j, method, len(history['x'])))
def get_iteration_num_for_fixed_n(condition_num_values,
n,
x_init,
method_name,
b_min=-1,
b_max=1):
"""
Returns an array of iterations number for gradient descent
for each value in condition_num_values for given n
:param condition_num_values: array of condition numbers
:param n: number of features
:param x_init: initial point for gradient descent
:param b_min: min value in b
:param b_max: max value in b
:param method_name: 'Wolfe', 'Armijo' or 'Constant'
:return: an array with number of iterations for each value in condition_num_values
"""
iteration_num_values = []
for condition_num in condition_num_values:
A, b = generate_task(condition_num, n, b_min, b_max)
oracle = oracles.QuadraticOracle(A, b)
[x_star, _, history] = optimization.gradient_descent(oracle, x_init, \
line_search_options={'method': method_name, 'c': 0.001}, \
trace=True)
# print(x_star)
# print(history['grad_norm'])
# print("====================================")
iteration_num_values.append(len(history['grad_norm']))
return iteration_num_values
def test_hess_finite_diff_1():
# Quadratic function.
A = np.eye(3)
b = np.array([1, 2, 3])
quadratic = oracles.QuadraticOracle(A, b)
H = oracles.hess_finite_diff(quadratic.func, np.zeros(3))
ok_(isinstance(H, np.ndarray))
ok_(np.allclose(H, A))
def test_grad_finite_diff_1():
# Quadratic function.
A = np.eye(3)
b = np.array([1, 2, 3])
quadratic = oracles.QuadraticOracle(A, b)
g = oracles.grad_finite_diff(quadratic.func, np.zeros(3))
ok_(isinstance(g, np.ndarray))
ok_(np.allclose(g, -b))
def second_experiment():
ns = [10, 100, 1000, 10000]
colors = ['g', 'r', 'b', 'y']
kappas = list(range(1, 1000, 100))
iterations = 10
T = {}
np.seterr(all='print')
t0 = datetime.now()
for n, color in zip(ns, colors):
T[n] = [[] for _ in range(iterations)]
for i in range(iterations):
for kappa in kappas:
np.random.seed(1000 * i + kappa)
diag = np.random.uniform(low=1, high=kappa, size=n)
diag[0], diag[-1] = 1, kappa
A = diags(diag)
b = np.random.uniform(low=1, high=kappa, size=n)
oracle = oracles.QuadraticOracle(A, b)
x_star, msg, history = optimization.gradient_descent(
oracle, np.zeros(n), trace=True)
if msg == 'success':
T[n][i].append(len(history['grad_norm']))
else:
T[n][i].append(10000)
plt.plot(kappas, T[n][i], ls='--', color=color, alpha=0.2)
plt.plot(kappas,
np.mean(T[n], axis=0),
color=color,
label='n = {}'.format(n))
print('n = {}, time = {}'.format(n, datetime.now() - t0))
plt.grid()
plt.legend()
plt.ylabel('iterations')
plt.xlabel(r'$\ae$')
plt.savefig('experiment_2/T(n, kappa)')
plt.ylabel('y')
plt.legend([
'Constant, iterations number = {}'.format(
len(history_constant['x']) - 1),
'Armije, iterations number = {}'.format(len(history_armijo['x']) - 1),
'Wolfe, iterations number = {}'.format(len(history_wolf['x']) - 1)
])
plt.savefig('./3.1/{}'.format(plot_name))
plt.show()
if __name__ == '__main__':
A = np.array([[1, 0], [0, 50]])
b = np.array([0, 0])
x_init = np.array([10.0, 4.0])
oracle = oracles.QuadraticOracle(A, b)
xrange = [-6, 20]
yrange = None
levels = [0, 16, 64, 128, 256]
save_comparison_plot(oracle,
x_init,
xrange,
yrange,
levels,
plot_name='plot1')
A = np.array([[1, 0], [0, 50]])
b = np.array([0, 0])
x_init = np.array([15.0, 0.0])
oracle = oracles.QuadraticOracle(A, b)
xrange = [-6, 20]
def get_quadratic(self):
# Quadratic function:
# f(x) = 1/2 x^T x - [1, 2, 3]^T x
A = np.eye(3)
b = np.array([1, 2, 3])
return oracles.QuadraticOracle(A, b)
x_values, y_values = zip(*history)
plt.plot(x_values,
y_values,
'-v',
linewidth=5.0,
ms=12.0,
alpha=1.0,
c='r',
label=label)
# Tries to adapt axis-ranges for the trajectory:
if fit_axis:
xmax, ymax = np.max(x_values), np.max(y_values)
COEF = 1.5
xrange = [-xmax * COEF, xmax * COEF]
yrange = [-ymax * COEF, ymax * COEF]
plt.xlim(xrange)
plt.ylim(yrange)
plt.show()
if __name__ == '__main__':
A = np.eye(2)
b = np.array([1, 2])
quadratic = oracles.QuadraticOracle(A, b)
plot_levels(quadratic.func)
x_star, msg, history = optimization.gradient_descent(quadratic,
np.array([3.0, 1.5]),
trace=True)
plot_trajectory(quadratic.func, history['x'])
def T(k):
quad = oracles.QuadraticOracle(Aa(k), b)
x_star, msg, history = optimization.gradient_descent(
quad, np.random.uniform(10, 30, n), trace=True)
return len(history['grad_norm'])
exps = [{
"A": np.array([[5, 1], [1, 1]]),
"b": np.zeros(2),
"st": "normal"
}, {
"A": np.array([[25, 2], [2, 1]]),
"b": np.array([0, 0]),
"st": "stratched"
}]
for ix, x_0 in enumerate([np.array([3, 4]), np.array([1, 25])]):
for ip, params in enumerate(exps):
for met in ["Wolfe", "Armijo", "Constant"]:
plt.clf()
qua1 = oracles.QuadraticOracle(params['A'], params['b'])
plot_trajectory_2d.plot_levels(qua1.func)
x_star, msg, history = optimization.gradient_descent(
qua1,
x_0,
trace=True,
line_search_options={
'method': met,
'c': 0.1
})
plot_trajectory_2d.plot_trajectory(qua1.func, history['x'])
plt.savefig("exp1/{0}-{1}-{2}.png".format(x_0, params['st'], met))
for e in exps:
print("{0} - {1}".format(e['st'], li.cond(e['A'])))
import oracles
from plot_trajectory_2d import plot_levels
from plot_trajectory_2d import plot_trajectory
import optimization
import numpy as np
import matplotlib.pyplot as plt
oracle = oracles.QuadraticOracle(np.array([[1.0, 2.0], [2.0, 5.0]]),
np.zeros(2))
[x_star, msg, history] = optimization.gradient_descent(oracle,
np.array([3.0, 1.5]),
trace=True)
plt.figure()
plot_levels(oracle.func)
plot_trajectory(oracle.func, history['x'])
plt.plot()
