def test_QuadraticOracle(): # Quadratic function: # f(x) = 1/2 x^T x - [1, 2, 3]^T x A = np.eye(3) b = np.array([1, 2, 3]) quadratic = oracles.QuadraticOracle(A, b) # Check at point x = [0, 0, 0] x = np.zeros(3) assert_almost_equal(quadratic.func(x), 0.0) ok_(np.allclose(quadratic.grad(x), -b)) ok_(np.allclose(quadratic.hess(x), A)) ok_(isinstance(quadratic.grad(x), np.ndarray)) ok_(isinstance(quadratic.hess(x), np.ndarray)) # Check at point x = [1, 1, 1] x = np.ones(3) assert_almost_equal(quadratic.func(x), -4.5) ok_(np.allclose(quadratic.grad(x), x - b)) ok_(np.allclose(quadratic.hess(x), A)) ok_(isinstance(quadratic.grad(x), np.ndarray)) ok_(isinstance(quadratic.hess(x), np.ndarray)) # Check func_direction and grad_direction oracles at # x = [1, 1, 1], d = [-1, -1, -1], alpha = 0.5 and 1.0 x = np.ones(3) d = -np.ones(3) assert_almost_equal(quadratic.func_directional(x, d, alpha=0.5), -2.625) assert_almost_equal(quadratic.grad_directional(x, d, alpha=0.5), 4.5) assert_almost_equal(quadratic.func_directional(x, d, alpha=1.0), 0.0) assert_almost_equal(quadratic.grad_directional(x, d, alpha=1.0), 6.0)
def first_experiment(): A_good = np.array([[1, 0.2], [0.2, 1.2]]) A_bad = np.array([[0.1, 0.1], [0.1, 1]]) for i, A in enumerate([A_good, A_bad]): cond = np.linalg.cond(A) oracle = oracles.QuadraticOracle(A, np.zeros(2)) np.random.seed(i * 42) x_0_s = [np.random.uniform(-5, 5, size=2) for _ in range(3)] for j in range(3): x_0 = x_0_s[j] for method in ['Wolfe', 'Armijo', 'Constant']: _, _, history = optimization.gradient_descent( oracle, x_0, line_search_options={ 'method': method, 'alpha_0': 100 }, trace=True) plot_trajectory_2d.plt.figure() plot_trajectory_2d.plot_levels(oracle.func) name = 'experiment_1/{}-{}-{}'.format(round(cond, 3), method, j) plot_trajectory_2d.plot_trajectory(oracle.func, history['x'], save=name) print('cond = {}, j = {}, method = {}, steps = {}'.format( round(cond, 3), j, method, len(history['x'])))
def get_iteration_num_for_fixed_n(condition_num_values, n, x_init, method_name, b_min=-1, b_max=1): """ Returns an array of iterations number for gradient descent for each value in condition_num_values for given n :param condition_num_values: array of condition numbers :param n: number of features :param x_init: initial point for gradient descent :param b_min: min value in b :param b_max: max value in b :param method_name: 'Wolfe', 'Armijo' or 'Constant' :return: an array with number of iterations for each value in condition_num_values """ iteration_num_values = [] for condition_num in condition_num_values: A, b = generate_task(condition_num, n, b_min, b_max) oracle = oracles.QuadraticOracle(A, b) [x_star, _, history] = optimization.gradient_descent(oracle, x_init, \ line_search_options={'method': method_name, 'c': 0.001}, \ trace=True) # print(x_star) # print(history['grad_norm']) # print("====================================") iteration_num_values.append(len(history['grad_norm'])) return iteration_num_values
def test_hess_finite_diff_1(): # Quadratic function. A = np.eye(3) b = np.array([1, 2, 3]) quadratic = oracles.QuadraticOracle(A, b) H = oracles.hess_finite_diff(quadratic.func, np.zeros(3)) ok_(isinstance(H, np.ndarray)) ok_(np.allclose(H, A))
def test_grad_finite_diff_1(): # Quadratic function. A = np.eye(3) b = np.array([1, 2, 3]) quadratic = oracles.QuadraticOracle(A, b) g = oracles.grad_finite_diff(quadratic.func, np.zeros(3)) ok_(isinstance(g, np.ndarray)) ok_(np.allclose(g, -b))
def second_experiment(): ns = [10, 100, 1000, 10000] colors = ['g', 'r', 'b', 'y'] kappas = list(range(1, 1000, 100)) iterations = 10 T = {} np.seterr(all='print') t0 = datetime.now() for n, color in zip(ns, colors): T[n] = [[] for _ in range(iterations)] for i in range(iterations): for kappa in kappas: np.random.seed(1000 * i + kappa) diag = np.random.uniform(low=1, high=kappa, size=n) diag[0], diag[-1] = 1, kappa A = diags(diag) b = np.random.uniform(low=1, high=kappa, size=n) oracle = oracles.QuadraticOracle(A, b) x_star, msg, history = optimization.gradient_descent( oracle, np.zeros(n), trace=True) if msg == 'success': T[n][i].append(len(history['grad_norm'])) else: T[n][i].append(10000) plt.plot(kappas, T[n][i], ls='--', color=color, alpha=0.2) plt.plot(kappas, np.mean(T[n], axis=0), color=color, label='n = {}'.format(n)) print('n = {}, time = {}'.format(n, datetime.now() - t0)) plt.grid() plt.legend() plt.ylabel('iterations') plt.xlabel(r'$\ae$') plt.savefig('experiment_2/T(n, kappa)')
plt.ylabel('y') plt.legend([ 'Constant, iterations number = {}'.format( len(history_constant['x']) - 1), 'Armije, iterations number = {}'.format(len(history_armijo['x']) - 1), 'Wolfe, iterations number = {}'.format(len(history_wolf['x']) - 1) ]) plt.savefig('./3.1/{}'.format(plot_name)) plt.show() if __name__ == '__main__': A = np.array([[1, 0], [0, 50]]) b = np.array([0, 0]) x_init = np.array([10.0, 4.0]) oracle = oracles.QuadraticOracle(A, b) xrange = [-6, 20] yrange = None levels = [0, 16, 64, 128, 256] save_comparison_plot(oracle, x_init, xrange, yrange, levels, plot_name='plot1') A = np.array([[1, 0], [0, 50]]) b = np.array([0, 0]) x_init = np.array([15.0, 0.0]) oracle = oracles.QuadraticOracle(A, b) xrange = [-6, 20]
def get_quadratic(self): # Quadratic function: # f(x) = 1/2 x^T x - [1, 2, 3]^T x A = np.eye(3) b = np.array([1, 2, 3]) return oracles.QuadraticOracle(A, b)
x_values, y_values = zip(*history) plt.plot(x_values, y_values, '-v', linewidth=5.0, ms=12.0, alpha=1.0, c='r', label=label) # Tries to adapt axis-ranges for the trajectory: if fit_axis: xmax, ymax = np.max(x_values), np.max(y_values) COEF = 1.5 xrange = [-xmax * COEF, xmax * COEF] yrange = [-ymax * COEF, ymax * COEF] plt.xlim(xrange) plt.ylim(yrange) plt.show() if __name__ == '__main__': A = np.eye(2) b = np.array([1, 2]) quadratic = oracles.QuadraticOracle(A, b) plot_levels(quadratic.func) x_star, msg, history = optimization.gradient_descent(quadratic, np.array([3.0, 1.5]), trace=True) plot_trajectory(quadratic.func, history['x'])
def T(k): quad = oracles.QuadraticOracle(Aa(k), b) x_star, msg, history = optimization.gradient_descent( quad, np.random.uniform(10, 30, n), trace=True) return len(history['grad_norm'])
exps = [{ "A": np.array([[5, 1], [1, 1]]), "b": np.zeros(2), "st": "normal" }, { "A": np.array([[25, 2], [2, 1]]), "b": np.array([0, 0]), "st": "stratched" }] for ix, x_0 in enumerate([np.array([3, 4]), np.array([1, 25])]): for ip, params in enumerate(exps): for met in ["Wolfe", "Armijo", "Constant"]: plt.clf() qua1 = oracles.QuadraticOracle(params['A'], params['b']) plot_trajectory_2d.plot_levels(qua1.func) x_star, msg, history = optimization.gradient_descent( qua1, x_0, trace=True, line_search_options={ 'method': met, 'c': 0.1 }) plot_trajectory_2d.plot_trajectory(qua1.func, history['x']) plt.savefig("exp1/{0}-{1}-{2}.png".format(x_0, params['st'], met)) for e in exps: print("{0} - {1}".format(e['st'], li.cond(e['A'])))
import oracles from plot_trajectory_2d import plot_levels from plot_trajectory_2d import plot_trajectory import optimization import numpy as np import matplotlib.pyplot as plt oracle = oracles.QuadraticOracle(np.array([[1.0, 2.0], [2.0, 5.0]]), np.zeros(2)) [x_star, msg, history] = optimization.gradient_descent(oracle, np.array([3.0, 1.5]), trace=True) plt.figure() plot_levels(oracle.func) plot_trajectory(oracle.func, history['x']) plt.plot()