def experiment_3(parameter='n', method='subgradient', seed=31415): # n, m and lambda n, m, reg_coef = 500, 500, 1.0 np.random.seed(seed) grids = dict() grids['n'] = [10, 100, 1000] grids['m'] = [10, 100, 1000] grids['reg_coef'] = [0.01, 0.1, 1.0, 10.0] fig1, ax1 = plt.subplots() fig2, ax2 = plt.subplots() ax1.set_xlabel('Номер итерации') ax1.set_ylabel('Гарантированная точность по зазору двойственности') ax1.grid() ax1.set_yscale('log') ax2.set_xlabel('Время от начала эксперимента') ax2.set_ylabel('Гарантированная точность по зазору двойственности') ax2.grid() ax2.set_yscale('log') os.makedirs("report/pics/3", exist_ok=True) experiment_parameters = {'n': n, 'm': m, 'reg_coef': 1.0} for value in grids[parameter]: experiment_parameters[parameter] = value A = np.random.randn(experiment_parameters['m'], experiment_parameters['n']) b = np.random.randn(experiment_parameters['m']) x_0 = np.ones(experiment_parameters['n']) reg_coef = experiment_parameters['reg_coef'] if method == 'subgradient': oracle = create_lasso_nonsmooth_oracle(A, b, reg_coef) x_opt, message, history = subgradient_method(oracle, x_0, trace=True, max_iter=10000) if method == 'proximal': oracle = create_lasso_prox_oracle(A, b, reg_coef) x_opt, message, history = proximal_gradient_method(oracle, x_0, trace=True, max_iter=10000) if method == 'proximal_fast': oracle = create_lasso_prox_oracle(A, b, reg_coef) x_opt, message, history = proximal_fast_gradient_method(oracle, x_0, trace=True, max_iter=10000) ax1.plot(history['duality_gap'], label=f'{parameter}={value}') ax2.plot(history['time'], history['duality_gap'], label=f'{parameter}={value}') ax1.legend() ax2.legend() os.makedirs(f"report/pics/3/{method}", exist_ok=True) fig1.savefig(f"report/pics/3/{method}/lasso_gap_vs_iter_{parameter}.pdf", bbox_inches='tight') fig2.savefig(f"report/pics/3/{method}/lasso_gap_vs_time_{parameter}.pdf", bbox_inches='tight')
def first_experiment(): A, b = load_svmlight_file('datasets/w8a') regcoef = 1 oracle = oracles.create_lasso_prox_oracle(A, b, regcoef) m, n = A.shape x_0 = np.zeros(n) def plot_iterations(history, name): name_ = name name = ' '.join(name.split('_')).title() y = history['iterations'] x = np.array(list(range(len(y)))) plt.figure() plt.title(name + '\nmin duality gap = {}'.format( round(min(history['duality_gap']), 2)), fontsize=14) plt.plot(x, y, label='line search iterations') plt.plot(x, 2 * x, label='double iterations') plt.xlabel('method iterations', fontsize=14) plt.ylabel('line search iterations', fontsize=14) plt.legend(fontsize=14) plt.grid() plt.savefig('experiment_1/' + name_ + '.png') for method in [ optimization.proximal_gradient_method, optimization.proximal_fast_gradient_method ]: print(method.__name__) x_star, message, history = method(oracle, x_0, trace=True) plot_iterations(history, method.__name__)
def test_proximal_fast_gm_nonsmooth(): # Minimize ||x||_1. oracle = oracles.create_lasso_prox_oracle(np.zeros([2, 2]), np.zeros(2), regcoef=1.0) x_0 = np.array([2.0, -1.0]) [x_star, status, hist] = optimization.proximal_fast_gradient_method(oracle, x_0, trace=True) eq_(status, 'success') ok_(np.allclose(x_star, np.array([7.64497775e-06, 0.0]))) ok_( np.allclose( np.array(hist['func']), np.array([[ 3.0, 1.0, 0.2679491924311227, 0.099179721203666166, 0.041815858679730096, 0.018850854415891891, 0.0088302704548977509, 0.0042337678350286671, 0.0020598280587097542, 0.0010116006643634953, 0.00049984569823222113, 0.00024797029202045391, 0.00012334133663333596, 6.1457610088328345e-05, 3.0658125993932962e-05, 1.5305578515886715e-05, 7.6449777454652732e-06 ]])))
def test_proximal_nonsmooth(): # Minimize ||x||_1. oracle = oracles.create_lasso_prox_oracle(np.zeros([2, 2]), np.zeros(2), regcoef=1.0) x_0 = np.array([2.0, -1.0]) [x_star, status, hist] = optimization.proximal_gradient_descent( oracle, x_0, trace=True) eq_(status, 'success') ok_(np.allclose(x_star, np.array([0.0, 0.0]))) ok_(np.allclose(np.array(hist['func']), np.array([3.0, 1.0, 0.0])))
def test_proximal_gd_one_step(): # Simple smooth quadratic task. A = np.eye(2) b = np.array([1.0, 0.0]) oracle = oracles.create_lasso_prox_oracle(A, b, regcoef=0.0) x_0 = np.zeros(2) [x_star, status, hist] = optimization.proximal_gradient_descent( oracle, x_0, trace=True) eq_(status, 'success') ok_(np.allclose(x_star, np.array([1.0, 0.0]))) ok_(np.allclose(np.array(hist['func']), np.array([0.5, 0.0])))
def second_experiment(): ns = [50, 100, 500] ms = [1000, 5000, 10000] regcoefs = [0.1, 1, 10] methods = [ optimization.proximal_gradient_method, optimization.proximal_fast_gradient_method, optimization.barrier_method_lasso ] def plot_accuracy(histories, n, m, regcoef): name = 'n={}, m={}, regcoef={}'.format(n, m, regcoef) for x_name in ['iterations', 'time']: plt.figure() plt.title(name) for history, method in zip(histories, methods): method_name = ' '.join(method.__name__.split('_')).title() y = history['duality_gap'] x = np.array( history['time']) * 1000 if x_name == 'time' else list( range(len(y))) plt.plot(x, y, label=method_name) plt.xlabel(x_name, fontsize=14) plt.ylabel('duality gap', fontsize=14) plt.yscale('log') plt.legend(fontsize=10, loc=1) plt.grid() plt.savefig('experiment_2/' + name + ' ' + x_name + '.png') plt.close('all') for iteration, (n, m, regcoef) in enumerate(itertools.product(ns, ms, regcoefs)): print(n, m, regcoef) np.random.seed(iteration * 50 + 42) A = np.random.uniform(low=-100, high=100, size=(m, n)) b = np.random.uniform(low=-100, high=100, size=m) x_0 = np.zeros(n) oracle = oracles.create_lasso_prox_oracle(A, b, regcoef) histories = [] for i, method in enumerate(methods): print(method.__name__) if i < 2: x_star, message, history = method(oracle, x_0, trace=True) else: u_0 = np.array([100] * n) x_star, message, history = method(A, b, regcoef, x_0, u_0, trace=True) histories.append(history) plot_accuracy(histories, n, m, regcoef)
def test_lasso_prox_oracle(): A = np.eye(2) b = np.array([1.0, 2.0]) oracle = oracles.create_lasso_prox_oracle(A, b, regcoef=1.0) # Checks at point x = [-3, 3] x = np.array([-3.0, 3.0]) assert_almost_equal(oracle.func(x), 14.5) ok_(np.allclose(oracle.grad(x), np.array([-4., 1.]))) ok_(isinstance(oracle.grad(x), np.ndarray)) ok_(np.allclose(oracle.prox(x, alpha=1.0), np.array([-2.0, 2.0]))) ok_(np.allclose(oracle.prox(x, alpha=2.0), np.array([-1.0, 1.0]))) ok_(isinstance(oracle.prox(x, alpha=1.0), np.ndarray)) assert_almost_equal(oracle.duality_gap(x), 14.53125)
def test_proximal_fast_gm_nonsmooth2(): oracle = oracles.create_lasso_prox_oracle(np.array([[1, 2, 3], [4, 5, 6]]), np.array([1, 4]), regcoef=1.0) x_0 = np.array([1, 1, -1]) x_star, status, hist = optimization.proximal_fast_gradient_method( oracle, x_0, trace=True, max_iter=3) eq_(status, 'iterations_exceeded') ok_(np.allclose(x_star, np.array([[1.01714486, 1.04658812, -0.85594606]]))) ok_( np.allclose( np.array(hist['func']), np.array( [4.0, 3.219970703125, 3.0946475565433502, 3.0380920410969945])))
def test_proximal_nonsmooth2(): oracle = oracles.create_lasso_prox_oracle(np.array([[1, 2, 3], [4, 5, 6]]), np.array([1, 4]), regcoef=1.0) x_0 = np.array([1, 1, -1]) [x_star, status, hist] = optimization.proximal_gradient_descent(oracle, x_0, trace=True, max_iter=3) eq_(status, 'iterations_exceeded') ok_(np.allclose(x_star, np.array([1.02216721, 1.05131721, -0.85703278]))) ok_( np.allclose( np.array(hist['func']), np.array( [[4.0, 3.219970703125, 3.1220934763550758, 3.0507238902373501]])))
def test_accelerated_proximal_nonsmooth2(): oracle = oracles.create_lasso_prox_oracle(np.array([[1, 2, 3], [4, 5, 6]]), np.array([1, 4]), regcoef=1.0) x_0 = np.array([1, 1, -1]) [x_star, status, hist] = optimization.accelerated_proximal_gradient_descent(oracle, x_0, trace=True, max_iter=3) eq_(status, 'iterations_exceeded') ok_(np.allclose(x_star, np.array([[1.01714486, 1.04658812, -0.85594606]]))) ok_( np.allclose( np.array(hist['func']), np.array( [4.0, 3.219970703125, 3.1220934763550758, 3.0380920410969945])))
def test_proximal_gd_prototype(): method = optimization.proximal_gradient_descent A = np.array([[1.0, 2.0], [3.0, 4.0]]) b = np.array([1.0, 2.0]) oracle = oracles.create_lasso_prox_oracle(A, b, regcoef=2.0) x_0 = np.array([-3.0, 0.0]) method(oracle, x_0) method(oracle, x_0, L_0=1) check_prototype_results(method(oracle, x_0, tolerance=1e10), [None, 'success', None]) check_prototype_results(method(oracle, x_0, tolerance=1e10, trace=True), [None, 'success', [0.0]]) check_prototype_results(method(oracle, x_0, max_iter=1), [None, 'iterations_exceeded', None]) check_prototype_results(method(oracle, x_0, max_iter=1, trace=True), [None, 'iterations_exceeded', [0.0, 0.0]]) method(oracle, x_0, display=True) method(oracle, x_0, 1, 1e-5, 100, True, True)
def experiment_2(): np.random.seed(31415) data_path = "data" datasets = ["bodyfat", "housing"] for dataset in datasets: print("___________________________") logging.info(f"{dataset} is in process...") A, b = load_svmlight_file(os.path.join(data_path, dataset)) fig, ax = plt.subplots(figsize=(12, 8)) ax.set_xlabel('Номер итерации') ax.set_ylabel('Суммарное число шагов подбора L') ax.grid() oracle = create_lasso_prox_oracle(A, b, 1.0) print(A.shape, 1 - A.size / (A.shape[0] * A.shape[1]), np.linalg.matrix_rank(A.toarray())) n = A.shape[1] x0 = np.random.randn(n) x_opt, message, history_usual = proximal_gradient_method(oracle, x0, trace=True, max_iter=10000) x_opt, message, history_fast = proximal_fast_gradient_method(oracle, x0, trace=True, max_iter=10000) sum_int_steps_usual = list(accumulate(history_usual['int_steps'])) sum_int_steps_fast = list(accumulate(history_fast['int_steps'])) ax.plot(sum_int_steps_usual, label="Usual proximal method") ax.plot(sum_int_steps_fast, label=f'Fast proximal method') ax.legend() os.makedirs("report/pics/2", exist_ok=True) fig.savefig(f"report/pics/2/prox_methods_steps_{dataset}.pdf", bbox_inches='tight')
oracle = oracles.create_lasso_nonsmooth_oracle(A, b, r) x_star, msg, hist = optimization.subgradient_method(oracle, x_0, alpha_0=a_0, trace=True, max_iter=10**4) res.append(len(hist['func'])) plt.plot([a / 10 for a in range(1, 50)], res) plt.xlabel("Alpha") plt.ylabel("Iterations") plt.savefig("exp1/{}.png".format(label)) # ------------------ Experiment 2 for n in [5, 50]: A = np.random.rand(n, n) b = np.random.rand(n) r = 1/n oracle = oracles.create_lasso_prox_oracle(A, b, r) x_star, msg, hist = optimization.proximal_gradient_descent(oracle, np.zeros(n), trace=True) plt.clf() plt.plot(range(len(hist['line_counter'])), hist['line_counter']) plt.xlabel("Iteration") plt.ylabel("Line search counter") plt.savefig("exp2/LCounter{}.png".format(n)) # ------------------- Experiment 3 for n in [50, 1000]: for eps, eps_name in zip([10**-2, 10**-7], ["low", "high"]): A = np.random.rand(n, n) b = np.random.rand(n) r = 1 x_0 = np.zeros(n)