def experiment_3(parameter='n', method='subgradient', seed=31415): # n, m and lambda n, m, reg_coef = 500, 500, 1.0 np.random.seed(seed) grids = dict() grids['n'] = [10, 100, 1000] grids['m'] = [10, 100, 1000] grids['reg_coef'] = [0.01, 0.1, 1.0, 10.0] fig1, ax1 = plt.subplots() fig2, ax2 = plt.subplots() ax1.set_xlabel('Номер итерации') ax1.set_ylabel('Гарантированная точность по зазору двойственности') ax1.grid() ax1.set_yscale('log') ax2.set_xlabel('Время от начала эксперимента') ax2.set_ylabel('Гарантированная точность по зазору двойственности') ax2.grid() ax2.set_yscale('log') os.makedirs("report/pics/3", exist_ok=True) experiment_parameters = {'n': n, 'm': m, 'reg_coef': 1.0} for value in grids[parameter]: experiment_parameters[parameter] = value A = np.random.randn(experiment_parameters['m'], experiment_parameters['n']) b = np.random.randn(experiment_parameters['m']) x_0 = np.ones(experiment_parameters['n']) reg_coef = experiment_parameters['reg_coef'] if method == 'subgradient': oracle = create_lasso_nonsmooth_oracle(A, b, reg_coef) x_opt, message, history = subgradient_method(oracle, x_0, trace=True, max_iter=10000) if method == 'proximal': oracle = create_lasso_prox_oracle(A, b, reg_coef) x_opt, message, history = proximal_gradient_method(oracle, x_0, trace=True, max_iter=10000) if method == 'proximal_fast': oracle = create_lasso_prox_oracle(A, b, reg_coef) x_opt, message, history = proximal_fast_gradient_method(oracle, x_0, trace=True, max_iter=10000) ax1.plot(history['duality_gap'], label=f'{parameter}={value}') ax2.plot(history['time'], history['duality_gap'], label=f'{parameter}={value}') ax1.legend() ax2.legend() os.makedirs(f"report/pics/3/{method}", exist_ok=True) fig1.savefig(f"report/pics/3/{method}/lasso_gap_vs_iter_{parameter}.pdf", bbox_inches='tight') fig2.savefig(f"report/pics/3/{method}/lasso_gap_vs_time_{parameter}.pdf", bbox_inches='tight')
def test_subgradient_one_step_nonsmooth(): # Minimize 0.5 * ||x - b||_2^2 + ||x||_1 # with small tolerance by one step. A = np.eye(2) b = np.array([3.0, 3.0]) oracle = oracles.create_lasso_nonsmooth_oracle(A, b, regcoef=1.0) x_0 = np.ones(2) [x_star, status, hist] = optimization.subgradient_method( oracle, x_0, tolerance=1e-1, trace=True) eq_(status, 'success') ok_(np.allclose(x_star, np.array([ 1.70710678, 1.70710678]))) ok_(np.allclose(np.array(hist['func']), np.array([6.0, 5.085786437626])))
def test_subgradient_one_step(): # Simple smooth quadratic task. A = np.eye(2) b = np.array([1.0, 0.0]) oracle = oracles.create_lasso_nonsmooth_oracle(A, b, regcoef=0.0) x_0 = np.zeros(2) [x_star, status, hist] = optimization.subgradient_method( oracle, x_0, trace=True) eq_(status, 'success') ok_(np.allclose(x_star, np.array([1.0, 0.0]))) ok_(np.allclose(np.array(hist['func']), np.array([0.5, 0.0])))
def experiment_1(): np.random.seed(31415) data_path = "data" datasets = ["bodyfat", "housing"] alpha_grid = [1e-3, 1e-2, 1e-1, 1.0, 5.0] for dataset in datasets: print("___________________________") logging.info(f"{dataset} is in process...") A, b = load_svmlight_file(os.path.join(data_path, dataset)) n = A.shape[1] x_grid = list() x_grid.append(np.zeros(A.shape[1])) x_grid.append(np.ones(A.shape[1])) x_grid.append(np.random.randn(n)) for i, x0 in enumerate(x_grid): fig1, ax1 = plt.subplots(figsize=(12, 8)) fig2, ax2 = plt.subplots(figsize=(12, 8)) ax1.set_xlabel('Номер итерации') ax1.set_ylabel('Гарантированная точность по зазору двойственности') ax1.grid() ax1.set_yscale('log') ax2.set_xlabel('Время от начала эксперимента') ax2.set_ylabel('Гарантированная точность по зазору двойственности') ax2.grid() ax2.set_yscale('log') oracle = create_lasso_nonsmooth_oracle(A, b, 1.0) # oracle = create_lasso_prox_oracle(A, b, 1.0) print(A.shape, 1 - A.size / (A.shape[0] * A.shape[1]), np.linalg.matrix_rank(A.toarray())) os.makedirs("report/pics/1", exist_ok=True) for alpha in alpha_grid: x_opt, message, history = subgradient_method(oracle, x0, alpha_0=alpha, trace=True, max_iter=1000000) ax1.plot(history['duality_gap'], label=f'alpha={alpha}') ax2.plot(history['time'], history['duality_gap'], label=f'alpha={alpha}') ax1.legend() ax2.legend() os.makedirs("report/pics/1", exist_ok=True) fig1.savefig(f"report/pics/1/subgradient_gap_vs_iter_alpha_x_{i}_{dataset}.pdf", bbox_inches='tight') fig2.savefig(f"report/pics/1/subgradient_gap_vs_time_alpha_x_{i}_{dataset}.pdf", bbox_inches='tight')
for d in ['exp1', 'exp2', 'exp3', 'exp4']: if not os.path.exists(d): os.makedirs(d) plt.clf() for x_0, label in zip([np.zeros(5), np.array([3, 5, 8, 2, 5]), np.array([3, 5, 8, 2, 5])*10, np.array([3, 5, 8, 2, 5])*100], ["Zero", "Near", "Far", "ReallyFar"]): plt.clf() res = [] for a_0 in [a / 10 for a in range(1, 50)]: A = np.random.rand(5, 5) b = np.random.rand(5) r = 1/5 oracle = oracles.create_lasso_nonsmooth_oracle(A, b, r) x_star, msg, hist = optimization.subgradient_method(oracle, x_0, alpha_0=a_0, trace=True, max_iter=10**4) res.append(len(hist['func'])) plt.plot([a / 10 for a in range(1, 50)], res) plt.xlabel("Alpha") plt.ylabel("Iterations") plt.savefig("exp1/{}.png".format(label)) # ------------------ Experiment 2 for n in [5, 50]: A = np.random.rand(n, n) b = np.random.rand(n) r = 1/n oracle = oracles.create_lasso_prox_oracle(A, b, r) x_star, msg, hist = optimization.proximal_gradient_descent(oracle, np.zeros(n), trace=True)