def experiment_3(parameter='n', method='subgradient', seed=31415):
# n, m and lambda
n, m, reg_coef = 500, 500, 1.0
np.random.seed(seed)
grids = dict()
grids['n'] = [10, 100, 1000]
grids['m'] = [10, 100, 1000]
grids['reg_coef'] = [0.01, 0.1, 1.0, 10.0]
fig1, ax1 = plt.subplots()
fig2, ax2 = plt.subplots()
ax1.set_xlabel('Номер итерации')
ax1.set_ylabel('Гарантированная точность по зазору двойственности')
ax1.grid()
ax1.set_yscale('log')
ax2.set_xlabel('Время от начала эксперимента')
ax2.set_ylabel('Гарантированная точность по зазору двойственности')
ax2.grid()
ax2.set_yscale('log')
os.makedirs("report/pics/3", exist_ok=True)
experiment_parameters = {'n': n, 'm': m, 'reg_coef': 1.0}
for value in grids[parameter]:
experiment_parameters[parameter] = value
A = np.random.randn(experiment_parameters['m'], experiment_parameters['n'])
b = np.random.randn(experiment_parameters['m'])
x_0 = np.ones(experiment_parameters['n'])
reg_coef = experiment_parameters['reg_coef']
if method == 'subgradient':
oracle = create_lasso_nonsmooth_oracle(A, b, reg_coef)
x_opt, message, history = subgradient_method(oracle, x_0, trace=True, max_iter=10000)
if method == 'proximal':
oracle = create_lasso_prox_oracle(A, b, reg_coef)
x_opt, message, history = proximal_gradient_method(oracle, x_0, trace=True, max_iter=10000)
if method == 'proximal_fast':
oracle = create_lasso_prox_oracle(A, b, reg_coef)
x_opt, message, history = proximal_fast_gradient_method(oracle, x_0, trace=True, max_iter=10000)
ax1.plot(history['duality_gap'], label=f'{parameter}={value}')
ax2.plot(history['time'], history['duality_gap'], label=f'{parameter}={value}')
ax1.legend()
ax2.legend()
os.makedirs(f"report/pics/3/{method}", exist_ok=True)
fig1.savefig(f"report/pics/3/{method}/lasso_gap_vs_iter_{parameter}.pdf", bbox_inches='tight')
fig2.savefig(f"report/pics/3/{method}/lasso_gap_vs_time_{parameter}.pdf", bbox_inches='tight')
def test_subgradient_one_step_nonsmooth():
# Minimize 0.5 * ||x - b||_2^2 + ||x||_1
# with small tolerance by one step.
A = np.eye(2)
b = np.array([3.0, 3.0])
oracle = oracles.create_lasso_nonsmooth_oracle(A, b, regcoef=1.0)
x_0 = np.ones(2)
[x_star, status, hist] = optimization.subgradient_method(
oracle, x_0, tolerance=1e-1, trace=True)
eq_(status, 'success')
ok_(np.allclose(x_star, np.array([ 1.70710678, 1.70710678])))
ok_(np.allclose(np.array(hist['func']), np.array([6.0, 5.085786437626])))
def test_subgradient_one_step():
# Simple smooth quadratic task.
A = np.eye(2)
b = np.array([1.0, 0.0])
oracle = oracles.create_lasso_nonsmooth_oracle(A, b, regcoef=0.0)
x_0 = np.zeros(2)
[x_star, status, hist] = optimization.subgradient_method(
oracle, x_0, trace=True)
eq_(status, 'success')
ok_(np.allclose(x_star, np.array([1.0, 0.0])))
ok_(np.allclose(np.array(hist['func']), np.array([0.5, 0.0])))
def experiment_1():
np.random.seed(31415)
data_path = "data"
datasets = ["bodyfat", "housing"]
alpha_grid = [1e-3, 1e-2, 1e-1, 1.0, 5.0]
for dataset in datasets:
print("___________________________")
logging.info(f"{dataset} is in process...")
A, b = load_svmlight_file(os.path.join(data_path, dataset))
n = A.shape[1]
x_grid = list()
x_grid.append(np.zeros(A.shape[1]))
x_grid.append(np.ones(A.shape[1]))
x_grid.append(np.random.randn(n))
for i, x0 in enumerate(x_grid):
fig1, ax1 = plt.subplots(figsize=(12, 8))
fig2, ax2 = plt.subplots(figsize=(12, 8))
ax1.set_xlabel('Номер итерации')
ax1.set_ylabel('Гарантированная точность по зазору двойственности')
ax1.grid()
ax1.set_yscale('log')
ax2.set_xlabel('Время от начала эксперимента')
ax2.set_ylabel('Гарантированная точность по зазору двойственности')
ax2.grid()
ax2.set_yscale('log')
oracle = create_lasso_nonsmooth_oracle(A, b, 1.0)
# oracle = create_lasso_prox_oracle(A, b, 1.0)
print(A.shape, 1 - A.size / (A.shape[0] * A.shape[1]), np.linalg.matrix_rank(A.toarray()))
os.makedirs("report/pics/1", exist_ok=True)
for alpha in alpha_grid:
x_opt, message, history = subgradient_method(oracle, x0, alpha_0=alpha, trace=True, max_iter=1000000)
ax1.plot(history['duality_gap'], label=f'alpha={alpha}')
ax2.plot(history['time'], history['duality_gap'], label=f'alpha={alpha}')
ax1.legend()
ax2.legend()
os.makedirs("report/pics/1", exist_ok=True)
fig1.savefig(f"report/pics/1/subgradient_gap_vs_iter_alpha_x_{i}_{dataset}.pdf", bbox_inches='tight')
fig2.savefig(f"report/pics/1/subgradient_gap_vs_time_alpha_x_{i}_{dataset}.pdf", bbox_inches='tight')
def test_lasso_nonsmooth_oracle():
A = np.eye(2)
b = np.array([1.0, 2.0])
oracle = oracles.create_lasso_nonsmooth_oracle(A, b, regcoef=2.0)
# Checks at point x = [1, 0]
x = np.array([-3.0, 0.0])
assert_almost_equal(oracle.func(x), 16.0)
assert_almost_equal(oracle.duality_gap(x), 14.5)
# Checks a subgradient
g = oracle.subgrad(x)
ok_(isinstance(g, np.ndarray))
assert_almost_equal(g[0], -6.0)
assert_almost_equal(g[1], -2.0)
def test_subgradient_prototype():
method = optimization.subgradient_method
A = np.array([[1.0, 2.0], [3.0, 4.0]])
b = np.array([1.0, 2.0])
oracle = oracles.create_lasso_nonsmooth_oracle(A, b, regcoef=2.0)
x_0 = np.array([-3.0, 0.0])
method(oracle, x_0)
check_prototype_results(method(oracle, x_0, tolerance=1e10),
[x_0, 'success', None])
check_prototype_results(method(oracle, x_0, tolerance=1e10, trace=True),
[None, 'success', [0.0]])
check_prototype_results(method(oracle, x_0, max_iter=1),
[None, 'iterations_exceeded', None])
check_prototype_results(method(oracle, x_0, max_iter=1, trace=True),
[None, 'iterations_exceeded', [0.0, 0.0]])
method(oracle, x_0, alpha_0=1)
method(oracle, x_0, display=True)
method(oracle, x_0, 1e-2, 100, 1, True, True)
# -------------- Experiment 1 ----------
for d in ['exp1', 'exp2', 'exp3', 'exp4']:
if not os.path.exists(d):
os.makedirs(d)
plt.clf()
for x_0, label in zip([np.zeros(5), np.array([3, 5, 8, 2, 5]), np.array([3, 5, 8, 2, 5])*10, np.array([3, 5, 8, 2, 5])*100], ["Zero", "Near", "Far", "ReallyFar"]):
plt.clf()
res = []
for a_0 in [a / 10 for a in range(1, 50)]:
A = np.random.rand(5, 5)
b = np.random.rand(5)
r = 1/5
oracle = oracles.create_lasso_nonsmooth_oracle(A, b, r)
x_star, msg, hist = optimization.subgradient_method(oracle, x_0, alpha_0=a_0, trace=True, max_iter=10**4)
res.append(len(hist['func']))
plt.plot([a / 10 for a in range(1, 50)], res)
plt.xlabel("Alpha")
plt.ylabel("Iterations")
plt.savefig("exp1/{}.png".format(label))
# ------------------ Experiment 2
for n in [5, 50]:
A = np.random.rand(n, n)
b = np.random.rand(n)
r = 1/n
oracle = oracles.create_lasso_prox_oracle(A, b, r)
