def experiment_3(parameter='n', method='subgradient', seed=31415):
# n, m and lambda
n, m, reg_coef = 500, 500, 1.0
np.random.seed(seed)
grids = dict()
grids['n'] = [10, 100, 1000]
grids['m'] = [10, 100, 1000]
grids['reg_coef'] = [0.01, 0.1, 1.0, 10.0]
fig1, ax1 = plt.subplots()
fig2, ax2 = plt.subplots()
ax1.set_xlabel('Номер итерации')
ax1.set_ylabel('Гарантированная точность по зазору двойственности')
ax1.grid()
ax1.set_yscale('log')
ax2.set_xlabel('Время от начала эксперимента')
ax2.set_ylabel('Гарантированная точность по зазору двойственности')
ax2.grid()
ax2.set_yscale('log')
os.makedirs("report/pics/3", exist_ok=True)
experiment_parameters = {'n': n, 'm': m, 'reg_coef': 1.0}
for value in grids[parameter]:
experiment_parameters[parameter] = value
A = np.random.randn(experiment_parameters['m'], experiment_parameters['n'])
b = np.random.randn(experiment_parameters['m'])
x_0 = np.ones(experiment_parameters['n'])
reg_coef = experiment_parameters['reg_coef']
if method == 'subgradient':
oracle = create_lasso_nonsmooth_oracle(A, b, reg_coef)
x_opt, message, history = subgradient_method(oracle, x_0, trace=True, max_iter=10000)
if method == 'proximal':
oracle = create_lasso_prox_oracle(A, b, reg_coef)
x_opt, message, history = proximal_gradient_method(oracle, x_0, trace=True, max_iter=10000)
if method == 'proximal_fast':
oracle = create_lasso_prox_oracle(A, b, reg_coef)
x_opt, message, history = proximal_fast_gradient_method(oracle, x_0, trace=True, max_iter=10000)
ax1.plot(history['duality_gap'], label=f'{parameter}={value}')
ax2.plot(history['time'], history['duality_gap'], label=f'{parameter}={value}')
ax1.legend()
ax2.legend()
os.makedirs(f"report/pics/3/{method}", exist_ok=True)
fig1.savefig(f"report/pics/3/{method}/lasso_gap_vs_iter_{parameter}.pdf", bbox_inches='tight')
fig2.savefig(f"report/pics/3/{method}/lasso_gap_vs_time_{parameter}.pdf", bbox_inches='tight')
def first_experiment():
A, b = load_svmlight_file('datasets/w8a')
regcoef = 1
oracle = oracles.create_lasso_prox_oracle(A, b, regcoef)
m, n = A.shape
x_0 = np.zeros(n)
def plot_iterations(history, name):
name_ = name
name = ' '.join(name.split('_')).title()
y = history['iterations']
x = np.array(list(range(len(y))))
plt.figure()
plt.title(name + '\nmin duality gap = {}'.format(
round(min(history['duality_gap']), 2)),
fontsize=14)
plt.plot(x, y, label='line search iterations')
plt.plot(x, 2 * x, label='double iterations')
plt.xlabel('method iterations', fontsize=14)
plt.ylabel('line search iterations', fontsize=14)
plt.legend(fontsize=14)
plt.grid()
plt.savefig('experiment_1/' + name_ + '.png')
for method in [
optimization.proximal_gradient_method,
optimization.proximal_fast_gradient_method
]:
print(method.__name__)
x_star, message, history = method(oracle, x_0, trace=True)
plot_iterations(history, method.__name__)
def test_proximal_fast_gm_nonsmooth():
# Minimize ||x||_1.
oracle = oracles.create_lasso_prox_oracle(np.zeros([2, 2]),
np.zeros(2),
regcoef=1.0)
x_0 = np.array([2.0, -1.0])
[x_star, status,
hist] = optimization.proximal_fast_gradient_method(oracle,
x_0,
trace=True)
eq_(status, 'success')
ok_(np.allclose(x_star, np.array([7.64497775e-06, 0.0])))
ok_(
np.allclose(
np.array(hist['func']),
np.array([[
3.0, 1.0, 0.2679491924311227, 0.099179721203666166,
0.041815858679730096, 0.018850854415891891,
0.0088302704548977509, 0.0042337678350286671,
0.0020598280587097542, 0.0010116006643634953,
0.00049984569823222113, 0.00024797029202045391,
0.00012334133663333596, 6.1457610088328345e-05,
3.0658125993932962e-05, 1.5305578515886715e-05,
7.6449777454652732e-06
]])))
def test_proximal_nonsmooth():
# Minimize ||x||_1.
oracle = oracles.create_lasso_prox_oracle(np.zeros([2, 2]),
np.zeros(2),
regcoef=1.0)
x_0 = np.array([2.0, -1.0])
[x_star, status, hist] = optimization.proximal_gradient_descent(
oracle, x_0, trace=True)
eq_(status, 'success')
ok_(np.allclose(x_star, np.array([0.0, 0.0])))
ok_(np.allclose(np.array(hist['func']), np.array([3.0, 1.0, 0.0])))
def test_proximal_gd_one_step():
# Simple smooth quadratic task.
A = np.eye(2)
b = np.array([1.0, 0.0])
oracle = oracles.create_lasso_prox_oracle(A, b, regcoef=0.0)
x_0 = np.zeros(2)
[x_star, status, hist] = optimization.proximal_gradient_descent(
oracle, x_0, trace=True)
eq_(status, 'success')
ok_(np.allclose(x_star, np.array([1.0, 0.0])))
ok_(np.allclose(np.array(hist['func']), np.array([0.5, 0.0])))
def second_experiment():
ns = [50, 100, 500]
ms = [1000, 5000, 10000]
regcoefs = [0.1, 1, 10]
methods = [
optimization.proximal_gradient_method,
optimization.proximal_fast_gradient_method,
optimization.barrier_method_lasso
]
def plot_accuracy(histories, n, m, regcoef):
name = 'n={}, m={}, regcoef={}'.format(n, m, regcoef)
for x_name in ['iterations', 'time']:
plt.figure()
plt.title(name)
for history, method in zip(histories, methods):
method_name = ' '.join(method.__name__.split('_')).title()
y = history['duality_gap']
x = np.array(
history['time']) * 1000 if x_name == 'time' else list(
range(len(y)))
plt.plot(x, y, label=method_name)
plt.xlabel(x_name, fontsize=14)
plt.ylabel('duality gap', fontsize=14)
plt.yscale('log')
plt.legend(fontsize=10, loc=1)
plt.grid()
plt.savefig('experiment_2/' + name + ' ' + x_name + '.png')
plt.close('all')
for iteration, (n, m,
regcoef) in enumerate(itertools.product(ns, ms, regcoefs)):
print(n, m, regcoef)
np.random.seed(iteration * 50 + 42)
A = np.random.uniform(low=-100, high=100, size=(m, n))
b = np.random.uniform(low=-100, high=100, size=m)
x_0 = np.zeros(n)
oracle = oracles.create_lasso_prox_oracle(A, b, regcoef)
histories = []
for i, method in enumerate(methods):
print(method.__name__)
if i < 2:
x_star, message, history = method(oracle, x_0, trace=True)
else:
u_0 = np.array([100] * n)
x_star, message, history = method(A,
b,
regcoef,
x_0,
u_0,
trace=True)
histories.append(history)
plot_accuracy(histories, n, m, regcoef)
def test_lasso_prox_oracle():
A = np.eye(2)
b = np.array([1.0, 2.0])
oracle = oracles.create_lasso_prox_oracle(A, b, regcoef=1.0)
# Checks at point x = [-3, 3]
x = np.array([-3.0, 3.0])
assert_almost_equal(oracle.func(x), 14.5)
ok_(np.allclose(oracle.grad(x), np.array([-4., 1.])))
ok_(isinstance(oracle.grad(x), np.ndarray))
ok_(np.allclose(oracle.prox(x, alpha=1.0), np.array([-2.0, 2.0])))
ok_(np.allclose(oracle.prox(x, alpha=2.0), np.array([-1.0, 1.0])))
ok_(isinstance(oracle.prox(x, alpha=1.0), np.ndarray))
assert_almost_equal(oracle.duality_gap(x), 14.53125)
def test_proximal_fast_gm_nonsmooth2():
oracle = oracles.create_lasso_prox_oracle(np.array([[1, 2, 3], [4, 5, 6]]),
np.array([1, 4]),
regcoef=1.0)
x_0 = np.array([1, 1, -1])
x_star, status, hist = optimization.proximal_fast_gradient_method(
oracle, x_0, trace=True, max_iter=3)
eq_(status, 'iterations_exceeded')
ok_(np.allclose(x_star, np.array([[1.01714486, 1.04658812, -0.85594606]])))
ok_(
np.allclose(
np.array(hist['func']),
np.array(
[4.0, 3.219970703125, 3.0946475565433502,
3.0380920410969945])))
def test_proximal_nonsmooth2():
oracle = oracles.create_lasso_prox_oracle(np.array([[1, 2, 3], [4, 5, 6]]),
np.array([1, 4]),
regcoef=1.0)
x_0 = np.array([1, 1, -1])
[x_star, status, hist] = optimization.proximal_gradient_descent(oracle,
x_0,
trace=True,
max_iter=3)
eq_(status, 'iterations_exceeded')
ok_(np.allclose(x_star, np.array([1.02216721, 1.05131721, -0.85703278])))
ok_(
np.allclose(
np.array(hist['func']),
np.array(
[[4.0, 3.219970703125, 3.1220934763550758,
3.0507238902373501]])))
def test_accelerated_proximal_nonsmooth2():
oracle = oracles.create_lasso_prox_oracle(np.array([[1, 2, 3], [4, 5, 6]]),
np.array([1, 4]),
regcoef=1.0)
x_0 = np.array([1, 1, -1])
[x_star, status,
hist] = optimization.accelerated_proximal_gradient_descent(oracle,
x_0,
trace=True,
max_iter=3)
eq_(status, 'iterations_exceeded')
ok_(np.allclose(x_star, np.array([[1.01714486, 1.04658812, -0.85594606]])))
ok_(
np.allclose(
np.array(hist['func']),
np.array(
[4.0, 3.219970703125, 3.1220934763550758,
3.0380920410969945])))
def test_proximal_gd_prototype():
method = optimization.proximal_gradient_descent
A = np.array([[1.0, 2.0], [3.0, 4.0]])
b = np.array([1.0, 2.0])
oracle = oracles.create_lasso_prox_oracle(A, b, regcoef=2.0)
x_0 = np.array([-3.0, 0.0])
method(oracle, x_0)
method(oracle, x_0, L_0=1)
check_prototype_results(method(oracle, x_0, tolerance=1e10),
[None, 'success', None])
check_prototype_results(method(oracle, x_0, tolerance=1e10, trace=True),
[None, 'success', [0.0]])
check_prototype_results(method(oracle, x_0, max_iter=1),
[None, 'iterations_exceeded', None])
check_prototype_results(method(oracle, x_0, max_iter=1, trace=True),
[None, 'iterations_exceeded', [0.0, 0.0]])
method(oracle, x_0, display=True)
method(oracle, x_0, 1, 1e-5, 100, True, True)
def experiment_2():
np.random.seed(31415)
data_path = "data"
datasets = ["bodyfat", "housing"]
for dataset in datasets:
print("___________________________")
logging.info(f"{dataset} is in process...")
A, b = load_svmlight_file(os.path.join(data_path, dataset))
fig, ax = plt.subplots(figsize=(12, 8))
ax.set_xlabel('Номер итерации')
ax.set_ylabel('Суммарное число шагов подбора L')
ax.grid()
oracle = create_lasso_prox_oracle(A, b, 1.0)
print(A.shape, 1 - A.size / (A.shape[0] * A.shape[1]), np.linalg.matrix_rank(A.toarray()))
n = A.shape[1]
x0 = np.random.randn(n)
x_opt, message, history_usual = proximal_gradient_method(oracle, x0, trace=True, max_iter=10000)
x_opt, message, history_fast = proximal_fast_gradient_method(oracle, x0, trace=True, max_iter=10000)
sum_int_steps_usual = list(accumulate(history_usual['int_steps']))
sum_int_steps_fast = list(accumulate(history_fast['int_steps']))
ax.plot(sum_int_steps_usual, label="Usual proximal method")
ax.plot(sum_int_steps_fast, label=f'Fast proximal method')
ax.legend()
os.makedirs("report/pics/2", exist_ok=True)
fig.savefig(f"report/pics/2/prox_methods_steps_{dataset}.pdf", bbox_inches='tight')
oracle = oracles.create_lasso_nonsmooth_oracle(A, b, r)
x_star, msg, hist = optimization.subgradient_method(oracle, x_0, alpha_0=a_0, trace=True, max_iter=10**4)
res.append(len(hist['func']))
plt.plot([a / 10 for a in range(1, 50)], res)
plt.xlabel("Alpha")
plt.ylabel("Iterations")
plt.savefig("exp1/{}.png".format(label))
# ------------------ Experiment 2
for n in [5, 50]:
A = np.random.rand(n, n)
b = np.random.rand(n)
r = 1/n
oracle = oracles.create_lasso_prox_oracle(A, b, r)
x_star, msg, hist = optimization.proximal_gradient_descent(oracle, np.zeros(n), trace=True)
plt.clf()
plt.plot(range(len(hist['line_counter'])), hist['line_counter'])
plt.xlabel("Iteration")
plt.ylabel("Line search counter")
plt.savefig("exp2/LCounter{}.png".format(n))
# ------------------- Experiment 3
for n in [50, 1000]:
for eps, eps_name in zip([10**-2, 10**-7], ["low", "high"]):
A = np.random.rand(n, n)
b = np.random.rand(n)
r = 1
x_0 = np.zeros(n)
