def run_experiment(n, line_search, max_iters):
print('Experiment: \t n = %d, \t line_search = %s, \t max_iters = %d.' %
(n, str(line_search), max_iters))
oracle = PDifferenceOracle(3)
x_star = np.zeros(n)
f_star = oracle.func(x_star)
x_0 = np.ones(n)
H_0 = 1.0
tolerance = get_tolerance({
'criterion': 'func',
'f_star': f_star,
'tolerance': 1e-9
})
power_strategy_1 = get_tolerance_strategy({
'strategy': 'power',
'c': 1.0,
'alpha': 1,
'label': r'$1/k$'
})
power_strategy_3 = get_tolerance_strategy({
'strategy': 'power',
'c': 1.0,
'alpha': 3,
'label': r'$1/k^3$'
})
adaptive_strategy = get_tolerance_strategy({
'strategy': 'adaptive',
'c': 1.0,
'alpha': 1,
'label': 'adaptive'
})
subsolver = 'NCG'
stopping_criterion_inner = 'grad_uniform_convex'
histories = []
labels = []
_, status, history_CN_power = \
cubic_newton(oracle, x_0, tolerance,
max_iters=max_iters,
H_0=H_0,
line_search=line_search,
inner_tolerance_strategy=power_strategy_3,
subsolver=subsolver,
trace=True,
stopping_criterion_subproblem=stopping_criterion_inner)
histories.append(history_CN_power)
labels.append(r'CN, $1/k^3$')
_, status, history_CN_adaptive = \
cubic_newton(oracle, x_0, tolerance,
max_iters=max_iters,
H_0=H_0,
line_search=line_search,
inner_tolerance_strategy=adaptive_strategy,
subsolver=subsolver,
trace=True,
stopping_criterion_subproblem=stopping_criterion_inner)
histories.append(history_CN_adaptive)
labels.append('CN, adaptive')
_, status, history_CN_averaging = \
cubic_newton(oracle, x_0, tolerance,
max_iters=max_iters,
H_0=H_0,
line_search=line_search,
inner_tolerance_strategy=power_strategy_3,
subsolver=subsolver,
trace=True,
stopping_criterion_subproblem=stopping_criterion_inner,
averaging=True)
histories.append(history_CN_averaging)
labels.append(r'Averaging, $1/k^3$')
if not line_search:
_, status, history_CN_contracting = \
contracting_cubic_newton(oracle, x_0, tolerance,
max_iters=max_iters,
H_0=H_0,
prox_steps_tolerance_strategy=
power_strategy_1,
newton_steps_tolerance_strategy=
power_strategy_1,
trace=True)
histories.append(history_CN_contracting)
labels.append(r'Contracting')
filename = os.getcwd() + '/plots/averaging_%d' % n
title = r'$n = %d$' % n
if line_search:
filename += '_ls'
title += ', line search'
plot_func_residual(histories,
None,
f_star,
labels, ['blue', 'red', 'tab:green', 'tab:purple'],
['-.', '-', '-', ':'], [3, 2, 5, 5], [0.6, 1, 0.8, 0.8],
title,
'Iterations',
filename=filename + '.pdf',
figsize=(5.5, 5))
def run_experiment(dataset_filename, name, max_iters):
print('Experiment: \t %s, \t file: %s, \t max_iters = %d.' %
(name, dataset_filename, max_iters))
X, y = load_svmlight_file(dataset_filename)
oracle = create_log_reg_oracle(X, y, 1 / X.shape[0])
x_0 = np.zeros(X.shape[1])
print('Minimize by scipy ... ', flush=True, end='')
f_star = \
scipy.optimize.minimize(oracle.func, x_0, jac=oracle.grad, tol=1e-9).fun
print('f_star = %g.' % f_star)
H_0 = 1.0
line_search = True
tolerance = get_tolerance({'criterion': 'func',
'f_star': f_star,
'tolerance': 1e-8})
subsolver = 'FGM'
stopping_criterion_subproblem = 'grad_uniform_convex'
constant_strategies = get_constant_strategies()
power_strategies = get_power_strategies()
adaptive_strategy = get_tolerance_strategy({'strategy': 'adaptive',
'c': 1.0,
'alpha': 1,
'label': 'adaptive'})
strategies_1 = constant_strategies + [adaptive_strategy]
strategies_2 = power_strategies + [adaptive_strategy]
method = lambda strategy: cubic_newton(oracle, x_0, tolerance,
max_iters=max_iters,
H_0=H_0,
line_search=line_search,
inner_tolerance_strategy=strategy,
subsolver=subsolver,
trace=True,
B=None,
Binv=None,
stopping_criterion_subproblem=
stopping_criterion_subproblem)
labels_1 = get_labels(strategies_1)
histories_1 = run_method(method, strategies_1, labels_1)
filename = os.getcwd() + '/plots/logreg_%s_time' % (name)
plot_func_residual(histories_1, 'time', f_star, labels_1,
['grey', 'grey', 'grey', 'grey', 'red'],
['-', '--', '-.', ':', '-'],
[5, 4, 3, 4, 2],
[0.8, 0.8, 0.8, 0.8, 1],
'Log-reg: %s' % name,
'Time, s',
filename=filename+'_const.pdf')
labels_2 = get_labels(strategies_2)
histories_2 = run_method(method, strategies_2, labels_2)
plot_func_residual(histories_2, 'time', f_star, labels_2,
['blue', 'blue', 'blue', 'blue', 'red'],
['-', '--', '-.', ':', '-'],
[5, 4, 3, 2, 2],
[0.6, 0.6, 0.6, 0.6, 1],
'Log-reg: %s' % name,
'Time, s',
filename=filename+'_powers.pdf')
def run_experiment(n, mu, max_iters):
print('Experiment: \t n = %d, \t mu = %g, \t max_iters = %d.' %
(n, mu, max_iters))
oracle, x_star, f_star, B, Binv = generate_logsumexp(n, mu)
x_0 = np.ones(n)
H_0 = 1.0
line_search = True
tolerance = get_tolerance({
'criterion': 'func',
'f_star': f_star,
'tolerance': 1e-8
})
subsolver = 'FGM'
stopping_criterion_subproblem = 'func'
constant_strategies = get_constant_strategies()
power_strategies = get_power_strategies()
adaptive_strategy = get_tolerance_strategy({
'strategy': 'adaptive',
'c': 1.0,
'alpha': 1,
'label': 'adaptive'
})
adaptive_15_strategy = get_tolerance_strategy({
'strategy': 'adaptive',
'c': 1.0,
'alpha': 1.5,
'label': r'adaptive $1.5$'
})
adaptive_2_strategy = get_tolerance_strategy({
'strategy': 'adaptive',
'c': 1.0,
'alpha': 2,
'label': r'adaptive $2$'
})
strategies_1 = constant_strategies
strategies_2 = power_strategies + [constant_strategies[-1]]
strategies_3 = [
adaptive_strategy, adaptive_15_strategy, adaptive_2_strategy,
constant_strategies[-1]
]
method = lambda strategy: cubic_newton(oracle,
x_0,
tolerance,
max_iters=max_iters,
H_0=H_0,
line_search=line_search,
inner_tolerance_strategy=strategy,
subsolver=subsolver,
trace=True,
B=B,
Binv=Binv,
stopping_criterion_subproblem=
stopping_criterion_subproblem)
mu_str = ('%g' % mu)[2:]
filename = os.getcwd() + '/plots/exact_logsumexp_%d_%s' % (n, mu_str)
labels_1 = get_labels(strategies_1)
histories_1 = run_method(method, strategies_1, labels_1)
plot_func_residual_iter(histories_1,
'hess_vec_calls',
f_star,
labels_1, ['grey', 'grey', 'grey', 'grey'],
['-', '--', '-.', ':'], [5, 4, 3, 4], [1, 1, 1, 1],
r'Log-sum-exp, $\mu = %g$: constant strategies' %
mu,
'Hessian-vector products',
filename=filename + '_const.pdf')
labels_2 = get_labels(strategies_2)
histories_2 = run_method(method, strategies_2, labels_2)
plot_func_residual_iter(histories_2,
'hess_vec_calls',
f_star,
labels_2, ['blue', 'blue', 'blue', 'blue', 'gray'],
['-', '--', '-.', ':', ':'], [5, 4, 3, 2, 4],
[0.6, 0.6, 0.6, 0.6, 0.8],
r'Log-sum-exp, $\mu = %g$: dynamic strategies' %
mu,
'Hessian-vector products',
filename=filename + '_power.pdf')
labels_3 = get_labels(strategies_3)
histories_3 = run_method(method, strategies_3, labels_3)
plot_func_residual_iter(
histories_3,
'hess_vec_calls',
f_star,
labels_3, ['red', 'tab:orange', 'tab:orange', 'gray'],
['-', '--', '-.', ':'], [2, 4, 2, 4], [1, 1, 1, 0.8],
r'Log-sum-exp, $\mu = %g$: adaptive strategies' % mu,
'Hessian-vector products',
filename=filename + '_adaptive.pdf')
def run_experiment(n, mu, max_iters):
print('Experiment: \t n = %d, \t mu = %g, \t max_iters = %d.' %
(n, mu, max_iters))
oracle, x_star, f_star, B, Binv = generate_logsumexp(n, mu)
x_0 = np.ones(n)
H_0 = 1.0
line_search = False
tolerance = get_tolerance({
'criterion': 'func',
'f_star': f_star,
'tolerance': 1e-8
})
subsolver = 'FGM'
stopping_criterion_subproblem = 'grad_uniform_convex'
constant_strategies = get_constant_strategies()
power_strategies = get_power_strategies()
adaptive_strategy = get_tolerance_strategy({
'strategy': 'adaptive',
'c': 1.0,
'alpha': 1,
'label': 'adaptive'
})
strategies_1 = constant_strategies + [adaptive_strategy]
strategies_2 = power_strategies + [adaptive_strategy]
method = lambda strategy: cubic_newton(oracle,
x_0,
tolerance,
max_iters=max_iters,
H_0=H_0,
line_search=line_search,
inner_tolerance_strategy=strategy,
subsolver=subsolver,
trace=True,
B=B,
Binv=Binv,
stopping_criterion_subproblem=
stopping_criterion_subproblem)
labels_1 = get_labels(strategies_1)
histories_1 = run_method(method, strategies_1, labels_1)
mu_str = ('%g' % mu)[2:]
filename = os.getcwd() + '/plots/logsumexp_%d_%s_time' % (n, mu_str)
plot_func_residual(histories_1,
'time',
f_star,
labels_1, ['grey', 'grey', 'grey', 'grey', 'red'],
['-', '--', '-.', ':', '-'], [5, 4, 3, 4, 2],
[0.8, 0.8, 0.8, 0.8, 1],
r'Log-sum-exp, $\mu = %g$' % mu,
'Time, s',
filename=filename + '_const.pdf')
labels_2 = get_labels(strategies_2)
histories_2 = run_method(method, strategies_2, labels_2)
plot_func_residual(histories_2,
'time',
f_star,
labels_2, ['blue', 'blue', 'blue', 'blue', 'red'],
['-', '--', '-.', ':', '-'], [5, 4, 3, 2, 2],
[0.6, 0.6, 0.6, 0.6, 1],
r'Log-sum-exp, $\mu = %g$' % mu,
'Time, s',
filename=filename + '_powers.pdf')
def run_experiment(dataset_filename, name, max_iters):
print('Experiment: \t %s, \t file: %s, \t max_iters = %d.' %
(name, dataset_filename, max_iters))
X, y = load_svmlight_file(dataset_filename)
oracle = create_log_reg_oracle(X, y, 1 / X.shape[0])
x_0 = np.zeros(X.shape[1])
print('Minimize by scipy ... ', flush=True, end='')
f_star = \
scipy.optimize.minimize(oracle.func, x_0, jac=oracle.grad, tol=1e-9).fun
print('f_star = %g.' % f_star)
H_0 = 1.0
line_search = True
tolerance = get_tolerance({
'criterion': 'func',
'f_star': f_star,
'tolerance': 1e-8
})
subsolver = 'FGM'
stopping_criterion_subproblem = 'func'
constant_strategies = get_constant_strategies()
power_strategies = get_power_strategies()
adaptive_strategy = get_tolerance_strategy({
'strategy': 'adaptive',
'c': 1.0,
'alpha': 1,
'label': 'adaptive'
})
adaptive_15_strategy = get_tolerance_strategy({
'strategy': 'adaptive',
'c': 1.0,
'alpha': 1.5,
'label': r'adaptive $1.5$'
})
adaptive_2_strategy = get_tolerance_strategy({
'strategy': 'adaptive',
'c': 1.0,
'alpha': 2,
'label': r'adaptive $2$'
})
strategies_1 = constant_strategies
strategies_2 = power_strategies + [constant_strategies[-1]]
strategies_3 = [
adaptive_strategy, adaptive_15_strategy, adaptive_2_strategy,
constant_strategies[-1]
]
method = lambda strategy: cubic_newton(oracle,
x_0,
tolerance,
max_iters=max_iters,
H_0=H_0,
line_search=line_search,
inner_tolerance_strategy=strategy,
subsolver=subsolver,
trace=True,
B=None,
Binv=None,
stopping_criterion_subproblem=
stopping_criterion_subproblem)
filename = os.getcwd() + '/plots/exact_logreg_%s' % (name)
labels_1 = get_labels(strategies_1)
histories_1 = run_method(method, strategies_1, labels_1)
plot_func_residual_iter(histories_1,
'hess_vec_calls',
f_star,
labels_1, ['grey', 'grey', 'grey', 'grey'],
['-', '--', '-.', ':'], [5, 4, 3, 4], [1, 1, 1, 1],
r'Log-reg, %s: constant strategies' % name,
'Hessian-vector products',
filename=filename + '_const.pdf')
labels_2 = get_labels(strategies_2)
histories_2 = run_method(method, strategies_2, labels_2)
plot_func_residual_iter(histories_2,
'hess_vec_calls',
f_star,
labels_2, ['blue', 'blue', 'blue', 'blue', 'gray'],
['-', '--', '-.', ':', ':'], [5, 4, 3, 2, 4],
[0.6, 0.6, 0.6, 0.6, 0.8],
r'Log-reg, %s: dynamic strategies' % name,
'Hessian-vector products',
filename=filename + '_power.pdf')
labels_3 = get_labels(strategies_3)
histories_3 = run_method(method, strategies_3, labels_3)
plot_func_residual_iter(histories_3,
'hess_vec_calls',
f_star,
labels_3,
['red', 'tab:orange', 'tab:orange', 'gray'],
['-', '--', '-.', ':'], [2, 4, 2, 4],
[1, 1, 1, 0.8],
r'Log-reg, %s: adaptive strategies' % name,
'Hessian-vector products',
filename=filename + '_adaptive.pdf')