def __init__(self, step_size_getter=None):
super(SteepestDescent, self).__init__()
if step_size_getter is None:
step_size_getter = WolfeLineSearch(
initial_step_getter=FOChangeInitialStep())
self._step_size_getter = step_size_getter
def __init__(self,
step_size_getter=None,
initial_hessian_func=initial_hessian_identity):
super(BFGS, self).__init__()
if step_size_getter is None:
step_size_getter = WolfeLineSearch(
# Values recommended by Numerical Optimization 2nd, pp. 161
c_1=1e-4,
c_2=0.9,
initial_step_getter=IncrPrevStep())
self._step_size_getter = step_size_getter
# NOTE: It is recommended to scale the identiy matrix
# (Numerical Optimization 2nd, pp. 162)
# as performed in initial_hessian_scaled_identity,
# but empirical tests show it is more effective
# to not scale identity matrix
# TODO: More testing needed
self._initial_hessian_func = initial_hessian_func
# BFGS Parameters
self._prev_step = None
self._prev_jacobian = None
self._prev_inv_hessian = None
def __init__(self,
step_size_getter=None,
initial_hessian_func=initial_hessian_identity,
iterations_per_reset=100):
super(BFGS, self).__init__()
if step_size_getter is None:
step_size_getter = WolfeLineSearch(
# Values recommended by Numerical Optimization 2nd, pp. 161
c_1=1e-4,
c_2=0.9,
initial_step_getter=FOChangeInitialStep())
self._step_size_getter = step_size_getter
# NOTE: It is recommended to scale the identiy matrix
# (Numerical Optimization 2nd, pp. 162)
# as performed in initial_hessian_scaled_identity,
# but empirical tests show it is more effective
# to not scale identity matrix
# TODO: More testing needed
self._initial_hessian_func = initial_hessian_func
# Hessian approximation can become inaccurate
# after many iterations on non-convex problems.
# Periodically resetting can fix this.
self._iterations_per_reset = iterations_per_reset
# BFGS Parameters
self._iteration = 0
self._prev_step = None
self._prev_jacobian = None
self._prev_inv_hessian = None
def __init__(self, step_size_getter=None, momentum_rate=0.2):
super(SteepestDescentMomentum, self).__init__()
if step_size_getter is None:
step_size_getter = WolfeLineSearch(
initial_step_getter=FOChangeInitialStep())
self._step_size_getter = step_size_getter
self._momentum_rate = momentum_rate
# Store previous step (step_size*direction) for momentum
self._prev_step = None
def test_LBFGS_approx_equal_BFGS_infinite_num_remembered_iterations():
# When LBFGS has no limit on remembered iterations, it should approximately
# equal BFGS, given initial hessian is the same on all iterations
# "During its first m - 1 iterations,
# Algorithm 7.5 is equivalent to the BFGS algorithm of Chapter 6
# if the initial matrix H_0 is the same in both methods,
# and if L-BFGS chooses H_0^k = H_0 at each iteration."
# ~ Numerical Optimization 2nd pp. 179
# Rosenbrock function
f = lambda vec: 100.0 * (vec[1] - vec[0]**2)**2 + (vec[0] - 1.0)**2
df = lambda vec: numpy.array([
2.0 *
(200.0 * vec[0]**3 - 200.0 * vec[0] * vec[1] + vec[0] - 1.0), 200.0 *
(vec[1] - vec[0]**2)
])
problem = Problem(obj_func=f, jac_func=df)
# Optimize
bfgs_vec = numpy.random.random(2)
lbfgs_vec = numpy.copy(bfgs_vec)
# Same identity hessian, for both optimizers
bfgs_optimizer = BFGS(
step_size_getter=WolfeLineSearch(),
initial_hessian_func=optimizer.initial_hessian_identity)
lbfgs_optimizer = LBFGS(
step_size_getter=WolfeLineSearch(),
num_remembered_iterations=float('inf'),
initial_hessian_scalar_func=optimizer.initial_hessian_one_scalar)
for i in range(10):
_, bfgs_vec = bfgs_optimizer.next(problem, bfgs_vec)
_, lbfgs_vec = lbfgs_optimizer.next(problem, lbfgs_vec)
print i
assert helpers.approx_equal(bfgs_vec, lbfgs_vec)
def __init__(self, step_size_getter=None):
super(BFGS, self).__init__()
if step_size_getter is None:
step_size_getter = WolfeLineSearch(
# Values recommended by Numerical Optimization 2nd, pp. 161
c_1=1e-4,
c_2=0.9,
initial_step_getter=IncrPrevStep())
self._step_size_getter = step_size_getter
# BFGS Parameters
self._prev_params = None
self._prev_jacobian = None
self._prev_inv_hessian = None
def __init__(self,
step_size_getter=None,
num_remembered_iterations=5,
initial_hessian_scalar_func=initial_hessian_gamma_scalar):
super(LBFGS, self).__init__()
if step_size_getter is None:
step_size_getter = WolfeLineSearch(
# Values recommended by Numerical Optimization 2nd, pp. 161
c_1=1e-4,
c_2=0.9,
initial_step_getter=IncrPrevStep())
self._step_size_getter = step_size_getter
self._initial_hessian_scalar_func = initial_hessian_scalar_func
# L-BFGS Parameters
self._num_remembered_iterations = num_remembered_iterations
self._prev_step = None
self._prev_jacobian = None
self._prev_param_diffs = [] # Previous s_k values
self._prev_jac_diffs = [] # Previous y_k values
def test_LBFGS_wolfe_line_search():
check_optimize_sphere_function(LBFGS(step_size_getter=WolfeLineSearch()))
def test_steepest_descent_wolfe_line_search():
check_optimize_sphere_function(
SteepestDescent(step_size_getter=WolfeLineSearch()))