def line_search(self, oracle, x_k, d_k, previous_alpha=None):
"""
Finds the step size alpha for a given starting point x_k
and for a given search direction d_k that satisfies necessary
conditions for phi(alpha) = oracle.func(x_k + alpha * d_k).
Parameters
----------
oracle : BaseSmoothOracle-descendant object
Oracle with .func_directional() and .grad_directional() methods implemented for computing
function values and its directional derivatives.
x_k : np.array
Starting point
d_k : np.array
Search direction
previous_alpha : float or None
Starting point to use instead of self.alpha_0 to keep the progress from
previous steps. If None, self.alpha_0, is used as a starting point.
Returns
-------
alpha : float or None if failure
Chosen step size
"""
# TODO: Implement line search procedures for Armijo, Wolfe and Constant steps.
try:
phi = lambda alpha: oracle.func_directional(x_k, d_k, alpha)
grad_phi = lambda alpha: oracle.grad_directional(x_k, d_k, alpha)
if self._method == 'Wolfe':
#Wolfe
alpha, _, _, _ = scalar_search_wolfe2(phi,
derphi=grad_phi,
phi0=phi(0),
derphi0=grad_phi(0),
c1=self.c1,
c2=self.c2)
print('Wolfe, alpha', alpha)
if (alpha):
return alpha
if self._method == 'Wolfe' or self._method == 'Armijo':
c = self.c1
#TODO if Newton alpha = 1
if not previous_alpha:
alpha = self.alpha_0
else:
alpha = previous_alpha
#Armijo
while phi(alpha) > phi(0) + c * alpha * grad_phi(
0): #could replace phi(0) with func; is grad = phi'()?
alpha /= float(2)
print('Armijo, alpha', alpha)
return alpha
if self._method == 'Constant':
return self.c
else:
return None
except:
return None
def test_scalar_search_wolfe2(self):
for name, phi, derphi, old_phi0 in self.scalar_iter():
s, phi1, phi0, derphi1 = ls.scalar_search_wolfe2(
phi, derphi, phi(0), old_phi0, derphi(0))
assert_equal(phi0, phi(0), name)
assert_equal(phi1, phi(s), name)
if derphi1 is not None:
assert_equal(derphi1, derphi(s), name)
assert_wolfe(s, phi, derphi, err_msg="%s %g" % (name, old_phi0))
def test_scalar_search_wolfe2(self):
for name, phi, derphi, old_phi0 in self.scalar_iter():
s, phi1, phi0, derphi1 = ls.scalar_search_wolfe2(
phi, derphi, phi(0), old_phi0, derphi(0))
assert_fp_equal(phi0, phi(0), name)
assert_fp_equal(phi1, phi(s), name)
if derphi1 is not None:
assert_fp_equal(derphi1, derphi(s), name)
assert_wolfe(s, phi, derphi, err_msg="%s %g" % (name, old_phi0))
def line_search(self, oracle, x_k, d_k, previous_alpha=None):
"""
Finds the step size alpha for a given starting point x_k
and for a given search direction d_k that satisfies necessary
conditions for phi(alpha) = oracle.func(x_k + alpha * d_k).
Parameters
----------
oracle : BaseSmoothOracle-descendant object
Oracle with .func_directional() and .grad_directional() methods implemented for computing
function values and its directional derivatives.
x_k : np.array
Starting point
d_k : np.array
Search direction
previous_alpha : float or None
Starting point to use instead of self.alpha_0 to keep the progress from
previous steps. If None, self.alpha_0, is used as a starting point.
Returns
-------
alpha : float or None if failure
Chosen step size
"""
# TODO: Implement line search procedures for Armijo, Wolfe and Constant steps.
if self._method == 'Constant':
return self.c
elif self._method == 'Armijo':
alpha = self.alpha_0 if not previous_alpha else previous_alpha
phi_0 = oracle.func(x_k)
phi_der_0 = oracle.grad_directional(x_k, d_k, 0)
while oracle.func_directional(x_k, d_k, alpha) > phi_0 + self.c1 * alpha * phi_der_0:
alpha /= 2
return alpha
elif self._method == 'Wolfe':
phi_0 = oracle.func(x_k)
phi_der_0 = oracle.grad_directional(x_k, d_k, 0)
phi = lambda alpha: oracle.func_directional(x_k, d_k, alpha)
derphi = lambda alpha: oracle.grad_directional(x_k, d_k, alpha)
alpha_wolfe, _, _, _ = scalar_search_wolfe2(phi, derphi, phi_0, None, phi_der_0, self.c1, self.c2)
if alpha_wolfe:
return alpha_wolfe
else:
alpha = self.alpha_0 if not previous_alpha else previous_alpha
phi_0 = oracle.func(x_k)
phi_der_0 = oracle.grad(x_k) @ d_k
while oracle.func_directional(x_k, d_k, alpha) > phi_0 + self.c1 * alpha * phi_der_0:
alpha /= 2
return alpha
return None
def line_search_wolfe2(
state: OptimisationState,
old_state: Optional[OptimisationState] = None,
c1=1e-4,
c2=0.9,
amax=None,
extra_condition=None,
maxiter=10,
**kwargs,
) -> Tuple[Optional[float], OptimisationState]:
"""
As `scalar_search_wolfe1` but do a line search to direction `pk`
Parameters
----------
f : callable
Function `f(x)`
fprime : callable
Gradient of `f`
xk : array_like
Current point
pk : array_like
Search direction
gk : array_like, optional
Gradient of `f` at point `xk`
old_fval : float, optional
Value of `f` at point `xk`
old_old_fval : float, optional
Value of `f` at point preceding `xk`
The rest of the parameters are the same as for `scalar_search_wolfe1`.
Returns
-------
stp, f_count, g_count, fval, old_fval
As in `line_search_wolfe1`
gval : array
Gradient of `f` at the final point
"""
derphi0 = state.derphi(0)
old_fval = state.value
stepsize, _, _, _ = linesearch.scalar_search_wolfe2(
state.phi,
state.derphi,
-old_fval, # we are actually performing maximisation
old_state and -old_state.value,
derphi0,
c1=c1,
c2=c2,
amax=amax,
maxiter=maxiter,
)
next_state = state.step(stepsize)
if stepsize is not None and extra_condition is not None:
if not extra_condition(stepsize, next_state):
stepsize = None
return stepsize, next_state
def line_search(self, oracle, x_k, d_k, previous_alpha=None):
"""
Finds the step size alpha for a given starting point x_k
and for a given search direction d_k that satisfies necessary
conditions for phi(alpha) = oracle.func(x_k + alpha * d_k).
Parameters
----------
oracle : BaseSmoothOracle-descendant object
Oracle with .func_directional() and .grad_directional() methods implemented for computing
function values and its directional derivatives.
x_k : np.array
Starting point
d_k : np.array
Search direction
previous_alpha : float or None
Starting point to use instead of self.alpha_0 to keep the progress from
previous steps. If None, self.alpha_0, is used as a starting point.
Returns
-------
alpha : float or None if failure
Chosen step size
"""
# TODO: BONUS: Also fallback into oracle.minimize_directional() if method == 'Best'
phi = lambda a: oracle.func_directional(x_k, d_k, a)
derphi = lambda a: oracle.grad_directional(x_k, d_k, a)
phi0, derphi0 = phi(0), derphi(0)
if self._method == 'Constant':
return self.c
if self._method == 'Wolfe':
alpha, _, _, _ = scalar_search_wolfe2(phi=phi,
derphi=derphi,
phi0=phi0,
derphi0=derphi0,
c1=self.c1,
c2=self.c2)
if alpha:
return alpha
else:
return LineSearchTool(method='Armijo',
c1=self.c1,
alpha=self.alpha_0).line_search(
oracle, x_k, d_k, previous_alpha)
if self._method == 'Armijo':
alpha = previous_alpha if previous_alpha else self.alpha_0
while phi(alpha) > phi0 + self.c1 * alpha * derphi0:
alpha /= 2
return alpha
def line_search(self, oracle, x_k, d_k, previous_alpha=None):
"""
Finds the step size alpha for a given starting point x_k
and for a given search direction d_k that satisfies necessary
conditions for phi(alpha) = oracle.func(x_k + alpha * d_k).
Parameters
----------
oracle : BaseSmoothOracle-descendant object
Oracle with .func_directional() and .grad_directional() methods implemented for computing
function values and its directional derivatives.
x_k : np.array
Starting point
d_k : np.array
Search direction
previous_alpha : float or None
Starting point to use instead of self.alpha_0 to keep the progress from
previous steps. If None, self.alpha_0, is used as a starting point.
Returns
-------
alpha : float or None if failure
Chosen step size
"""
phi = lambda a: oracle.func_directional(x_k, d_k, a)
dphi = lambda a: oracle.grad_directional(x_k, d_k, a)
if self._method != 'Constant':
alpha = self.alpha_0 if previous_alpha is None else previous_alpha
def backtracking(a_0):
phi_0 = phi(0)
dphi_0 = dphi(0)
while phi(a_0) > phi_0 + self.c1 * a_0 * dphi_0:
a_0 /= 2
return a_0
if self._method == 'Armijo':
return backtracking(alpha)
elif self._method == 'Wolfe':
a_wolf, b, bb, bbb = ls.scalar_search_wolfe2(phi,
derphi=dphi,
c1=self.c1,
c2=self.c2)
if a_wolf is None:
return backtracking(alpha)
else:
return a_wolf
elif self._method == 'Constant':
return self.c
return None
def line_search(self, oracle, x_k, d_k, previous_alpha=None):
"""
Finds the step size alpha for a given starting point x_k
and for a given search direction d_k that satisfies necessary
conditions for phi(alpha) = oracle.func(x_k + alpha * d_k).
Parameters
----------
oracle : BaseSmoothOracle-descendant object
Oracle with .func_directional() and .grad_directional() methods implemented for computing
function values and its directional derivatives.
x_k : np.array
Starting point
d_k : np.array
Search direction
previous_alpha : float or None
Starting point to use instead of self.alpha_0 to keep the progress from
previous steps. If None, self.alpha_0, is used as a starting point.
Returns
-------
alpha : float or None if failure
Chosen step size
"""
phi = lambda alpha: oracle.func_directional(x_k, d_k, alpha)
derphi = lambda alpha: oracle.grad_directional(x_k, d_k, alpha)
def backtracking(phi, derphi, c1, alpha0=1):
alpha = alpha0
while phi(alpha) > phi(0) + c1 * alpha * derphi(0):
alpha /= 2
return alpha
if self._method == 'Constant':
return self.c
elif self._method == 'Armijo':
previous_alpha = previous_alpha or self.alpha_0
return backtracking(phi, derphi, self.c1, previous_alpha)
elif self._method == 'Wolfe':
best_alpha = scalar_search_wolfe2(phi,
derphi,
c1=self.c1,
c2=self.c2)[0]
return best_alpha if best_alpha is not None else backtracking(
phi=phi, derphi=derphi, c1=self.c1)
def line_search(self, oracle, x_k, d_k, previous_alpha=None):
if self._method == 'Constant':
return self.c
elif self._method == 'Armijo':
alpha_0 = previous_alpha if previous_alpha is not None else self.alpha_0
return self.armijo_search(oracle, x_k, d_k, alpha_0)
elif self._method == 'Wolfe':
alpha = scalar_search_wolfe2(
phi=lambda step: oracle.func_directional(x_k, d_k, step),
derphi=lambda step: oracle.grad_directional(x_k, d_k, step),
c1=self.c1,
c2=self.c2)[0]
if alpha is None:
return self.armijo_search(oracle, x_k, d_k, self.alpha_0)
else:
return alpha
return None
def test_scalar_search_wolfe2_regression(self):
# Regression test for gh-12157
# This phi has its minimum at alpha=4/3 ~ 1.333.
def phi(alpha):
if alpha < 1:
return -3 * np.pi / 2 * (alpha - 1)
else:
return np.cos(3 * np.pi / 2 * alpha - np.pi)
def derphi(alpha):
if alpha < 1:
return -3 * np.pi / 2
else:
return -3 * np.pi / 2 * np.sin(3 * np.pi / 2 * alpha - np.pi)
s, _, _, _ = ls.scalar_search_wolfe2(phi, derphi)
# Without the fix in gh-13073, the scalar_search_wolfe2
# returned s=2.0 instead.
assert s < 1.5
def line_search(self, oracle, x_k, d_k, previous_alpha=None):
"""
Finds the step size alpha for a given starting point x_k
and for a given search direction d_k that satisfies necessary
conditions for phi(alpha) = oracle.func(x_k + alpha * d_k).
Parameters
----------
oracle : BaseSmoothOracle-descendant object
Oracle with .func_directional() and .grad_directional() methods implemented for computing
function values and its directional derivatives.
x_k : np.array
Starting point
d_k : np.array
Search direction
previous_alpha : float or None
Starting point to use instead of self.alpha_0 to keep the progress from
previous steps. If None, self.alpha_0, is used as a starting point.
Returns
-------
alpha : float or None if failure
Chosen step size
"""
# TODO: Implement line search procedures for Armijo, Wolfe and Constant steps.
from scipy.optimize.linesearch import scalar_search_wolfe2
phi = lambda a: oracle.func_directional(x_k, d_k, a)
derphi = lambda a: oracle.grad_directional(x_k, d_k, a)
if self._method == "Wolfe":
alpha = scalar_search_wolfe2(phi, derphi, c1 = self.c1, c2 = self.c2)[0]
if alpha == None:
alpha = self.armijo_linesearch(phi, derphi, previous_alpha)
elif self._method == "Armijo":
alpha = self.armijo_linesearch(phi, derphi, previous_alpha)
else:
alpha = self.c
return alpha
def strong_wolfe_line_search(
phi,
derphi,
phi0=None,
old_phi0=None,
derphi0=None,
c1=1e-4,
c2=0.9,
amax=None,
**kwargs,
):
"""
Scalar line search method to find step size satisfying strong Wolfe conditions.
Parameters
----------
c1 : float, optional
Parameter for Armijo condition rule.
c2 : float, optional
Parameter for curvature condition rule.
amax : float, optional
Maximum step size
Returns
-------
step_size : float
The next step size
"""
step_size, _, _, _ = scalar_search_wolfe2(
phi,
derphi,
phi0=phi0,
old_phi0=old_phi0,
c1=c1,
c2=c2,
amax=amax,
)
return step_size
