def line_search(self, oracle, x_k, d_k, previous_alpha=None): """ Finds the step size alpha for a given starting point x_k and for a given search direction d_k that satisfies necessary conditions for phi(alpha) = oracle.func(x_k + alpha * d_k). Parameters ---------- oracle : BaseSmoothOracle-descendant object Oracle with .func_directional() and .grad_directional() methods implemented for computing function values and its directional derivatives. x_k : np.array Starting point d_k : np.array Search direction previous_alpha : float or None Starting point to use instead of self.alpha_0 to keep the progress from previous steps. If None, self.alpha_0, is used as a starting point. Returns ------- alpha : float or None if failure Chosen step size """ # TODO: Implement line search procedures for Armijo, Wolfe and Constant steps. try: phi = lambda alpha: oracle.func_directional(x_k, d_k, alpha) grad_phi = lambda alpha: oracle.grad_directional(x_k, d_k, alpha) if self._method == 'Wolfe': #Wolfe alpha, _, _, _ = scalar_search_wolfe2(phi, derphi=grad_phi, phi0=phi(0), derphi0=grad_phi(0), c1=self.c1, c2=self.c2) print('Wolfe, alpha', alpha) if (alpha): return alpha if self._method == 'Wolfe' or self._method == 'Armijo': c = self.c1 #TODO if Newton alpha = 1 if not previous_alpha: alpha = self.alpha_0 else: alpha = previous_alpha #Armijo while phi(alpha) > phi(0) + c * alpha * grad_phi( 0): #could replace phi(0) with func; is grad = phi'()? alpha /= float(2) print('Armijo, alpha', alpha) return alpha if self._method == 'Constant': return self.c else: return None except: return None
def test_scalar_search_wolfe2(self): for name, phi, derphi, old_phi0 in self.scalar_iter(): s, phi1, phi0, derphi1 = ls.scalar_search_wolfe2( phi, derphi, phi(0), old_phi0, derphi(0)) assert_equal(phi0, phi(0), name) assert_equal(phi1, phi(s), name) if derphi1 is not None: assert_equal(derphi1, derphi(s), name) assert_wolfe(s, phi, derphi, err_msg="%s %g" % (name, old_phi0))
def test_scalar_search_wolfe2(self): for name, phi, derphi, old_phi0 in self.scalar_iter(): s, phi1, phi0, derphi1 = ls.scalar_search_wolfe2( phi, derphi, phi(0), old_phi0, derphi(0)) assert_fp_equal(phi0, phi(0), name) assert_fp_equal(phi1, phi(s), name) if derphi1 is not None: assert_fp_equal(derphi1, derphi(s), name) assert_wolfe(s, phi, derphi, err_msg="%s %g" % (name, old_phi0))
def line_search(self, oracle, x_k, d_k, previous_alpha=None): """ Finds the step size alpha for a given starting point x_k and for a given search direction d_k that satisfies necessary conditions for phi(alpha) = oracle.func(x_k + alpha * d_k). Parameters ---------- oracle : BaseSmoothOracle-descendant object Oracle with .func_directional() and .grad_directional() methods implemented for computing function values and its directional derivatives. x_k : np.array Starting point d_k : np.array Search direction previous_alpha : float or None Starting point to use instead of self.alpha_0 to keep the progress from previous steps. If None, self.alpha_0, is used as a starting point. Returns ------- alpha : float or None if failure Chosen step size """ # TODO: Implement line search procedures for Armijo, Wolfe and Constant steps. if self._method == 'Constant': return self.c elif self._method == 'Armijo': alpha = self.alpha_0 if not previous_alpha else previous_alpha phi_0 = oracle.func(x_k) phi_der_0 = oracle.grad_directional(x_k, d_k, 0) while oracle.func_directional(x_k, d_k, alpha) > phi_0 + self.c1 * alpha * phi_der_0: alpha /= 2 return alpha elif self._method == 'Wolfe': phi_0 = oracle.func(x_k) phi_der_0 = oracle.grad_directional(x_k, d_k, 0) phi = lambda alpha: oracle.func_directional(x_k, d_k, alpha) derphi = lambda alpha: oracle.grad_directional(x_k, d_k, alpha) alpha_wolfe, _, _, _ = scalar_search_wolfe2(phi, derphi, phi_0, None, phi_der_0, self.c1, self.c2) if alpha_wolfe: return alpha_wolfe else: alpha = self.alpha_0 if not previous_alpha else previous_alpha phi_0 = oracle.func(x_k) phi_der_0 = oracle.grad(x_k) @ d_k while oracle.func_directional(x_k, d_k, alpha) > phi_0 + self.c1 * alpha * phi_der_0: alpha /= 2 return alpha return None
def line_search_wolfe2( state: OptimisationState, old_state: Optional[OptimisationState] = None, c1=1e-4, c2=0.9, amax=None, extra_condition=None, maxiter=10, **kwargs, ) -> Tuple[Optional[float], OptimisationState]: """ As `scalar_search_wolfe1` but do a line search to direction `pk` Parameters ---------- f : callable Function `f(x)` fprime : callable Gradient of `f` xk : array_like Current point pk : array_like Search direction gk : array_like, optional Gradient of `f` at point `xk` old_fval : float, optional Value of `f` at point `xk` old_old_fval : float, optional Value of `f` at point preceding `xk` The rest of the parameters are the same as for `scalar_search_wolfe1`. Returns ------- stp, f_count, g_count, fval, old_fval As in `line_search_wolfe1` gval : array Gradient of `f` at the final point """ derphi0 = state.derphi(0) old_fval = state.value stepsize, _, _, _ = linesearch.scalar_search_wolfe2( state.phi, state.derphi, -old_fval, # we are actually performing maximisation old_state and -old_state.value, derphi0, c1=c1, c2=c2, amax=amax, maxiter=maxiter, ) next_state = state.step(stepsize) if stepsize is not None and extra_condition is not None: if not extra_condition(stepsize, next_state): stepsize = None return stepsize, next_state
def line_search(self, oracle, x_k, d_k, previous_alpha=None): """ Finds the step size alpha for a given starting point x_k and for a given search direction d_k that satisfies necessary conditions for phi(alpha) = oracle.func(x_k + alpha * d_k). Parameters ---------- oracle : BaseSmoothOracle-descendant object Oracle with .func_directional() and .grad_directional() methods implemented for computing function values and its directional derivatives. x_k : np.array Starting point d_k : np.array Search direction previous_alpha : float or None Starting point to use instead of self.alpha_0 to keep the progress from previous steps. If None, self.alpha_0, is used as a starting point. Returns ------- alpha : float or None if failure Chosen step size """ # TODO: BONUS: Also fallback into oracle.minimize_directional() if method == 'Best' phi = lambda a: oracle.func_directional(x_k, d_k, a) derphi = lambda a: oracle.grad_directional(x_k, d_k, a) phi0, derphi0 = phi(0), derphi(0) if self._method == 'Constant': return self.c if self._method == 'Wolfe': alpha, _, _, _ = scalar_search_wolfe2(phi=phi, derphi=derphi, phi0=phi0, derphi0=derphi0, c1=self.c1, c2=self.c2) if alpha: return alpha else: return LineSearchTool(method='Armijo', c1=self.c1, alpha=self.alpha_0).line_search( oracle, x_k, d_k, previous_alpha) if self._method == 'Armijo': alpha = previous_alpha if previous_alpha else self.alpha_0 while phi(alpha) > phi0 + self.c1 * alpha * derphi0: alpha /= 2 return alpha
def line_search(self, oracle, x_k, d_k, previous_alpha=None): """ Finds the step size alpha for a given starting point x_k and for a given search direction d_k that satisfies necessary conditions for phi(alpha) = oracle.func(x_k + alpha * d_k). Parameters ---------- oracle : BaseSmoothOracle-descendant object Oracle with .func_directional() and .grad_directional() methods implemented for computing function values and its directional derivatives. x_k : np.array Starting point d_k : np.array Search direction previous_alpha : float or None Starting point to use instead of self.alpha_0 to keep the progress from previous steps. If None, self.alpha_0, is used as a starting point. Returns ------- alpha : float or None if failure Chosen step size """ phi = lambda a: oracle.func_directional(x_k, d_k, a) dphi = lambda a: oracle.grad_directional(x_k, d_k, a) if self._method != 'Constant': alpha = self.alpha_0 if previous_alpha is None else previous_alpha def backtracking(a_0): phi_0 = phi(0) dphi_0 = dphi(0) while phi(a_0) > phi_0 + self.c1 * a_0 * dphi_0: a_0 /= 2 return a_0 if self._method == 'Armijo': return backtracking(alpha) elif self._method == 'Wolfe': a_wolf, b, bb, bbb = ls.scalar_search_wolfe2(phi, derphi=dphi, c1=self.c1, c2=self.c2) if a_wolf is None: return backtracking(alpha) else: return a_wolf elif self._method == 'Constant': return self.c return None
def line_search(self, oracle, x_k, d_k, previous_alpha=None): """ Finds the step size alpha for a given starting point x_k and for a given search direction d_k that satisfies necessary conditions for phi(alpha) = oracle.func(x_k + alpha * d_k). Parameters ---------- oracle : BaseSmoothOracle-descendant object Oracle with .func_directional() and .grad_directional() methods implemented for computing function values and its directional derivatives. x_k : np.array Starting point d_k : np.array Search direction previous_alpha : float or None Starting point to use instead of self.alpha_0 to keep the progress from previous steps. If None, self.alpha_0, is used as a starting point. Returns ------- alpha : float or None if failure Chosen step size """ phi = lambda alpha: oracle.func_directional(x_k, d_k, alpha) derphi = lambda alpha: oracle.grad_directional(x_k, d_k, alpha) def backtracking(phi, derphi, c1, alpha0=1): alpha = alpha0 while phi(alpha) > phi(0) + c1 * alpha * derphi(0): alpha /= 2 return alpha if self._method == 'Constant': return self.c elif self._method == 'Armijo': previous_alpha = previous_alpha or self.alpha_0 return backtracking(phi, derphi, self.c1, previous_alpha) elif self._method == 'Wolfe': best_alpha = scalar_search_wolfe2(phi, derphi, c1=self.c1, c2=self.c2)[0] return best_alpha if best_alpha is not None else backtracking( phi=phi, derphi=derphi, c1=self.c1)
def line_search(self, oracle, x_k, d_k, previous_alpha=None): if self._method == 'Constant': return self.c elif self._method == 'Armijo': alpha_0 = previous_alpha if previous_alpha is not None else self.alpha_0 return self.armijo_search(oracle, x_k, d_k, alpha_0) elif self._method == 'Wolfe': alpha = scalar_search_wolfe2( phi=lambda step: oracle.func_directional(x_k, d_k, step), derphi=lambda step: oracle.grad_directional(x_k, d_k, step), c1=self.c1, c2=self.c2)[0] if alpha is None: return self.armijo_search(oracle, x_k, d_k, self.alpha_0) else: return alpha return None
def test_scalar_search_wolfe2_regression(self): # Regression test for gh-12157 # This phi has its minimum at alpha=4/3 ~ 1.333. def phi(alpha): if alpha < 1: return -3 * np.pi / 2 * (alpha - 1) else: return np.cos(3 * np.pi / 2 * alpha - np.pi) def derphi(alpha): if alpha < 1: return -3 * np.pi / 2 else: return -3 * np.pi / 2 * np.sin(3 * np.pi / 2 * alpha - np.pi) s, _, _, _ = ls.scalar_search_wolfe2(phi, derphi) # Without the fix in gh-13073, the scalar_search_wolfe2 # returned s=2.0 instead. assert s < 1.5
def line_search(self, oracle, x_k, d_k, previous_alpha=None): """ Finds the step size alpha for a given starting point x_k and for a given search direction d_k that satisfies necessary conditions for phi(alpha) = oracle.func(x_k + alpha * d_k). Parameters ---------- oracle : BaseSmoothOracle-descendant object Oracle with .func_directional() and .grad_directional() methods implemented for computing function values and its directional derivatives. x_k : np.array Starting point d_k : np.array Search direction previous_alpha : float or None Starting point to use instead of self.alpha_0 to keep the progress from previous steps. If None, self.alpha_0, is used as a starting point. Returns ------- alpha : float or None if failure Chosen step size """ # TODO: Implement line search procedures for Armijo, Wolfe and Constant steps. from scipy.optimize.linesearch import scalar_search_wolfe2 phi = lambda a: oracle.func_directional(x_k, d_k, a) derphi = lambda a: oracle.grad_directional(x_k, d_k, a) if self._method == "Wolfe": alpha = scalar_search_wolfe2(phi, derphi, c1 = self.c1, c2 = self.c2)[0] if alpha == None: alpha = self.armijo_linesearch(phi, derphi, previous_alpha) elif self._method == "Armijo": alpha = self.armijo_linesearch(phi, derphi, previous_alpha) else: alpha = self.c return alpha
def strong_wolfe_line_search( phi, derphi, phi0=None, old_phi0=None, derphi0=None, c1=1e-4, c2=0.9, amax=None, **kwargs, ): """ Scalar line search method to find step size satisfying strong Wolfe conditions. Parameters ---------- c1 : float, optional Parameter for Armijo condition rule. c2 : float, optional Parameter for curvature condition rule. amax : float, optional Maximum step size Returns ------- step_size : float The next step size """ step_size, _, _, _ = scalar_search_wolfe2( phi, derphi, phi0=phi0, old_phi0=old_phi0, c1=c1, c2=c2, amax=amax, ) return step_size