def _reconstruct(self, observation, out): observation = self.observation_space.element(observation) out[:] = self.x0 if self.op_is_symmetric: iterative.conjugate_gradient(self.op, out, observation, self.niter, self.callback) else: iterative.conjugate_gradient_normal(self.op, out, observation, self.niter, self.callback) return out
def _reconstruct(self, observation, out): observation = self.observation_space.element(observation) out_ = out if out not in self.reco_space: out_ = self.reco_space.zero() out_[:] = self.x0 if self.op_is_symmetric: iterative.conjugate_gradient(self.op, out_, observation, self.niter, self.callback) else: iterative.conjugate_gradient_normal(self.op, out_, observation, self.niter, self.callback) if out not in self.reco_space: out[:] = out_ return out
def newtons_method(op, x, line_search, num_iter=10, cg_iter=None, partial=None): """Newton's method for solving a system of equations. This is a general and optimized implementation of Newton's method for solving the problem:: f(x) = 0 of finding a root of a function. The algorithm is well-known and there is a vast literature about it. Among others, the method is described in [BV2004]_, Sections 9.5 and 10.2 (`book available online <http://stanford.edu/~boyd/cvxbook/bv_cvxbook.pdf>`_), [GNS2009]_, Section 2.7 for solving nonlinear equations and Section 11.3 for its use in minimization, and wikipedia on `Newton's_method <https://en.wikipedia.org/wiki/Newton's_method>`_. Parameters ---------- op : `Operator` Gradient of the objective function, ``x --> grad f(x)`` x : element in the domain of ``op`` Starting point of the iteration line_search : `LineSearch` Strategy to choose the step length num_iter : `int`, optional Number of iterations cg_iter : `int`, optional Number of iterations in the the conjugate gradient solver, for computing the search direction. partial : `Partial`, optional Object executing code per iteration, e.g. plotting each iterate Notes ---------- The algorithm works by iteratively solving :math:`\partial f(x_k)p_k = -f(x_k)` and then updating as :math:`x_{k+1} = x_k + \\alpha x_k`, where :math:`\\alpha` is a suitable step length (see the references). In this implementation the system of equations are solved using the conjugate gradient method. """ # TODO: update doc if cg_iter is None: # Motivated by that if it is Ax = b, x and b in Rn, it takes at most n # iterations to solve with cg cg_iter = op.domain.size # TODO: optimize by using lincomb and avoiding to create copies for _ in range(num_iter): # Initialize the search direction to 0 search_direction = x.space.zero() # Compute hessian (as operator) and gradient in the current point hessian = op.derivative(x) deriv_in_point = op(x).copy() # Solving A*x = b for x, in this case f'(x)*p = -f(x) # TODO: Let the user provide/choose method for how to solve this? conjugate_gradient(hessian, search_direction, -1 * deriv_in_point, cg_iter) # Computing step length dir_deriv = search_direction.inner(deriv_in_point) step_length = line_search(x, search_direction, dir_deriv) # Updating x += step_length * search_direction if partial is not None: partial(x)
def newtons_method(f, x, line_search=1.0, maxiter=1000, tol=1e-16, cg_iter=None, callback=None): """Newton's method for minimizing a functional. Notes ----- This is a general and optimized implementation of Newton's method for solving the problem: .. math:: \min f(x) for a differentiable function :math:`f: \mathcal{X}\\to \mathbb{R}` on a Hilbert space :math:`\mathcal{X}`. It does so by finding a zero of the gradient .. math:: \\nabla f: \mathcal{X} \\to \mathcal{X}. of finding a root of a function. The algorithm is well-known and there is a vast literature about it. Among others, the method is described in [BV2004], Sections 9.5 and 10.2 (`book available online <http://stanford.edu/~boyd/cvxbook/bv_cvxbook.pdf>`_), [GNS2009], Section 2.7 for solving nonlinear equations and Section 11.3 for its use in minimization, and wikipedia on `Newton's_method <https://en.wikipedia.org/wiki/Newton's_method>`_. The algorithm works by iteratively solving .. math:: \partial f(x_k)p_k = -f(x_k) and then updating as .. math:: x_{k+1} = x_k + \\alpha x_k, where :math:`\\alpha` is a suitable step length (see the references). In this implementation the system of equations are solved using the conjugate gradient method. Parameters ---------- f : `Functional` Goal functional. Needs to have ``f.gradient`` and ``f.gradient.derivative``. x : ``op.domain`` element Starting point of the iteration line_search : float or `LineSearch`, optional Strategy to choose the step length. If a float is given, uses it as a fixed step length. maxiter : int, optional Maximum number of iterations. tol : float, optional Tolerance that should be used for terminating the iteration. cg_iter : int, optional Number of iterations in the the conjugate gradient solver, for computing the search direction. callback : callable, optional Object executing code per iteration, e.g. plotting each iterate References ---------- [BV2004] Boyd, S, and Vandenberghe, L. *Convex optimization*. Cambridge university press, 2004. [GNS2009] Griva, I, Nash, S G, and Sofer, A. *Linear and nonlinear optimization*. Siam, 2009. """ # TODO: update doc grad = f.gradient if x not in grad.domain: raise TypeError('`x` {!r} is not in the domain of `f` {!r}' ''.format(x, grad.domain)) if not callable(line_search): line_search = ConstantLineSearch(line_search) if cg_iter is None: # Motivated by that if it is Ax = b, x and b in Rn, it takes at most n # iterations to solve with cg cg_iter = grad.domain.size # TODO: optimize by using lincomb and avoiding to create copies for _ in range(maxiter): # Initialize the search direction to 0 search_direction = x.space.zero() # Compute hessian (as operator) and gradient in the current point hessian = grad.derivative(x) deriv_in_point = grad(x) # Solving A*x = b for x, in this case f''(x)*p = -f'(x) # TODO: Let the user provide/choose method for how to solve this? try: hessian_inverse = hessian.inverse except NotImplementedError: conjugate_gradient(hessian, search_direction, -deriv_in_point, cg_iter) else: hessian_inverse(-deriv_in_point, out=search_direction) # Computing step length dir_deriv = search_direction.inner(deriv_in_point) if np.abs(dir_deriv) <= tol: return step_length = line_search(x, search_direction, dir_deriv) # Updating x += step_length * search_direction if callback is not None: callback(x)
def newtons_method(f, x, line_search=1.0, maxiter=1000, tol=1e-16, cg_iter=None, callback=None): """Newton's method for minimizing a functional. Notes ----- This is a general and optimized implementation of Newton's method for solving the problem: :math:`\min f(x)` for a differentiable function :math:`f: \mathcal{X}\\to \mathbb{R}` on a Hilbert space :math:`\mathcal{X}`. It does so by finding a zero of the gradient :math:`\\nabla f: \mathcal{X} \\to \mathcal{X}`. of finding a root of a function. The algorithm is well-known and there is a vast literature about it. Among others, the method is described in [BV2004]_, Sections 9.5 and 10.2 (`book available online <http://stanford.edu/~boyd/cvxbook/bv_cvxbook.pdf>`_), [GNS2009]_, Section 2.7 for solving nonlinear equations and Section 11.3 for its use in minimization, and wikipedia on `Newton's_method <https://en.wikipedia.org/wiki/Newton's_method>`_. The algorithm works by iteratively solving :math:`\partial f(x_k)p_k = -f(x_k)` and then updating as :math:`x_{k+1} = x_k + \\alpha x_k`, where :math:`\\alpha` is a suitable step length (see the references). In this implementation the system of equations are solved using the conjugate gradient method. Parameters ---------- f : `Functional` Goal functional. Needs to have ``f.gradient`` and ``f.gradient.derivative``. x : ``op.domain`` element Starting point of the iteration line_search : float or `LineSearch`, optional Strategy to choose the step length. If a float is given, uses it as a fixed step length. maxiter : int, optional Maximum number of iterations. tol : float, optional Tolerance that should be used for terminating the iteration. cg_iter : int, optional Number of iterations in the the conjugate gradient solver, for computing the search direction. callback : callable, optional Object executing code per iteration, e.g. plotting each iterate """ # TODO: update doc grad = f.gradient if x not in grad.domain: raise TypeError('`x` {!r} is not in the domain of `f` {!r}' ''.format(x, grad.domain)) if not callable(line_search): line_search = ConstantLineSearch(line_search) if cg_iter is None: # Motivated by that if it is Ax = b, x and b in Rn, it takes at most n # iterations to solve with cg cg_iter = grad.domain.size # TODO: optimize by using lincomb and avoiding to create copies for _ in range(maxiter): # Initialize the search direction to 0 search_direction = x.space.zero() # Compute hessian (as operator) and gradient in the current point hessian = grad.derivative(x) deriv_in_point = grad(x) # Solving A*x = b for x, in this case f''(x)*p = -f'(x) # TODO: Let the user provide/choose method for how to solve this? conjugate_gradient(hessian, search_direction, -deriv_in_point, cg_iter) # Computing step length dir_deriv = search_direction.inner(deriv_in_point) if np.abs(dir_deriv) <= tol: return step_length = line_search(x, search_direction, dir_deriv) # Updating x += step_length * search_direction if callback is not None: callback(x)
def newtons_method(op, x, line_search, num_iter=10, cg_iter=None, callback=None): """Newton's method for solving a system of equations. This is a general and optimized implementation of Newton's method for solving the problem:: f(x) = 0 of finding a root of a function. The algorithm is well-known and there is a vast literature about it. Among others, the method is described in [BV2004]_, Sections 9.5 and 10.2 (`book available online <http://stanford.edu/~boyd/cvxbook/bv_cvxbook.pdf>`_), [GNS2009]_, Section 2.7 for solving nonlinear equations and Section 11.3 for its use in minimization, and wikipedia on `Newton's_method <https://en.wikipedia.org/wiki/Newton's_method>`_. Parameters ---------- op : `Operator` Gradient of the objective function, ``x --> grad f(x)`` x : element in the domain of ``op`` Starting point of the iteration line_search : `LineSearch` Strategy to choose the step length num_iter : `int`, optional Number of iterations cg_iter : `int`, optional Number of iterations in the the conjugate gradient solver, for computing the search direction. callback : `callable`, optional Object executing code per iteration, e.g. plotting each iterate Notes ---------- The algorithm works by iteratively solving :math:`\partial f(x_k)p_k = -f(x_k)` and then updating as :math:`x_{k+1} = x_k + \\alpha x_k`, where :math:`\\alpha` is a suitable step length (see the references). In this implementation the system of equations are solved using the conjugate gradient method. """ # TODO: update doc if cg_iter is None: # Motivated by that if it is Ax = b, x and b in Rn, it takes at most n # iterations to solve with cg cg_iter = op.domain.size # TODO: optimize by using lincomb and avoiding to create copies for _ in range(num_iter): # Initialize the search direction to 0 search_direction = x.space.zero() # Compute hessian (as operator) and gradient in the current point hessian = op.derivative(x) deriv_in_point = op(x).copy() # Solving A*x = b for x, in this case f'(x)*p = -f(x) # TODO: Let the user provide/choose method for how to solve this? conjugate_gradient(hessian, search_direction, -1 * deriv_in_point, cg_iter) # Computing step length dir_deriv = search_direction.inner(deriv_in_point) step_length = line_search(x, search_direction, dir_deriv) # Updating x += step_length * search_direction if callback is not None: callback(x)