def _quadratic_assignment_2opt(
A,
B,
maximize=False,
partial_match=None,
rng=None,
partial_guess=None,
**unknown_options,
):
r"""
Solve the quadratic assignment problem (approximately).
This function solves the Quadratic Assignment Problem (QAP) and the
Graph Matching Problem (GMP) using the 2-opt algorithm [3]_.
Quadratic assignment solves problems of the following form:
.. math::
\min_P & \ {\ \text{trace}(A^T P B P^T)}\\
\mbox{s.t. } & {P \ \epsilon \ \mathcal{P}}\\
where :math:`\mathcal{P}` is the set of all permutation matrices,
and :math:`A` and :math:`B` are square matrices.
Graph matching tries to *maximize* the same objective function.
This algorithm can be thought of as finding the alignment of the
nodes of two graphs that minimizes the number of induced edge
disagreements, or, in the case of weighted graphs, the sum of squared
edge weight differences.
Note that the quadratic assignment problem is NP-hard, is not
known to be solvable in polynomial time, and is computationally
intractable. Therefore, the results given are approximations,
not guaranteed to be exact solutions.
Parameters
----------
A : 2d-array, square
The square matrix :math:`A` in the objective function above.
B : 2d-array, square
The square matrix :math:`B` in the objective function above.
method : str in {'faq', '2opt'} (default: 'faq')
The algorithm used to solve the problem. This is the method-specific
documentation for '2opt'.
:ref:`'faq' <optimize.qap-faq>` is also available.
Options
-------
maximize : bool (default = False)
Setting `maximize` to ``True`` solves the Graph Matching Problem (GMP)
rather than the Quadratic Assingnment Problem (QAP).
rng : {None, int, `~np.random.RandomState`, `~np.random.Generator`}
This parameter defines the object to use for drawing random
variates.
If `rng` is ``None`` the `~np.random.RandomState` singleton is
used.
If `rng` is an int, a new ``RandomState`` instance is used,
seeded with `rng`.
If `rng` is already a ``RandomState`` or ``Generator``
instance, then that object is used.
Default is None.
partial_match : 2d-array of integers, optional, (default = None)
Allows the user to fix part of the matching between the two
matrices. In the literature, a partial match is also known as a
"seed".
Each row of `partial_match` specifies the indices of a pair of
corresponding nodes, that is, node ``partial_match[i, 0]`` of `A` is
matched to node ``partial_match[i, 1]`` of `B`. Accordingly,
``partial_match`` is an array of size ``(m , 2)``, where ``m`` is
not greater than the number of nodes.
partial_guess : 2d-array of integers, optional, (default = None)
Allows the user to provide a guess for the matching between the two
matrices. Unlike `partial_match`, `partial_guess` does not fix the
indices; they are still free to be optimized.
Each row of `partial_guess` specifies the indices of a pair of
corresponding nodes, that is, node ``partial_guess[i, 0]`` of `A` is
matched to node ``partial_guess[i, 1]`` of `B`. Accordingly,
``partial_guess`` is an array of size ``(m , 2)``, where ``m`` is
less than or equal to the number of nodes.
Returns
-------
res : OptimizeResult
A :class:`scipy.optimize.OptimizeResult` containing the following
fields.
col_ind : 1-D array
An array of column indices corresponding with the best
permutation of the nodes of `B` found.
fun : float
The corresponding value of the objective function.
nit : int
The number of iterations performed during optimization.
Notes
-----
This is a greedy algorithm that works similarly to bubble sort: beginning
with an initial permutation, it iteratively swaps pairs of indices to
improve the objective function until no such improvements are possible.
References
----------
.. [3] "2-opt," Wikipedia.
https://en.wikipedia.org/wiki/2-opt
"""
_check_unknown_options(unknown_options)
rng = check_random_state(rng)
A, B, partial_match = _common_input_validation(A, B, partial_match)
N = len(A)
# check outlier cases
if N == 0 or partial_match.shape[0] == N:
score = _calc_score(A, B, partial_match[:, 1])
res = {"col_ind": partial_match[:, 1], "fun": score, "nit": 0}
return OptimizeResult(res)
if partial_guess is None:
partial_guess = np.array([[], []]).T
partial_guess = np.atleast_2d(partial_guess).astype(int)
msg = None
if partial_guess.shape[0] > A.shape[0]:
msg = "`partial_guess` can have only as " "many entries as there are nodes"
elif partial_guess.shape[1] != 2:
msg = "`partial_guess` must have two columns"
elif partial_guess.ndim != 2:
msg = "`partial_guess` must have exactly two dimensions"
elif (partial_guess < 0).any():
msg = "`partial_guess` must contain only positive indices"
elif (partial_guess >= len(A)).any():
msg = "`partial_guess` entries must be less than number of nodes"
elif not len(set(partial_guess[:, 0])) == len(partial_guess[:, 0]) or not len(
set(partial_guess[:, 1])
) == len(partial_guess[:, 1]):
msg = "`partial_guess` column entries must be unique"
if msg is not None:
raise ValueError(msg)
fixed_rows = None
if partial_match.size or partial_guess.size:
# use partial_match and partial_guess for initial permutation,
# but randomly permute the rest.
guess_rows = np.zeros(N, dtype=bool)
guess_cols = np.zeros(N, dtype=bool)
fixed_rows = np.zeros(N, dtype=bool)
fixed_cols = np.zeros(N, dtype=bool)
perm = np.zeros(N, dtype=int)
rg, cg = partial_guess.T
guess_rows[rg] = True
guess_cols[cg] = True
perm[guess_rows] = cg
# match overrides guess
rf, cf = partial_match.T
fixed_rows[rf] = True
fixed_cols[cf] = True
perm[fixed_rows] = cf
random_rows = ~fixed_rows & ~guess_rows
random_cols = ~fixed_cols & ~guess_cols
perm[random_rows] = rng.permutation(np.arange(N)[random_cols])
else:
perm = rng.permutation(np.arange(N))
best_score = _calc_score(A, B, perm)
i_free = np.arange(N)
if fixed_rows is not None:
i_free = i_free[~fixed_rows]
better = operator.gt if maximize else operator.lt
n_iter = 0
done = False
while not done:
# equivalent to nested for loops i in range(N), j in range(i, N)
for i, j in itertools.combinations_with_replacement(i_free, 2):
n_iter += 1
perm[i], perm[j] = perm[j], perm[i]
score = _calc_score(A, B, perm)
if better(score, best_score):
best_score = score
break
# faster to swap back than to create a new list every time
perm[i], perm[j] = perm[j], perm[i]
else: # no swaps made
done = True
res = {"col_ind": perm, "fun": best_score, "nit": n_iter}
return OptimizeResult(res)
def basicipm(
func,
x,
args,
reduced=False,
maxiter=50,
stepsize=1e-7, # for Hesse Matrix/Gradient
gradtol=1,
**kwargs):
'''
Basic interior point method for nonconvex functions
'''
m = 2 * len(kwargs['bounds'].lb)
bdry = numpy.zeros(m)
for i in range(m / 2):
bdry[2 * i] = kwargs['bounds'].lb[i]
bdry[2 * i + 1] = kwargs['bounds'].ub[i]
print(bdry)
niter = 0
n = len(x)
#starting parameter
z = numpy.ones(m)
s = numpy.ones(m)
my = 20.0
delta_old = 0
#textdummy = open("DUMMY3.txt", "a")
gradtol = 0.000000001
#setup A
A = numpy.zeros((m, n))
for i in range(n):
A[2 * i, i] = 1.0
A[2 * i + 1, i] = -1.0
#c_x = eval_c(x, bdry)
#temp = 20
#while (numpy.linalg.norm(grad(func, x, stepsize))>gradtol and niter<maxiter) :
while niter < 3:
niter = niter + 1
#print('NITER=')
#print(niter)
#error = ipm_error(func, x, A, c_x, z, s, my, textdummy)
#erroralt = 0
testit = 0
#while (error > 30*my and testit<22) :
while (testit < 22):
testit = testit + 1
#erroralt = error
print('INERTIA CORRECTION....')
delta, gamma = inertia_correction_ldl(setup_Jac_ipm,
func,
A,
x,
z,
s,
my,
delta_old,
a=1,
b=0.5)
#delta =1
print('INERTIA CORRETION Done')
print('DELTA=')
print(delta)
print('WAITING FOR NEWTON...')
if reduced:
x, s, z = solve_ipm_newton(setup_F_ipm_reduced,
setup_Jac_ipm_reduced,
func,
A,
x,
z,
s,
my,
bdry,
delta,
delta_old,
reduced=reduced,
variant='gmres')
else:
x, s, z = solve_ipm_newton(setup_F_ipm,
setup_Jac_ipm,
func,
A,
x,
z,
s,
my,
bdry,
delta,
delta_old,
reduced=reduced,
variant='gmres')
#error = ipm_error(func, x, A, c_x, z, s, my, textdummy)
delta_old = delta
wertwert = func(x)
#my = my/2
my = update_my(s, z)
print(
'#################################################################'
)
print('NEW MY IS')
print(my)
result = OptimizeResult(fun=func, x=x, nit=niter)
print('gradient=')
print(numpy.linalg.norm(grad(func, x, stepsize)))
print('X=')
print(x)
print('S=')
print(s)
print('Z=')
print(z)
return result
def lsq_linear(A,
b,
bounds=(-np.inf, np.inf),
method='bvls',
tol=1e-10,
lsq_solver=None,
lsmr_tol=None,
max_iter=None,
verbose=0):
r"""Solve a linear least-squares problem with bounds on the variables.
Given a m-by-n design matrix A and a target vector b with m elements,
`lsq_linear` solves the following optimization problem::
minimize 0.5 * ||A x - b||**2
subject to lb <= x <= ub
This optimization problem is convex, hence a found minimum (if iterations
have converged) is guaranteed to be global.
Parameters
----------
A : array_like, sparse matrix of LinearOperator, shape (m, n)
Design matrix. Can be `scipy.sparse.linalg.LinearOperator`.
b : array_like, shape (m,)
Target vector.
bounds : 2-tuple of array_like, optional
Lower and upper bounds on independent variables. Defaults to no bounds.
Each array must have shape (n,) or be a scalar, in the latter
case a bound will be the same for all variables. Use ``np.inf`` with
an appropriate sign to disable bounds on all or some variables.
method : only 'bvls', optional
Method to perform minimization.
* 'bvls' : Bounded-Variable Least-Squares algorithm. This is
an active set method, which requires the number of iterations
comparable to the number of variables. Can't be used when `A` is
sparse or LinearOperator.
Default is 'bvls'.
tol : float, optional
Tolerance parameter. The algorithm terminates if a relative change
of the cost function is less than `tol` on the last iteration.
Additionally the first-order optimality measure is considered:
* ``method='bvls'`` terminates if Karush-Kuhn-Tucker conditions
are satisfied within `tol` tolerance.
lsq_solver : {None, 'exact', 'lsmr'}, optional
Method of solving unbounded least-squares problems throughout
iterations:
* 'exact' : Use dense QR or SVD decomposition approach. Can't be
used when `A` is sparse or LinearOperator.
* 'lsmr' : Use `scipy.sparse.linalg.lsmr` iterative procedure
which requires only matrix-vector product evaluations. Can't
be used with ``method='bvls'``.
If None (default) the solver is chosen based on type of `A`.
lsmr_tol : None, float or 'auto', optional
Tolerance parameters 'atol' and 'btol' for `scipy.sparse.linalg.lsmr`
If None (default), it is set to ``1e-2 * tol``. If 'auto', the
tolerance will be adjusted based on the optimality of the current
iterate, which can speed up the optimization process, but is not always
reliable.
max_iter : None or int, optional
Maximum number of iterations before termination. If None (default), it
is set to the number of variables for ``method='bvls'``
(not counting iterations for 'bvls' initialization).
verbose : {0, 1, 2}, optional
Level of algorithm's verbosity:
* 0 : work silently (default).
* 1 : display a termination report.
* 2 : display progress during iterations.
Returns
-------
OptimizeResult with the following fields defined:
x : ndarray, shape (n,)
Solution found.
cost : float
Value of the cost function at the solution.
fun : ndarray, shape (m,)
Vector of residuals at the solution.
optimality : float
First-order optimality measure. The exact meaning depends on `method`,
refer to the description of `tol` parameter.
active_mask : ndarray of int, shape (n,)
Each component shows whether a corresponding constraint is active
(that is, whether a variable is at the bound):
* 0 : a constraint is not active.
* -1 : a lower bound is active.
* 1 : an upper bound is active.
nit : int
Number of iterations. Zero if the unconstrained solution is optimal.
status : int
Reason for algorithm termination:
* -1 : the algorithm was not able to make progress on the last
iteration.
* 0 : the maximum number of iterations is exceeded.
* 1 : the first-order optimality measure is less than `tol`.
* 2 : the relative change of the cost function is less than `tol`.
* 3 : the unconstrained solution is optimal.
message : str
Verbal description of the termination reason.
success : bool
True if one of the convergence criteria is satisfied (`status` > 0).
See Also
--------
nnls : Linear least squares with non-negativity constraint.
least_squares : Nonlinear least squares with bounds on the variables.
Notes
-----
The algorithm first computes the unconstrained least-squares solution by
`numpy.linalg.lstsq` or `scipy.sparse.linalg.lsmr` depending on
`lsq_solver`. This solution is returned as optimal if it lies within the
bounds.
Method 'bvls' runs a Python implementation of the algorithm described in
[BVLS]_. The algorithm maintains active and free sets of variables, on
each iteration chooses a new variable to move from the active set to the
free set and then solves the unconstrained least-squares problem on free
variables. This algorithm is guaranteed to give an accurate solution
eventually, but may require up to n iterations for a problem with n
variables. Additionally, an ad-hoc initialization procedure is
implemented, that determines which variables to set free or active
initially. It takes some number of iterations before actual BVLS starts,
but can significantly reduce the number of further iterations.
References
----------
.. [STIR] M. A. Branch, T. F. Coleman, and Y. Li, "A Subspace, Interior,
and Conjugate Gradient Method for Large-Scale Bound-Constrained
Minimization Problems," SIAM Journal on Scientific Computing,
Vol. 21, Number 1, pp 1-23, 1999.
.. [BVLS] P. B. Start and R. L. Parker, "Bounded-Variable Least-Squares:
an Algorithm and Applications", Computational Statistics, 10,
129-141, 1995.
Examples
--------
In this example a problem with a large sparse matrix and bounds on the
variables is solved.
>>> from scipy.sparse import rand
>>> from scipy.optimize import lsq_linear
...
>>> np.random.seed(0)
...
>>> m = 20000
>>> n = 10000
...
>>> A = rand(m, n, density=1e-4)
>>> b = np.random.randn(m)
...
>>> lb = np.random.randn(n)
>>> ub = lb + 1
...
>>> res = lsq_linear(A, b, bounds=(lb, ub), lsmr_tol='auto', verbose=1)
# may vary
The relative change of the cost function is less than `tol`.
Number of iterations 16, initial cost 1.5039e+04, final cost 1.1112e+04,
first-order optimality 4.66e-08.
"""
if method not in ['bvls']:
raise ValueError("`method` must be 'bvls'")
if lsq_solver not in [None, 'exact', 'lsmr']:
raise ValueError("`solver` must be None, 'exact' or 'lsmr'.")
if verbose not in [0, 1, 2]:
raise ValueError("`verbose` must be in [0, 1, 2].")
if issparse(A):
A = csr_matrix(A)
elif not isinstance(A, LinearOperator):
A = np.atleast_2d(A)
if method == 'bvls':
if lsq_solver == 'lsmr':
raise ValueError("method='bvls' can't be used with "
"lsq_solver='lsmr'")
if not isinstance(A, np.ndarray):
raise ValueError("method='bvls' can't be used with `A` being "
"sparse or LinearOperator.")
if lsq_solver is None:
if isinstance(A, np.ndarray):
lsq_solver = 'exact'
else:
lsq_solver = 'lsmr'
elif lsq_solver == 'exact' and not isinstance(A, np.ndarray):
raise ValueError("`exact` solver can't be used when `A` is "
"sparse or LinearOperator.")
if len(A.shape) != 2: # No ndim for LinearOperator.
raise ValueError("`A` must have at most 2 dimensions.")
if len(bounds) != 2:
raise ValueError("`bounds` must contain 2 elements.")
if max_iter is not None and max_iter <= 0:
raise ValueError("`max_iter` must be None or positive integer.")
m, n = A.shape
b = np.atleast_1d(b)
if b.ndim != 1:
raise ValueError("`b` must have at most 1 dimension.")
if b.size != m:
raise ValueError("Inconsistent shapes between `A` and `b`.")
lb, ub = prepare_bounds(bounds, n)
if lb.shape != (n, ) and ub.shape != (n, ):
raise ValueError("Bounds have wrong shape.")
if np.any(lb >= ub):
raise ValueError("Each lower bound must be strictly less than each "
"upper bound.")
if lsq_solver == 'exact':
x_lsq = np.linalg.lstsq(A, b)[0]
elif lsq_solver == 'lsmr':
x_lsq = lsmr(A, b, atol=tol, btol=tol)[0]
if in_bounds(x_lsq, lb, ub):
r = A.dot(x_lsq) - b
cost = 0.5 * np.dot(r, r)
termination_status = 3
termination_message = TERMINATION_MESSAGES[termination_status]
g = compute_grad(A, r)
g_norm = norm(g, ord=np.inf)
if verbose > 0:
print(termination_message)
print("Final cost {0:.4e}, first-order optimality {1:.2e}".format(
cost, g_norm))
return OptimizeResult(x=x_lsq,
fun=r,
cost=cost,
optimality=g_norm,
active_mask=np.zeros(n),
nit=0,
status=termination_status,
message=termination_message,
success=True)
if method == 'bvls':
res = bvls(A, b, x_lsq, lb, ub, tol, max_iter, verbose)
res.message = TERMINATION_MESSAGES[res.status]
res.success = res.status > 0
if verbose > 0:
print(res.message)
print("Number of iterations {0}, initial cost {1:.4e}, "
"final cost {2:.4e}, first-order optimality {3:.2e}.".format(
res.nit, res.initial_cost, res.cost, res.optimality))
del res.initial_cost
return res
def minimize(fun,
x0,
args=(),
method=None,
jac=None,
hess=None,
hessp=None,
bounds=None,
constraints=None,
tol=None,
callback=None,
options=None):
"""An interface to MIGRAD using the ``scipy.optimize.minimize`` API.
For a general description of the arguments, see ``scipy.optimize.minimize``.
The ``method`` argument is ignored. The optimisation is always done using MIGRAD.
The `options` argument can be used to pass special settings to Minuit.
All are optional.
**Options:**
- *disp* (bool): Set to true to print convergence messages. Default: False.
- *maxfev* (int): Maximum allowed number of iterations. Default: 10000.
- *eps* (sequence): Initial step size to numerical compute derivative.
Minuit automatically refines this in subsequent iterations and is very
insensitive to the initial choice. Default: 1.
**Returns: OptimizeResult** (dict with attribute access)
- *x* (ndarray): Solution of optimization.
- *fun* (float): Value of objective function at minimum.
- *message* (str): Description of cause of termination.
- *hess_inv* (ndarray): Inverse of Hesse matrix at minimum (may not be exact).
- nfev (int): Number of function evaluations.
- njev (int): Number of jacobian evaluations.
- minuit (object): Minuit object internally used to do the minimization.
Use this to extract more information about the parameter errors.
"""
from scipy.optimize import OptimizeResult
if method not in {None, 'migrad'}:
warnings.warn("method argument is ignored")
if constraints is not None:
raise ValueError(
"Constraints are not supported by Minuit, only bounds")
if hess or hessp:
warnings.warn(
"hess and hessp arguments cannot be handled and are ignored")
if tol:
warnings.warn("tol argument has no effect on Minuit")
def wrapped(func, args, callback=None):
if callback is None:
return lambda x: func(x, *args)
else:
def f(x):
callback(x)
return func(x, *args)
return f
wrapped_fun = wrapped(fun, args, callback)
maxfev = 10000
error = None
kwargs = {'print_level': 0, 'errordef': 1}
if options:
if "disp" in options:
kwargs['print_level'] = 1
if "maxiter" in options:
warnings.warn("maxiter not supported, acts like maxfev instead")
maxfev = options["maxiter"]
if "maxfev" in options:
maxfev = options["maxfev"]
if "eps" in options:
error = options["eps"]
# prevent warnings from Minuit about missing initial step
if error is None:
error = np.ones_like(x0)
if bool(jac):
if jac is True:
raise ValueError("jac=True is not supported, only jac=callable")
assert hasattr(jac, "__call__")
wrapped_grad = wrapped(jac, args)
else:
wrapped_grad = None
m = Minuit.from_array_func(wrapped_fun,
x0,
error=error,
limit=bounds,
grad=wrapped_grad,
**kwargs)
m.migrad(ncall=maxfev)
message = "Optimization terminated successfully."
success = m.migrad_ok()
if not success:
message = "Optimization failed."
fmin = m.get_fmin()
if fmin.has_reached_call_limit:
message += " Call limit was reached."
if fmin.is_above_max_edm:
message += " Estimated distance to minimum too large."
return OptimizeResult(
x=m.np_values(),
success=success,
fun=m.fval,
hess_inv=m.np_covariance(),
message=message,
nfev=m.get_num_call_fcn(),
njev=m.get_num_call_grad(),
minuit=m,
)
def result(self): return OptimizeResult(x=self.x, nit=self.nit, status=0, success=True, ctol=self.ctol)
def dual_annealing(func,
bounds,
args=(),
maxiter=1000,
minimizer_kwargs=None,
initial_temp=5230.,
restart_temp_ratio=2.e-5,
visit=2.62,
accept=-5.0,
maxfun=1e7,
seed=None,
no_local_search=False,
callback=None,
x0=None):
"""
Find the global minimum of a function using Dual Annealing.
Parameters
----------
func : callable
The objective function to be minimized. Must be in the form
``f(x, *args)``, where ``x`` is the argument in the form of a 1-D array
and ``args`` is a tuple of any additional fixed parameters needed to
completely specify the function.
bounds : sequence or `Bounds`
Bounds for variables. There are two ways to specify the bounds:
1. Instance of `Bounds` class.
2. Sequence of ``(min, max)`` pairs for each element in `x`.
args : tuple, optional
Any additional fixed parameters needed to completely specify the
objective function.
maxiter : int, optional
The maximum number of global search iterations. Default value is 1000.
minimizer_kwargs : dict, optional
Extra keyword arguments to be passed to the local minimizer
(`minimize`). Some important options could be:
``method`` for the minimizer method to use and ``args`` for
objective function additional arguments.
initial_temp : float, optional
The initial temperature, use higher values to facilitates a wider
search of the energy landscape, allowing dual_annealing to escape
local minima that it is trapped in. Default value is 5230. Range is
(0.01, 5.e4].
restart_temp_ratio : float, optional
During the annealing process, temperature is decreasing, when it
reaches ``initial_temp * restart_temp_ratio``, the reannealing process
is triggered. Default value of the ratio is 2e-5. Range is (0, 1).
visit : float, optional
Parameter for visiting distribution. Default value is 2.62. Higher
values give the visiting distribution a heavier tail, this makes
the algorithm jump to a more distant region. The value range is (1, 3].
accept : float, optional
Parameter for acceptance distribution. It is used to control the
probability of acceptance. The lower the acceptance parameter, the
smaller the probability of acceptance. Default value is -5.0 with
a range (-1e4, -5].
maxfun : int, optional
Soft limit for the number of objective function calls. If the
algorithm is in the middle of a local search, this number will be
exceeded, the algorithm will stop just after the local search is
done. Default value is 1e7.
seed : {None, int, `numpy.random.Generator`,
`numpy.random.RandomState`}, optional
If `seed` is None (or `np.random`), the `numpy.random.RandomState`
singleton is used.
If `seed` is an int, a new ``RandomState`` instance is used,
seeded with `seed`.
If `seed` is already a ``Generator`` or ``RandomState`` instance then
that instance is used.
Specify `seed` for repeatable minimizations. The random numbers
generated with this seed only affect the visiting distribution function
and new coordinates generation.
no_local_search : bool, optional
If `no_local_search` is set to True, a traditional Generalized
Simulated Annealing will be performed with no local search
strategy applied.
callback : callable, optional
A callback function with signature ``callback(x, f, context)``,
which will be called for all minima found.
``x`` and ``f`` are the coordinates and function value of the
latest minimum found, and ``context`` has value in [0, 1, 2], with the
following meaning:
- 0: minimum detected in the annealing process.
- 1: detection occurred in the local search process.
- 2: detection done in the dual annealing process.
If the callback implementation returns True, the algorithm will stop.
x0 : ndarray, shape(n,), optional
Coordinates of a single N-D starting point.
Returns
-------
res : OptimizeResult
The optimization result represented as a `OptimizeResult` object.
Important attributes are: ``x`` the solution array, ``fun`` the value
of the function at the solution, and ``message`` which describes the
cause of the termination.
See `OptimizeResult` for a description of other attributes.
Notes
-----
This function implements the Dual Annealing optimization. This stochastic
approach derived from [3]_ combines the generalization of CSA (Classical
Simulated Annealing) and FSA (Fast Simulated Annealing) [1]_ [2]_ coupled
to a strategy for applying a local search on accepted locations [4]_.
An alternative implementation of this same algorithm is described in [5]_
and benchmarks are presented in [6]_. This approach introduces an advanced
method to refine the solution found by the generalized annealing
process. This algorithm uses a distorted Cauchy-Lorentz visiting
distribution, with its shape controlled by the parameter :math:`q_{v}`
.. math::
g_{q_{v}}(\\Delta x(t)) \\propto \\frac{ \\
\\left[T_{q_{v}}(t) \\right]^{-\\frac{D}{3-q_{v}}}}{ \\
\\left[{1+(q_{v}-1)\\frac{(\\Delta x(t))^{2}} { \\
\\left[T_{q_{v}}(t)\\right]^{\\frac{2}{3-q_{v}}}}}\\right]^{ \\
\\frac{1}{q_{v}-1}+\\frac{D-1}{2}}}
Where :math:`t` is the artificial time. This visiting distribution is used
to generate a trial jump distance :math:`\\Delta x(t)` of variable
:math:`x(t)` under artificial temperature :math:`T_{q_{v}}(t)`.
From the starting point, after calling the visiting distribution
function, the acceptance probability is computed as follows:
.. math::
p_{q_{a}} = \\min{\\{1,\\left[1-(1-q_{a}) \\beta \\Delta E \\right]^{ \\
\\frac{1}{1-q_{a}}}\\}}
Where :math:`q_{a}` is a acceptance parameter. For :math:`q_{a}<1`, zero
acceptance probability is assigned to the cases where
.. math::
[1-(1-q_{a}) \\beta \\Delta E] < 0
The artificial temperature :math:`T_{q_{v}}(t)` is decreased according to
.. math::
T_{q_{v}}(t) = T_{q_{v}}(1) \\frac{2^{q_{v}-1}-1}{\\left( \\
1 + t\\right)^{q_{v}-1}-1}
Where :math:`q_{v}` is the visiting parameter.
.. versionadded:: 1.2.0
References
----------
.. [1] Tsallis C. Possible generalization of Boltzmann-Gibbs
statistics. Journal of Statistical Physics, 52, 479-487 (1998).
.. [2] Tsallis C, Stariolo DA. Generalized Simulated Annealing.
Physica A, 233, 395-406 (1996).
.. [3] Xiang Y, Sun DY, Fan W, Gong XG. Generalized Simulated
Annealing Algorithm and Its Application to the Thomson Model.
Physics Letters A, 233, 216-220 (1997).
.. [4] Xiang Y, Gong XG. Efficiency of Generalized Simulated
Annealing. Physical Review E, 62, 4473 (2000).
.. [5] Xiang Y, Gubian S, Suomela B, Hoeng J. Generalized
Simulated Annealing for Efficient Global Optimization: the GenSA
Package for R. The R Journal, Volume 5/1 (2013).
.. [6] Mullen, K. Continuous Global Optimization in R. Journal of
Statistical Software, 60(6), 1 - 45, (2014).
:doi:`10.18637/jss.v060.i06`
Examples
--------
The following example is a 10-D problem, with many local minima.
The function involved is called Rastrigin
(https://en.wikipedia.org/wiki/Rastrigin_function)
>>> from scipy.optimize import dual_annealing
>>> func = lambda x: np.sum(x*x - 10*np.cos(2*np.pi*x)) + 10*np.size(x)
>>> lw = [-5.12] * 10
>>> up = [5.12] * 10
>>> ret = dual_annealing(func, bounds=list(zip(lw, up)))
>>> ret.x
array([-4.26437714e-09, -3.91699361e-09, -1.86149218e-09, -3.97165720e-09,
-6.29151648e-09, -6.53145322e-09, -3.93616815e-09, -6.55623025e-09,
-6.05775280e-09, -5.00668935e-09]) # random
>>> ret.fun
0.000000
"""
if isinstance(bounds, Bounds):
bounds = new_bounds_to_old(bounds.lb, bounds.ub, len(bounds.lb))
# noqa: E501
if x0 is not None and not len(x0) == len(bounds):
raise ValueError('Bounds size does not match x0')
lu = list(zip(*bounds))
lower = np.array(lu[0])
upper = np.array(lu[1])
# Check that restart temperature ratio is correct
if restart_temp_ratio <= 0. or restart_temp_ratio >= 1.:
raise ValueError('Restart temperature ratio has to be in range (0, 1)')
# Checking bounds are valid
if (np.any(np.isinf(lower)) or np.any(np.isinf(upper))
or np.any(np.isnan(lower)) or np.any(np.isnan(upper))):
raise ValueError('Some bounds values are inf values or nan values')
# Checking that bounds are consistent
if not np.all(lower < upper):
raise ValueError('Bounds are not consistent min < max')
# Checking that bounds are the same length
if not len(lower) == len(upper):
raise ValueError('Bounds do not have the same dimensions')
# Wrapper for the objective function
func_wrapper = ObjectiveFunWrapper(func, maxfun, *args)
# minimizer_kwargs has to be a dict, not None
minimizer_kwargs = minimizer_kwargs or {}
minimizer_wrapper = LocalSearchWrapper(bounds, func_wrapper,
**minimizer_kwargs)
# Initialization of random Generator for reproducible runs if seed provided
rand_state = check_random_state(seed)
# Initialization of the energy state
energy_state = EnergyState(lower, upper, callback)
energy_state.reset(func_wrapper, rand_state, x0)
# Minimum value of annealing temperature reached to perform
# re-annealing
temperature_restart = initial_temp * restart_temp_ratio
# VisitingDistribution instance
visit_dist = VisitingDistribution(lower, upper, visit, rand_state)
# Strategy chain instance
strategy_chain = StrategyChain(accept, visit_dist, func_wrapper,
minimizer_wrapper, rand_state, energy_state)
need_to_stop = False
iteration = 0
message = []
# OptimizeResult object to be returned
optimize_res = OptimizeResult()
optimize_res.success = True
optimize_res.status = 0
t1 = np.exp((visit - 1) * np.log(2.0)) - 1.0
# Run the search loop
while (not need_to_stop):
for i in range(maxiter):
# Compute temperature for this step
s = float(i) + 2.0
t2 = np.exp((visit - 1) * np.log(s)) - 1.0
temperature = initial_temp * t1 / t2
if iteration >= maxiter:
message.append("Maximum number of iteration reached")
need_to_stop = True
break
# Need a re-annealing process?
if temperature < temperature_restart:
energy_state.reset(func_wrapper, rand_state)
break
# starting strategy chain
val = strategy_chain.run(i, temperature)
if val is not None:
message.append(val)
need_to_stop = True
optimize_res.success = False
break
# Possible local search at the end of the strategy chain
if not no_local_search:
val = strategy_chain.local_search()
if val is not None:
message.append(val)
need_to_stop = True
optimize_res.success = False
break
iteration += 1
# Setting the OptimizeResult values
optimize_res.x = energy_state.xbest
optimize_res.fun = energy_state.ebest
optimize_res.nit = iteration
optimize_res.nfev = func_wrapper.nfev
optimize_res.njev = func_wrapper.ngev
optimize_res.nhev = func_wrapper.nhev
optimize_res.message = message
return optimize_res
def minimize(fun,
bounds=None,
x0=None,
input_sigma=0.166,
popsize=0,
max_evaluations=100000,
stop_fitness=None,
rg=Generator(MT19937()),
runid=0):
"""Minimization of a scalar function of one or more variables using a
C++ SCMA implementation called via ctypes.
Parameters
----------
fun : callable
The objective function to be minimized.
``fun(x, *args) -> float``
where ``x`` is an 1-D array with shape (dim,) and ``args``
is a tuple of the fixed parameters needed to completely
specify the function.
bounds : sequence or `Bounds`, optional
Bounds on variables. There are two ways to specify the bounds:
1. Instance of the `scipy.Bounds` class.
2. Sequence of ``(min, max)`` pairs for each element in `x`. None
is used to specify no bound.
x0 : ndarray, shape (dim,)
Initial guess. Array of real elements of size (dim,),
where 'dim' is the number of independent variables.
input_sigma : ndarray, shape (dim,) or scalar
Initial step size for each dimension.
popsize = int, optional
CMA-ES population size.
max_evaluations : int, optional
Forced termination after ``max_evaluations`` function evaluations.
stop_fitness : float, optional
Limit for fitness value. If reached minimize terminates.
rg = numpy.random.Generator, optional
Random generator for creating random guesses.
runid : int, optional
id used to identify the run for debugging / logging.
Returns
-------
res : scipy.OptimizeResult
The optimization result is represented as an ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array,
``fun`` the best function value,
``nfev`` the number of function evaluations,
``nit`` the number of CMA-ES iterations,
``status`` the stopping critera and
``success`` a Boolean flag indicating if the optimizer exited successfully. """
lower, upper, guess = _check_bounds(bounds, x0, rg)
dim = guess.size
if lower is None:
lower = [0] * dim
upper = [0] * dim
if callable(input_sigma):
input_sigma = input_sigma()
if np.ndim(input_sigma) == 0:
input_sigma = [input_sigma] * dim
if stop_fitness is None:
stop_fitness = -math.inf
array_type = ct.c_double * dim
c_callback = mo_call_back_type(callback(fun, dim))
res = np.empty(dim + 4)
res_p = res.ctypes.data_as(ct.POINTER(ct.c_double))
try:
optimizeCsma_C(runid, c_callback, dim, int(rg.uniform(0, 2**32 - 1)),
array_type(*guess), array_type(*lower),
array_type(*upper), array_type(*input_sigma),
max_evaluations, stop_fitness, popsize, res_p)
x = res[:dim]
val = res[dim]
evals = int(res[dim + 1])
iterations = int(res[dim + 2])
stop = int(res[dim + 3])
return OptimizeResult(x=x,
fun=val,
nfev=evals,
nit=iterations,
status=stop,
success=True)
except Exception as ex:
return OptimizeResult(x=None,
fun=sys.float_info.max,
nfev=0,
nit=0,
status=-1,
success=False)
def minimize(fun,
bounds=None,
x0=None,
input_sigma = 0.3,
popsize = 31,
max_evaluations = 100000,
max_iterations = 100000,
accuracy = 1.0,
stop_fitness = None,
is_terminate = None,
rg = Generator(MT19937()),
runid=0,
workers = None,
normalize = True,
update_gap = None):
"""Minimization of a scalar function of one or more variables using a
C++ CMA-ES implementation called via ctypes.
Parameters
----------
fun : callable
The objective function to be minimized.
``fun(x, *args) -> float``
where ``x`` is an 1-D array with shape (n,) and ``args``
is a tuple of the fixed parameters needed to completely
specify the function.
bounds : sequence or `Bounds`, optional
Bounds on variables. There are two ways to specify the bounds:
1. Instance of the `scipy.Bounds` class.
2. Sequence of ``(min, max)`` pairs for each element in `x`. None
is used to specify no bound.
x0 : ndarray, shape (n,)
Initial guess. Array of real elements of size (n,),
where 'n' is the number of independent variables.
input_sigma : ndarray, shape (n,) or scalar
Initial step size for each dimension.
popsize = int, optional
CMA-ES population size.
max_evaluations : int, optional
Forced termination after ``max_evaluations`` function evaluations.
max_iterations : int, optional
Forced termination after ``max_iterations`` iterations.
accuracy : float, optional
values > 1.0 reduce the accuracy.
stop_fitness : float, optional
Limit for fitness value. If reached minimize terminates.
is_terminate : callable, optional
Callback to be used if the caller of minimize wants to
decide when to terminate.
rg = numpy.random.Generator, optional
Random generator for creating random guesses.
runid : int, optional
id used by the is_terminate callback to identify the CMA-ES run.
workers : int or None, optional
If not workers is None, function evaluation is performed in parallel for the whole population.
Useful for costly objective functions but is deactivated for parallel retry.
normalize : boolean, optional
pheno -> if true geno transformation maps arguments to interval [-1,1]
update_gap : int, optional
number of iterations without distribution update
Returns
-------
res : scipy.OptimizeResult
The optimization result is represented as an ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array,
``fun`` the best function value,
``nfev`` the number of function evaluations,
``nit`` the number of CMA-ES iterations,
``status`` the stopping critera and
``success`` a Boolean flag indicating if the optimizer exited successfully. """
lower, upper, guess = _check_bounds(bounds, x0, rg)
n = guess.size
if lower is None:
lower = [0]*n
upper = [0]*n
mu = int(popsize/2)
if np.ndim(input_sigma) == 0:
input_sigma = [input_sigma] * n
if stop_fitness is None:
stop_fitness = math.inf
if is_terminate is None:
is_terminate=_is_terminate_false
use_terminate = False
else:
use_terminate = True
parfun = None if workers is None else parallel(fun, workers)
array_type = ct.c_double * n
c_callback_par = call_back_par(callback_par(fun, parfun))
c_is_terminate = is_terminate_type(is_terminate)
try:
res = optimizeACMA_C(runid, c_callback_par, n, array_type(*guess), array_type(*lower), array_type(*upper),
array_type(*input_sigma), max_iterations, max_evaluations, stop_fitness, mu,
popsize, accuracy, use_terminate, c_is_terminate,
int(rg.uniform(0, 2**32 - 1)), normalize, -1 if update_gap is None else update_gap)
x = np.array(np.fromiter(res, dtype=np.float64, count=n))
val = res[n]
evals = int(res[n+1])
iterations = int(res[n+2])
stop = int(res[n+3])
freemem(res)
if not parfun is None:
parfun.stop() # stop all parallel evaluation processes
return OptimizeResult(x=x, fun=val, nfev=evals, nit=iterations, status=stop, success=True)
except Exception as ex:
if not workers is None:
fun.stop() # stop all parallel evaluation processes
return OptimizeResult(x=None, fun=sys.float_info.max, nfev=0, nit=0, status=-1, success=False)
def trf_no_bounds(fun, x0, f0=None, ftol=1e-8, xtol=1e-8, gtol=1e-8,
max_nfev=None, x_scale=1.0, tr_solver='lsmr',
tr_options=None, verbose=0):
if max_nfev is None:
max_nfev = x0.numel() * 100
if tr_options is None:
tr_options = {}
assert tr_solver in ['exact', 'lsmr', 'cgls']
if tr_solver == 'exact':
jacobian = jacobian_dense
else:
jacobian = jacobian_linop
x = x0.clone()
if f0 is None:
f = fun(x)
else:
f = f0
f_true = f.clone()
J = jacobian(fun, x)
nfev = njev = 1
m, n = J.shape
cost = 0.5 * f.dot(f)
g = J.T.mv(f)
scale = x_scale
Delta = (x0 / scale).norm()
if Delta == 0:
Delta.fill_(1.)
if tr_solver != 'exact':
damp = tr_options.pop('damp', 1e-4)
regularize = tr_options.pop('regularize', False)
reg_term = 0.
alpha = x0.new_tensor(0.) # "Levenberg-Marquardt" parameter
termination_status = None
iteration = 0
step_norm = None
actual_reduction = None
if verbose == 2:
print_header_nonlinear()
while True:
g_norm = g.norm(np.inf)
if g_norm < gtol:
termination_status = 1
if verbose == 2:
print_iteration_nonlinear(iteration, nfev, cost, actual_reduction,
step_norm, g_norm)
if termination_status is not None or nfev == max_nfev:
break
d = scale
g_h = d * g
if tr_solver == 'exact':
J_h = J * d
U, s, V = torch.linalg.svd(J_h, full_matrices=False)
V = V.T
uf = U.T.mv(f)
else:
J_h = right_multiplied_operator(J, d)
if regularize:
a, b = build_quadratic_1d(J_h, g_h, -g_h)
to_tr = Delta / g_h.norm()
ag_value = minimize_quadratic_1d(a, b, 0, to_tr)[1]
reg_term = -ag_value / Delta**2
damp_full = (damp**2 + reg_term)**0.5
if tr_solver == 'lsmr':
gn_h = lsmr(J_h, f, damp=damp_full, **tr_options)[0]
elif tr_solver == 'cgls':
gn_h = cgls(J_h, f, alpha=damp_full, max_iter=min(m,n), **tr_options)
else:
raise RuntimeError
S = torch.vstack((g_h, gn_h)).T # [n,2]
# Dispatch qr to CPU so long as pytorch/pytorch#22573 is not fixed
S = torch.linalg.qr(S.cpu(), mode='reduced')[0].to(S.device) # [n,2]
JS = J_h.matmul(S) # [m,2]
B_S = JS.T.matmul(JS) # [2,2]
g_S = S.T.mv(g_h) # [2]
actual_reduction = -1
while actual_reduction <= 0 and nfev < max_nfev:
if tr_solver == 'exact':
step_h, alpha, n_iter = solve_lsq_trust_region(
n, m, uf, s, V, Delta, initial_alpha=alpha)
else:
p_S, _ = solve_trust_region_2d(B_S, g_S, Delta)
step_h = S.matmul(p_S)
predicted_reduction = -evaluate_quadratic(J_h, g_h, step_h)
step = d * step_h
x_new = x + step
f_new = fun(x_new)
nfev += 1
step_h_norm = step_h.norm()
if not f_new.isfinite().all():
Delta = 0.25 * step_h_norm
continue
# Usual trust-region step quality estimation.
cost_new = 0.5 * f_new.dot(f_new)
actual_reduction = cost - cost_new
Delta_new, ratio = update_tr_radius(
Delta, actual_reduction, predicted_reduction,
step_h_norm, step_h_norm > 0.95 * Delta)
step_norm = step.norm()
termination_status = check_termination(
actual_reduction, cost, step_norm, x.norm(), ratio, ftol, xtol)
if termination_status is not None:
break
alpha *= Delta / Delta_new
Delta = Delta_new
if actual_reduction > 0:
x, f, cost = x_new, f_new, cost_new
f_true.copy_(f)
J = jacobian(fun, x)
g = J.T.mv(f)
njev += 1
else:
step_norm = 0
actual_reduction = 0
iteration += 1
if termination_status is None:
termination_status = 0
active_mask = torch.zeros_like(x)
return OptimizeResult(
x=x, cost=cost, fun=f_true, jac=J, grad=g, optimality=g_norm,
active_mask=active_mask, nfev=nfev, njev=njev,
status=termination_status)
def model_policy_gradient(
f: Callable[..., float],
x0: np.ndarray,
*,
args=(),
learning_rate: float = 1e-2,
decay_rate: float = 0.96,
decay_steps: int = 5,
log_sigma_init: float = -5.0,
max_iterations: int = 1000,
batch_size: int = 10,
radius_coeff: float = 3.0,
warmup_steps: int = 10,
batch_size_model: int = 65536,
save_func_vals: bool = False,
random_state: "cirq.RANDOM_STATE_OR_SEED_LIKE" = None,
known_values: Optional[Tuple[List[np.ndarray], List[float]]] = None,
max_evaluations: Optional[int] = None
) -> scipy.optimize.OptimizeResult:
"""Model policy gradient algorithm for black-box optimization.
The idea of this algorithm is to perform policy gradient, but estimate
the function values using a surrogate model.
The surrogate model is a least-squared quadratic
fit to points sampled from the vicinity of the current iterate.
Args:
f: The function to minimize.
x0: An initial guess.
args: Additional arguments to pass to the function.
learning_rate: The learning rate for the policy gradient.
decay_rate: the learning decay rate for the Adam optimizer.
decay_steps: the learning decay steps for the Adam optimizer.
log_sigma_init: the initial value for the sigma of the policy
in the log scale.
max_iterations: The maximum number of iterations to allow before
termination.
batch_size: The number of points to sample in each iteration. The cost
of evaluation of these samples are computed through the
quantum computer cost model.
radius_coeff: The ratio determining the size of the radius around
the current iterate to sample points from to build the quadratic model.
The ratio is with respect to the maximal ratio of the samples
from the current policy.
warmup_steps: The number of steps before the model policy gradient is performed.
before these steps, we use the policy gradient without the model.
batch_size_model: The model sample batch size.
After we fit the quadratic model, we use the model to evaluate
on big enough batch of samples.
save_func_vals: whether to compute and save the function values for
the current value of parameter.
random_state: A seed (int) or `np.random.RandomState` class to use when
generating random values. If not set, defaults to using the module
methods in `np.random`.
known_values: Any prior known values of the objective function.
This is given as a tuple where the first element is a list
of points and the second element is a list of the function values
at those points.
max_evaluations: The maximum number of function evaluations to allow
before termination.
Returns:
Scipy OptimizeResult
"""
random_state = value.parse_random_state(random_state)
if known_values is not None:
known_xs, known_ys = known_values
known_xs = [np.copy(x) for x in known_xs]
known_ys = [np.copy(y) for y in known_ys]
else:
known_xs, known_ys = [], []
if max_evaluations is None:
max_evaluations = np.inf
n = len(x0)
log_sigma = np.ones(n) * log_sigma_init
sigma = np.exp(log_sigma)
# set up the first and second moment estimate
m_mean = np.zeros(n)
v_mean = np.zeros(n)
m_log_sigma = np.zeros(n)
v_log_sigma = np.zeros(n)
# set up lr schedule and optimizer
lr_schedule1 = _ExponentialSchedule(learning_rate,
decay_steps=decay_steps,
decay_rate=decay_rate,
staircase=True)
lr_schedule2 = _ExponentialSchedule(learning_rate,
decay_steps=decay_steps,
decay_rate=decay_rate,
staircase=True)
_, f = wrap_function(f, args)
res = OptimizeResult()
current_x = np.copy(x0)
res.x_iters = [] # initializes as lists
res.xs_iters = []
res.ys_iters = []
res.func_vals = []
res.fun = 0
total_evals = 0
num_iter = 0
message = None
# stats
history_max = -np.inf
while num_iter < max_iterations:
# get samples from the current policy to evaluate
z = random_state.randn(batch_size, n)
new_xs = sigma * z + current_x
if total_evals + batch_size > max_evaluations:
message = "Reached maximum number of evaluations."
break
# Evaluate points
res.xs_iters.append(new_xs)
new_ys = [f(x) for x in new_xs]
res.ys_iters.append(new_ys)
total_evals += batch_size
known_xs.extend(new_xs)
known_ys.extend(new_ys)
# Save function value
if save_func_vals:
res.func_vals.append(f(current_x))
res.x_iters.append(np.copy(current_x))
res.fun = res.func_vals[-1]
# current sampling radius (maximal)
max_radius = 0
for x in new_xs:
if np.linalg.norm(x - current_x) > max_radius:
max_radius = np.linalg.norm(x - current_x)
reward = [-y for y in new_ys]
# warmup steps control whether to use the model to estimate the f
if num_iter >= warmup_steps:
# Determine points to use to build model
model_xs = []
model_ys = []
for x, y in zip(known_xs, known_ys):
if np.linalg.norm(x - current_x) < radius_coeff * max_radius:
model_xs.append(x)
model_ys.append(y)
# safer way without the `SVD` not converging
try:
model = _get_quadratic_model(model_xs, model_ys, x)
use_model = True
except ValueError:
use_model = False
if use_model:
# get samples (from model)
z = random_state.randn(batch_size_model, n)
new_xs = sigma * z + current_x
# use the model for prediction
new_ys = model.predict(new_xs - current_x)
reward = [-y for y in new_ys]
reward = np.array(reward)
# stats
reward_mean = np.mean(reward)
reward_max = np.max(reward)
if reward_max > history_max:
history_max = reward_max
# subtract baseline
reward = reward - reward_mean
# analytic derivatives (natural gradient policy gradient)
delta_mean = np.dot(z.T, reward) * sigma
delta_log_sigma = np.dot(z.T**2, reward) / np.sqrt(2)
delta_mean_norm = np.linalg.norm(np.dot(z.T, reward))
delta_log_sigma_norm = np.linalg.norm(np.dot(z.T**2, reward))
delta_mean = delta_mean / delta_mean_norm
delta_log_sigma = delta_log_sigma / delta_log_sigma_norm
# gradient ascend to update the parameters
current_x, m_mean, v_mean = _adam_update(delta_mean,
current_x,
num_iter,
m_mean,
v_mean,
lr_schedule=lr_schedule1)
log_sigma, m_log_sigma, v_log_sigma = _adam_update(
delta_log_sigma,
log_sigma,
num_iter,
m_log_sigma,
v_log_sigma,
lr_schedule=lr_schedule2,
)
log_sigma = np.clip(log_sigma, -20.0, 2.0)
sigma = np.exp(log_sigma)
num_iter += 1
final_val = f(current_x)
res.func_vals.append(final_val)
if message is None:
message = "Reached maximum number of iterations."
res.x_iters.append(current_x)
total_evals += 1
res.x = current_x
res.fun = final_val
res.nit = num_iter
res.nfev = total_evals
res.message = message
return res
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If `polish`
was employed, and a lower minimum was obtained by the polishing,
then OptimizeResult also contains the ``jac`` attribute.
"""
nfev, nit, warning_flag = 0, 0, False
status_message = _status_message['success']
# calculate energies to start with
for index, candidate in enumerate(self.population):
parameters = self._scale_parameters(candidate)
self.population_energies[index] = self.func(parameters, *self.args)
nfev += 1
if nfev > self.maxfun:
warning_flag = True
status_message = _status_message['maxfev']
break
minval = np.argmin(self.population_energies)
# put the lowest energy into the best solution position.
lowest_energy = self.population_energies[minval]
self.population_energies[minval] = self.population_energies[0]
self.population_energies[0] = lowest_energy
self.population[[0, minval], :] = self.population[[minval, 0], :]
if warning_flag:
return OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
# do the optimisation.
for nit in range(1, self.maxiter + 1):
if self.dither is not None:
self.scale = self.random_number_generator.rand() * (
self.dither[1] - self.dither[0]) + self.dither[0]
for candidate in range(self.num_population_members):
if nfev > self.maxfun:
warning_flag = True
status_message = _status_message['maxfev']
break
# create a trial solution
trial = self._mutate(candidate)
# ensuring that it's in the range [0, 1)
self._ensure_constraint(trial)
# scale from [0, 1) to the actual parameter value
parameters = self._scale_parameters(trial)
# determine the energy of the objective function
energy = self.func(parameters, *self.args)
nfev += 1
# if the energy of the trial candidate is lower than the
# original population member then replace it
if energy < self.population_energies[candidate]:
self.population[candidate] = trial
self.population_energies[candidate] = energy
# if the trial candidate also has a lower energy than the
# best solution then replace that as well
if energy < self.population_energies[0]:
self.population_energies[0] = energy
self.population[0] = trial
# stop when the fractional s.d. of the population is less than tol
# of the mean energy
convergence = (
np.std(self.population_energies) /
np.abs(np.mean(self.population_energies) + _MACHEPS))
if self.disp:
print("differential_evolution step %d: f(x)= %g" %
(nit, self.population_energies[0]))
if (self.callback and
self.callback(self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
break
if convergence < self.tol or warning_flag:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
nfev += result.nfev
DE_result.nfev = nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def _minimize_cg(fun, x0, max_iter=None, gtol=1e-5, normp=float('inf'),
callback=None, disp=0, return_all=False):
"""Minimize a scalar function of one or more variables using
nonlinear conjugate gradient.
The algorithm is described in Nocedal & Wright (2006) chapter 5.2.
Parameters
----------
fun : callable
Scalar objective function to minimize.
x0 : Tensor
Initialization point.
max_iter : int
Maximum number of iterations to perform. Defaults to
``200 * x0.numel()``.
gtol : float
Termination tolerance on 1st-order optimality (gradient norm).
normp : float
The norm type to use for termination conditions. Can be any value
supported by :func:`torch.norm`.
callback : callable, optional
Function to call after each iteration with the current parameter
state, e.g. ``callback(x)``
disp : int or bool
Display (verbosity) level. Set to >0 to print status messages.
return_all : bool, optional
Set to True to return a list of the best solution at each of the
iterations.
"""
disp = int(disp)
if max_iter is None:
max_iter = x0.numel() * 200
# Construct scalar objective function
sf = ScalarFunction(fun, x_shape=x0.shape)
closure = sf.closure
dir_evaluate = sf.dir_evaluate
# initialize
x = x0.detach().flatten()
f, g, _, _ = closure(x)
if disp > 1:
print('initial fval: %0.4f' % f)
if return_all:
allvecs = [x]
d = g.neg()
grad_norm = g.norm(p=normp)
old_f = f + g.norm() / 2 # Sets the initial step guess to dx ~ 1
for niter in range(1, max_iter + 1):
# delta/gtd
delta = dot(g, g)
gtd = dot(g, d)
# compute initial step guess based on (f - old_f) / gtd
t0 = torch.clamp(2.02 * (f - old_f) / gtd, max=1.0)
if t0 <= 0:
warnflag = 4
msg = 'Initial step guess is negative.'
break
old_f = f
# buffer to store next direction vector
cached_step = [None]
def polak_ribiere_powell_step(t, g_next):
y = g_next - g
beta = torch.clamp(dot(y, g_next) / delta, min=0)
d_next = -g_next + d.mul(beta)
torch.norm(g_next, p=normp, out=grad_norm)
return t, d_next
def descent_condition(t, f_next, g_next):
# Polak-Ribiere+ needs an explicit check of a sufficient
# descent condition, which is not guaranteed by strong Wolfe.
cached_step[:] = polak_ribiere_powell_step(t, g_next)
t, d_next = cached_step
# Accept step if it leads to convergence.
cond1 = grad_norm <= gtol
# Accept step if sufficient descent condition applies.
cond2 = dot(d_next, g_next) <= -0.01 * dot(g_next, g_next)
return cond1 | cond2
# Perform CG step
f, g, t, ls_evals = \
strong_wolfe(dir_evaluate, x, t0, d, f, g, gtd,
c2=0.4, extra_condition=descent_condition)
# Update x and then update d (in that order)
x = x + d.mul(t)
if t == cached_step[0]:
# Reuse already computed results if possible
d = cached_step[1]
else:
d = polak_ribiere_powell_step(t, g)[1]
if disp > 1:
print('iter %3d - fval: %0.4f' % (niter, f))
if return_all:
allvecs.append(x)
if callback is not None:
callback(x)
# check optimality
if grad_norm <= gtol:
warnflag = 0
msg = _status_message['success']
break
else:
# if we get to the end, the maximum iterations was reached
warnflag = 1
msg = _status_message['maxiter']
if disp:
print("%s%s" % ("Warning: " if warnflag != 0 else "", msg))
print(" Current function value: %f" % f)
print(" Iterations: %d" % niter)
print(" Function evaluations: %d" % sf.nfev)
result = OptimizeResult(x=x, fun=f, grad=g, nit=niter, nfev=sf.nfev,
status=warnflag, success=(warnflag == 0),
message=msg)
if return_all:
result['allvecs'] = allvecs
return result
def _root_df_sane(func,
x0,
args=(),
ftol=1e-8,
fatol=1e-300,
maxfev=1000,
fnorm=None,
callback=None,
disp=False,
M=10,
eta_strategy=None,
sigma_eps=1e-10,
sigma_0=1.0,
line_search='cruz',
**unknown_options):
r"""
Solve nonlinear equation with the DF-SANE method
Options
-------
ftol : float, optional
Relative norm tolerance.
fatol : float, optional
Absolute norm tolerance.
Algorithm terminates when ``||func(x)|| < fatol + ftol ||func(x_0)||``.
fnorm : callable, optional
Norm to use in the convergence check. If None, 2-norm is used.
maxfev : int, optional
Maximum number of function evaluations.
disp : bool, optional
Whether to print convergence process to stdout.
eta_strategy : callable, optional
Choice of the ``eta_k`` parameter, which gives slack for growth
of ``||F||**2``. Called as ``eta_k = eta_strategy(k, x, F)`` with
`k` the iteration number, `x` the current iterate and `F` the current
residual. Should satisfy ``eta_k > 0`` and ``sum(eta, k=0..inf) < inf``.
Default: ``||F||**2 / (1 + k)**2``.
sigma_eps : float, optional
The spectral coefficient is constrained to ``sigma_eps < sigma < 1/sigma_eps``.
Default: 1e-10
sigma_0 : float, optional
Initial spectral coefficient.
Default: 1.0
M : int, optional
Number of iterates to include in the nonmonotonic line search.
Default: 10
line_search : {'cruz', 'cheng'}
Type of line search to employ. 'cruz' is the original one defined in
[Martinez & Raydan. Math. Comp. 75, 1429 (2006)], 'cheng' is
a modified search defined in [Cheng & Li. IMA J. Numer. Anal. 29, 814 (2009)].
Default: 'cruz'
References
----------
.. [1] "Spectral residual method without gradient information for solving
large-scale nonlinear systems of equations." W. La Cruz,
J.M. Martinez, M. Raydan. Math. Comp. **75**, 1429 (2006).
.. [2] W. La Cruz, Opt. Meth. Software, 29, 24 (2014).
.. [3] W. Cheng, D.-H. Li. IMA J. Numer. Anal. **29**, 814 (2009).
"""
_check_unknown_options(unknown_options)
if line_search not in ('cheng', 'cruz'):
raise ValueError("Invalid value %r for 'line_search'" %
(line_search, ))
nexp = 2
if eta_strategy is None:
# Different choice from [1], as their eta is not invariant
# vs. scaling of F.
def eta_strategy(k, x, F):
# Obtain squared 2-norm of the initial residual from the outer scope
return f_0 / (1 + k)**2
if fnorm is None:
def fnorm(F):
# Obtain squared 2-norm of the current residual from the outer scope
return f_k**(1.0 / nexp)
def fmerit(F):
return np.linalg.norm(F)**nexp
nfev = [0]
f, x_k, x_shape, f_k, F_k, is_complex = _wrap_func(func, x0, fmerit, nfev,
maxfev, args)
k = 0
f_0 = f_k
sigma_k = sigma_0
F_0_norm = fnorm(F_k)
# For the 'cruz' line search
prev_fs = collections.deque([f_k], M)
# For the 'cheng' line search
Q = 1.0
C = f_0
converged = False
message = "too many function evaluations required"
while True:
F_k_norm = fnorm(F_k)
if disp:
print("iter %d: ||F|| = %g, sigma = %g" % (k, F_k_norm, sigma_k))
if callback is not None:
callback(x_k, F_k)
if F_k_norm < ftol * F_0_norm + fatol:
# Converged!
message = "successful convergence"
converged = True
break
# Control spectral parameter, from [2]
if abs(sigma_k) > 1 / sigma_eps:
sigma_k = 1 / sigma_eps * np.sign(sigma_k)
elif abs(sigma_k) < sigma_eps:
sigma_k = sigma_eps
# Line search direction
d = -sigma_k * F_k
# Nonmonotone line search
eta = eta_strategy(k, x_k, F_k)
try:
if line_search == 'cruz':
alpha, xp, fp, Fp = _nonmonotone_line_search_cruz(f,
x_k,
d,
prev_fs,
eta=eta)
elif line_search == 'cheng':
alpha, xp, fp, Fp, C, Q = _nonmonotone_line_search_cheng(
f, x_k, d, f_k, C, Q, eta=eta)
except _NoConvergence:
break
# Update spectral parameter
s_k = xp - x_k
y_k = Fp - F_k
sigma_k = np.vdot(s_k, s_k) / np.vdot(s_k, y_k)
# Take step
x_k = xp
F_k = Fp
f_k = fp
# Store function value
if line_search == 'cruz':
prev_fs.append(fp)
k += 1
x = _wrap_result(x_k, is_complex, shape=x_shape)
F = _wrap_result(F_k, is_complex)
result = OptimizeResult(x=x,
success=converged,
message=message,
fun=F,
nfev=nfev[0],
nit=k)
return result
def nakanishi_fujii_todo(fun,
x0,
args=(),
maxiter=None,
maxfev=1024,
reset_interval=32,
eps=1e-32,
callback=None,
**_):
"""
Find the global minimum of a function using the nakanishi_fujii_todo
algorithm [1].
Args:
fun (callable): ``f(x, *args)``
Function to be optimized. ``args`` can be passed as an optional item
in the dict ``minimizer_kwargs``.
This function must satisfy the three condition written in Ref. [1].
x0 (ndarray): shape (n,)
Initial guess. Array of real elements of size (n,),
where 'n' is the number of independent variables.
args (tuple, optional):
Extra arguments passed to the objective function.
maxiter (int):
Maximum number of iterations to perform.
Default: None.
maxfev (int):
Maximum number of function evaluations to perform.
Default: 1024.
reset_interval (int):
The minimum estimates directly once in ``reset_interval`` times.
Default: 32.
eps (float): eps
**_ : additional options
callback (callable, optional):
Called after each iteration.
Returns:
OptimizeResult:
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array. See
`OptimizeResult` for a description of other attributes.
Notes:
In this optimization method, the optimization function have to satisfy
three conditions written in [1].
References:
.. [1] K. M. Nakanishi, K. Fujii, and S. Todo. 2019.
Sequential minimal optimization for quantum-classical hybrid algorithms.
arXiv preprint arXiv:1903.12166.
"""
x0 = np.asarray(x0)
recycle_z0 = None
niter = 0
funcalls = 0
while True:
idx = niter % x0.size
if reset_interval > 0:
if niter % reset_interval == 0:
recycle_z0 = None
if recycle_z0 is None:
z0 = fun(np.copy(x0), *args)
funcalls += 1
else:
z0 = recycle_z0
p = np.copy(x0)
p[idx] = x0[idx] + np.pi / 2
z1 = fun(p, *args)
funcalls += 1
p = np.copy(x0)
p[idx] = x0[idx] - np.pi / 2
z3 = fun(p, *args)
funcalls += 1
z2 = z1 + z3 - z0
c = (z1 + z3) / 2
a = np.sqrt((z0 - z2)**2 + (z1 - z3)**2) / 2
b = np.arctan((z1 - z3) / ((z0 - z2) + eps * (z0 == z2))) + x0[idx]
b += 0.5 * np.pi + 0.5 * np.pi * np.sign((z0 - z2) + eps * (z0 == z2))
x0[idx] = b
recycle_z0 = c - a
niter += 1
if callback is not None:
callback(np.copy(x0))
if maxfev is not None:
if funcalls >= maxfev:
break
if maxiter is not None:
if niter >= maxiter:
break
return OptimizeResult(fun=fun(np.copy(x0)),
x=x0,
nit=niter,
nfev=funcalls,
success=(niter > 1))
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If `polish`
was employed, and a lower minimum was obtained by the polishing,
then OptimizeResult also contains the ``jac`` attribute.
"""
nit, warning_flag = 0, False
status_message = _status_message['success']
# The population may have just been initialized (all entries are
# np.inf). If it has you have to calculate the initial energies.
# Although this is also done in the evolve generator it's possible
# that someone can set maxiter=0, at which point we still want the
# initial energies to be calculated (the following loop isn't run).
if np.all(np.isinf(self.population_energies)):
self.population_energies[:] = self._calculate_population_energies(
self.population)
self._promote_lowest_energy()
# do the optimisation.
for nit in xrange(1, self.maxiter + 1):
# evolve the population by a generation
try:
next(self)
except StopIteration:
warning_flag = True
if self._nfev > self.maxfun:
status_message = _status_message['maxfev']
elif self._nfev == self.maxfun:
status_message = ('Maximum number of function evaluations'
' has been reached.')
break
if self.disp:
print("differential_evolution step %d: f(x)= %g"
% (nit,
self.population_energies[0]))
# should the solver terminate?
convergence = self.convergence
if (self.callback and
self.callback(self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
break
if np.any(np.isinf(self.population_energies)):
intol = False
else:
intol = (np.std(self.population_energies) <=
self.atol +
self.tol * np.abs(np.mean(self.population_energies)))
if warning_flag or intol:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(
x=self.x,
fun=self.population_energies[0],
nfev=self._nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T)
self._nfev += result.nfev
DE_result.nfev = self._nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def _quadratic_assignment_faq(
A,
B,
maximize=False,
partial_match=None,
S=None,
rng=None,
P0="barycenter",
shuffle_input=False,
maxiter=30,
tol=0.03,
):
r"""
Solve the quadratic assignment problem (approximately).
This function solves the Quadratic Assignment Problem (QAP) and the
Graph Matching Problem (GMP) using the Fast Approximate QAP Algorithm
(FAQ) [1]_.
Quadratic assignment solves problems of the following form:
.. math::
\min_P & \ {\ \text{trace}(A^T P B P^T)}\\
\mbox{s.t. } & {P \ \epsilon \ \mathcal{P}}\\
where :math:`\mathcal{P}` is the set of all permutation matrices,
and :math:`A` and :math:`B` are square matrices.
Graph matching tries to *maximize* the same objective function.
This algorithm can be thought of as finding the alignment of the
nodes of two graphs that minimizes the number of induced edge
disagreements, or, in the case of weighted graphs, the sum of squared
edge weight differences.
Note that the quadratic assignment problem is NP-hard, is not
known to be solvable in polynomial time, and is computationally
intractable. Therefore, the results given are approximations,
not guaranteed to be exact solutions.
Parameters
----------
A : 2d-array, square
The square matrix :math:`A` in the objective function above.
B : 2d-array, square
The square matrix :math:`B` in the objective function above.
method : str in {'faq', '2opt'} (default: 'faq')
The algorithm used to solve the problem. This is the method-specific
documentation for 'faq'.
:ref:`'2opt' <optimize.qap-2opt>` is also available.
Options
-------
maximize : bool (default = False)
Setting `maximize` to ``True`` solves the Graph Matching Problem (GMP)
rather than the Quadratic Assingnment Problem (QAP). This is
accomplished through trivial negation of the objective function.
rng : {None, int, `~np.random.RandomState`, `~np.random.Generator`}
This parameter defines the object to use for drawing random
variates.
If `rng` is ``None`` the `~np.random.RandomState` singleton is
used.
If `rng` is an int, a new ``RandomState`` instance is used,
seeded with `rng`.
If `rng` is already a ``RandomState`` or ``Generator``
instance, then that object is used.
Default is None.
partial_match : 2d-array of integers, optional, (default = None)
Allows the user to fix part of the matching between the two
matrices. In the literature, a partial match is also known as a
"seed".
Each row of `partial_match` specifies the indices of a pair of
corresponding nodes, that is, node ``partial_match[i, 0]`` of `A` is
matched to node ``partial_match[i, 1]`` of `B`. Accordingly,
``partial_match`` is an array of size ``(m , 2)``, where ``m`` is
not greater than the number of nodes, :math:`n`.
S : 2d-array, square
A similarity matrix. Should be same shape as ``A`` and ``B``.
Note: the scale of `S` may effect the weight placed on the term
:math:`\\text{trace}(S^T P)` relative to :math:`\\text{trace}(A^T PBP^T)`
during the optimization process.
P0 : 2d-array, "barycenter", or "randomized" (default = "barycenter")
The initial (guess) permutation matrix or search "position"
`P0`.
`P0` need not be a proper permutation matrix;
however, it must be :math:`m' x m'`, where :math:`m' = n - m`,
and it must be doubly stochastic: each of its rows and columns must
sum to 1.
If unspecified or ``"barycenter"``, the non-informative "flat
doubly stochastic matrix" :math:`J = 1*1^T/m'`, where :math:`1` is
a :math:`m' \times 1` array of ones, is used. This is the "barycenter"
of the search space of doubly-stochastic matrices.
If ``"randomized"``, the algorithm will start from the
randomized initial search position :math:`P_0 = (J + K)/2`,
where :math:`J` is the "barycenter" and :math:`K` is a random
doubly stochastic matrix.
shuffle_input : bool (default = False)
To avoid artificially high or low matching due to inherent
sorting of input matrices, gives users the option
to shuffle the nodes. Results are then unshuffled so that the
returned results correspond with the node order of inputs.
Shuffling may cause the algorithm to be non-deterministic,
unless a random seed is set or an `rng` option is provided.
maxiter : int, positive (default = 30)
Integer specifying the max number of Franke-Wolfe iterations performed.
tol : float (default = 0.03)
A threshold for the stopping criterion. Franke-Wolfe
iteration terminates when the change in search position between
iterations is sufficiently small, that is, when the relative Frobenius
norm, :math:`\frac{||P_{i}-P_{i+1}||_F}{\sqrt{len(P_{i})}} \leq tol`,
where :math:`i` is the iteration number.
Returns
-------
res : OptimizeResult
A :class:`scipy.optimize.OptimizeResult` containing the following
fields.
col_ind : 1-D array
An array of column indices corresponding with the best
permutation of the nodes of `B` found.
fun : float
The corresponding value of the objective function.
nit : int
The number of Franke-Wolfe iterations performed.
Notes
-----
The algorithm may be sensitive to the initial permutation matrix (or
search "position") due to the possibility of several local minima
within the feasible region. A barycenter initialization is more likely to
result in a better solution than a single random initialization. However,
``quadratic_assignment`` calling several times with different random
initializations may result in a better optimum at the cost of longer
total execution time.
Examples
--------
As mentioned above, a barycenter initialization often results in a better
solution than a single random initialization.
>>> np.random.seed(0)
>>> n = 15
>>> A = np.random.rand(n, n)
>>> B = np.random.rand(n, n)
>>> res = quadratic_assignment(A, B) # FAQ is default method
>>> print(res.fun)
46.871483385480545 # may vary
>>> options = {"P0": "randomized"} # use randomized initialization
>>> res = quadratic_assignment(A, B, options=options)
>>> print(res.fun)
47.224831071310625 # may vary
However, consider running from several randomized initializations and
keeping the best result.
>>> res = min([quadratic_assignment(A, B, options=options)
... for i in range(30)], key=lambda x: x.fun)
>>> print(res.fun)
46.671852533681516 # may vary
The '2-opt' method can be used to further refine the results.
>>> options = {"partial_guess": np.array([np.arange(n), res.col_ind]).T}
>>> res = quadratic_assignment(A, B, method="2opt", options=options)
>>> print(res.fun)
46.47160735721583 # may vary
References
----------
.. [1] J.T. Vogelstein, J.M. Conroy, V. Lyzinski, L.J. Podrazik,
S.G. Kratzer, E.T. Harley, D.E. Fishkind, R.J. Vogelstein, and
C.E. Priebe, "Fast approximate quadratic programming for graph
matching," PLOS one, vol. 10, no. 4, p. e0121002, 2015,
:doi:`10.1371/journal.pone.0121002`
.. [2] D. Fishkind, S. Adali, H. Patsolic, L. Meng, D. Singh, V. Lyzinski,
C. Priebe, "Seeded graph matching", Pattern Recognit. 87 (2019):
203-215, :doi:`10.1016/j.patcog.2018.09.014`
"""
maxiter = operator.index(maxiter)
# ValueError check
A, B, partial_match = _common_input_validation(A, B, partial_match)
msg = None
if isinstance(P0, str) and P0 not in {"barycenter", "randomized"}:
msg = "Invalid 'P0' parameter string"
elif maxiter <= 0:
msg = "'maxiter' must be a positive integer"
elif tol <= 0:
msg = "'tol' must be a positive float"
elif S.shape[0] != S.shape[1]:
msg = "`S` must be square"
elif S.ndim != 2:
msg = "`S` must have exactly two dimensions"
elif S.shape != A.shape:
msg = "`S`, `A`, and `B` matrices must be of equal size"
if msg is not None:
raise ValueError(msg)
rng = check_random_state(rng)
n = A.shape[0] # number of vertices in graphs
n_seeds = partial_match.shape[0] # number of seeds
n_unseed = n - n_seeds
# check outlier cases
if n == 0 or partial_match.shape[0] == n:
# Cannot assume partial_match is sorted.
partial_match = np.row_stack(sorted(partial_match, key=lambda x: x[0]))
score = _calc_score(A, B, S, partial_match[:, 1])
res = {"col_ind": partial_match[:, 1], "fun": score, "nit": 0}
return OptimizeResult(res)
obj_func_scalar = 1
if maximize:
obj_func_scalar = -1
nonseed_B = np.setdiff1d(range(n), partial_match[:, 1])
perm_S = np.copy(nonseed_B)
if shuffle_input:
nonseed_B = rng.permutation(nonseed_B)
# shuffle_input to avoid results from inputs that were already matched
nonseed_A = np.setdiff1d(range(n), partial_match[:, 0])
perm_A = np.concatenate([partial_match[:, 0], nonseed_A])
perm_B = np.concatenate([partial_match[:, 1], nonseed_B])
S = S[:, perm_B]
# definitions according to Seeded Graph Matching [2].
A11, A12, A21, A22 = _split_matrix(A[perm_A][:, perm_A], n_seeds)
B11, B12, B21, B22 = _split_matrix(B[perm_B][:, perm_B], n_seeds)
S22 = S[perm_S, n_seeds:]
# [1] Algorithm 1 Line 1 - choose initialization
if isinstance(P0, str):
# initialize J, a doubly stochastic barycenter
J = np.ones((n_unseed, n_unseed)) / n_unseed
if P0 == "barycenter":
P = J
elif P0 == "randomized":
# generate a nxn matrix where each entry is a random number [0, 1]
# would use rand, but Generators don't have it
# would use random, but old mtrand.RandomStates don't have it
K = rng.uniform(size=(n_unseed, n_unseed))
# Sinkhorn balancing
K = _doubly_stochastic(K)
P = J * 0.5 + K * 0.5
elif isinstance(P0, np.ndarray):
P0 = np.atleast_2d(P0)
_check_init_input(P0, n_unseed)
invert_inds = np.argsort(nonseed_B)
perm_nonseed_B = np.argsort(invert_inds)
P = P0[:, perm_nonseed_B]
else:
msg = "`init` must either be of type str or np.ndarray."
raise TypeError(msg)
const_sum = A21 @ B21.T + A12.T @ B12 + S22
# [1] Algorithm 1 Line 2 - loop while stopping criteria not met
for n_iter in range(1, maxiter + 1):
# [1] Algorithm 1 Line 3 - compute the gradient of f(P) = -tr(APB^tP^t)
grad_fp = const_sum + A22 @ P @ B22.T + A22.T @ P @ B22
# [1] Algorithm 1 Line 4 - get direction Q by solving Eq. 8
_, cols = linear_sum_assignment(grad_fp, maximize=maximize)
Q = np.eye(n_unseed)[cols]
# [1] Algorithm 1 Line 5 - compute the step size
# Noting that e.g. trace(Ax) = trace(A)*x, expand and re-collect
# terms as ax**2 + bx + c. c does not affect location of minimum
# and can be ignored. Also, note that trace(A@B) = (A.T*B).sum();
# apply where possible for efficiency.
R = P - Q
b21 = ((R.T @ A21) * B21).sum()
b12 = ((R.T @ A12.T) * B12.T).sum()
AR22 = A22.T @ R
BR22 = B22 @ R.T
b22a = (AR22 * B22.T[cols]).sum()
b22b = (A22 * BR22[cols]).sum()
s = (S22 * R).sum()
a = (AR22.T * BR22).sum()
b = b21 + b12 + b22a + b22b + s
# critical point of ax^2 + bx + c is at x = -d/(2*e)
# if a * obj_func_scalar > 0, it is a minimum
# if minimum is not in [0, 1], only endpoints need to be considered
if a * obj_func_scalar > 0 and 0 <= -b / (2 * a) <= 1:
alpha = -b / (2 * a)
else:
alpha = np.argmin([0, (b + a) * obj_func_scalar])
# [1] Algorithm 1 Line 6 - Update P
P_i1 = alpha * P + (1 - alpha) * Q
if np.linalg.norm(P - P_i1) / np.sqrt(n_unseed) < tol:
P = P_i1
break
P = P_i1
# [1] Algorithm 1 Line 7 - end main loop
# [1] Algorithm 1 Line 8 - project onto the set of permutation matrices
_, col = linear_sum_assignment(-P)
perm = np.concatenate((np.arange(n_seeds), col + n_seeds))
unshuffled_perm = np.zeros(n, dtype=int)
unshuffled_perm[perm_A] = perm_B[perm]
score = _calc_score(A, B, S, unshuffled_perm)
res = {"col_ind": unshuffled_perm, "fun": score, "nit": n_iter}
return OptimizeResult(res)
def minimize_ipopt_sparse(fun,
x0,
jacobianstructure,
args=(),
kwargs=None,
method=None,
jac=None,
hess=None,
hessp=None,
bounds=None,
constraints=(),
tol=None,
callback=None,
options=None):
if not CYIPOPT_INSTALLED:
raise ImportError(''' IPOPT functionality unavailable because
cyipopt is not installed properly. Please visit
https://github.com/matthias-k/cyipopt. The package is best installed
using the Anaconda environment.''')
import cyipopt
from ipopt.ipopt_wrapper import IpoptProblemWrapper, get_constraint_bounds, \
replace_option, convert_to_bytes, get_bounds
class SparseIpoptProblemWrapper(IpoptProblemWrapper):
def __init__(self, jacobianstructure, *args, **kwargs):
super().__init__(*args, **kwargs)
self.jacobianstructure = lambda: jacobianstructure
def jacobian(self, x):
return self._constraint_jacs[0](x, *self._constraint_args[0]).data
_x0 = np.atleast_1d(x0)
problem = SparseIpoptProblemWrapper(jacobianstructure,
fun,
args=args,
kwargs=kwargs,
jac=jac,
hess=hess,
hessp=hessp,
constraints=constraints)
lb, ub = get_bounds(bounds)
cl, cu = get_constraint_bounds(constraints, x0)
if options is None:
options = {}
nlp = cyipopt.problem(n=len(_x0),
m=len(cl),
problem_obj=problem,
lb=lb,
ub=ub,
cl=cl,
cu=cu)
# python3 compatibility
convert_to_bytes(options)
# Rename some default scipy options
replace_option(options, b'disp', b'print_level')
replace_option(options, b'maxiter', b'max_iter')
if b'print_level' not in options:
options[b'print_level'] = 0
if b'tol' not in options:
options[b'tol'] = tol or 1e-8
if b'mu_strategy' not in options:
options[b'mu_strategy'] = b'adaptive'
if b'hessian_approximation' not in options:
if hess is None and hessp is None:
options[b'hessian_approximation'] = b'limited-memory'
for option, value in options.items():
try:
nlp.addOption(option, value)
except TypeError as e:
raise TypeError(
'Invalid option for IPOPT: {0}: {1} (Original message: "{2}")'.
format(option, value, e))
x, info = nlp.solve(_x0)
if np.asarray(x0).shape == ():
x = x[0]
from scipy.optimize import OptimizeResult
return OptimizeResult(x=x,
success=info['status'] == 0,
status=info['status'],
message=info['status_msg'],
fun=info['obj_val'],
info=info,
nfev=problem.nfev,
njev=problem.njev,
nit=problem.nit)
def _minimize_neldermead_para(func, x0, callback=None,
xtol=1e-4, ftol=1e-4, maxiter=None, maxfev=None,
disp=False, return_all=False,pool=None, nthreads=1,
nverts=None, nactive=None):
maxfun = maxfev
retall = return_all
allvecs = []
x0 = np.asfarray(x0).flatten()
N = len(x0)
fcalls = [0]
allcalls = []
if nverts is None:
nverts = (N+1)*2
assert(nverts>=(N+1))
if nthreads>=nverts:
nthreads = nverts
pool = workerpool.pool(func, nthreads)
if nactive is None:
nactive = nthreads # number of active points
if nactive >= nverts:
nactive = nverts-1
def applicator(i, x):
#print x
pool.apply_async(i, x)
fcalls[0] +=1
allcalls.append(x.tolist())
rank = len(x0.shape)
if not -1 < rank < 2:
raise ValueError("Initial guess must be a scalar or rank-1 sequence.")
if maxiter is None:
maxiter = N * 200
if maxfun is None:
maxfun = N * 200
rho = 1
chi = 2
psi = 0.5
sigma = 0.5
nonzdelt = 0.05
zdelt = 0.00025
excess = nverts - N -1
one2np1 = list(range(1, nverts))
if rank == 0:
sim = np.zeros((nverts,), dtype=x0.dtype)
else:
sim = np.zeros((nverts, N), dtype=x0.dtype)
fsim = np.zeros((nverts,), float)
applicator(0, x0)
sim[0] = x0
if False:
for k in range(0, N):
y = np.array(x0, copy=True)
if y[k] != 0:
y[k] = (1 + nonzdelt) * y[k]
else:
y[k] = zdelt
sim[k + 1] = y
applicator(k + 1, y)
nover = int(np.ceil(excess *1. / N))
for k in range(N, nverts-1):
pos1 = (k-N) / nover
pos2 = (k-N) % nover
#print k,pos1,pos2
y = (sim[1+pos1]-x0) * (pos2+1)*1./(nover+1)+x0
#y = np.array(x0, copy=True)
#fac= np.random.uniform(0,zdelt,size=N)
#y = (x0*(1+fac))*(x0!=0).astype(int)+(fac)*(x0==0).astype(int)
sim[k + 1] = y
applicator(k + 1, y)
state=np.random.get_state()
np.random.seed(1)
for k in range(0, nverts-1):
y = np.array(x0, copy=True)
rand=np.random.normal(0,1, len(x0))
rand=rand/(rand**2).sum()**.5+1./len(x0)**.5
y = (x0)*(1+rand*nonzdelt)*(x0!=0).astype(int)+(x0==0).astype(int)*(rand)*nonzdelt
sim[k + 1] = y
applicator(k + 1, y)
np.random.set_state(state)
for k in range(0, nverts):
fsim[k] = pool.get(k)
ind = np.argsort(fsim)
fsim = np.take(fsim, ind, 0)
sim = np.take(sim, ind, 0)
iterations = 1
simR = sim.copy() # reflection
simE = sim.copy() # expansion
simC = sim.copy() # contraction
simFinal = sim.copy() # final
fsimR = fsim.copy()
fsimE = fsim.copy()
fsimC = fsim.copy()
fsimFinal = fsim.copy()
while (fcalls[0] < maxfun and iterations < maxiter):
if (np.max(np.ravel(np.abs(sim[1:] - sim[0]))) <= xtol and
np.max(np.abs(fsim[0] - fsim[1:])) <= ftol):
break
xbar = np.add.reduce(sim[:-nactive], 0) / (nverts - nactive)
state = np.zeros(nactive, dtype=int)
actives = set(range(nactive))
shrinks = []
for i in range(nactive):
# reflect bad points
#xbar = (np.add.reduce(sim[:], 0)-sim[-i-1,:]) / N
simR[-i - 1, :] = (1 + rho) * xbar - rho * sim[-i-1,:]
applicator(i, simR[-i-1, :])
state[i] = 1
#print 'A'
while len(actives)>0:
#print 'ST', state
i, curfsim = pool.get_any()
#xbar = (np.add.reduce(sim[:], 0)-sim[-i-1,:]) / N
if state[i] == 1: # point just got evaled after first reflection
fsimR[-i-1] = curfsim
if fsimR[-i-1] < fsim[0]: # expand
simE[-i-1, :] = (1 + rho * chi) * xbar - rho * chi* sim[-i -1, :]
applicator(i, simE[-i - 1, :])
state[i] = 2
else:
if fsimR[-i-1] < fsim[-i-2]: # better than the next worst
simFinal[-i-1, :] = simR[-i-1,:]
fsimFinal[-i-1] = fsimR[-i-1]
state[i]= -1 # we stopped after one reflection
actives.remove(i)
else: # contract
if fsimR[-i-1] < fsim[-i-1]:
simC[-i-1, :] = (1 + rho * psi) * xbar - rho * psi* sim[-i-1, :]
applicator(i, simC[-i-1,:])
state[i] = 3
else:
simC[-i- 1, :] = (1 - psi) * xbar + psi * sim[-i-1, :]
applicator(i, simC[-i-1,:])
state[i] = 4
elif state[i] == 2: # the expansion step has been done
fsimE[-i-1] = curfsim
#print 'TTT', fsimE[-i-1] , fsim[0]
if fsimE[-i-1] < fsimR[-i-1]: # still better than the best
simFinal[-i-1, :] = simE[-i-1,:]
fsimFinal[-i-1] = fsimE[-i-1]
state[i] = -2
actives.remove(i)
else:
simFinal[-i-1, :] = simR[-i-1,:]
fsimFinal[-i-1] = fsimR[-i-1]
state[i] = -3
actives.remove(i)
elif state[i] == 3: # contraction1 just done
fsimC[-i-1] = curfsim
if fsimC[-i-1] < fsimR[-i-1]: # still better than the reflected
simFinal[-i-1, :] = simC[-i-1,:]
fsimFinal[-i-1] = fsimC[-i-1]
state[i] = -4
actives.remove(i)
else:
simFinal[-i-1, :] = sim[-i-1,:]
fsimFinal[-i-1] = fsim[-i-1]
shrinks.append(i)
state[i] = -10
actives.remove(i)
elif state[i] == 4: # contraction2 just done
fsimC[-i-1] = curfsim
if fsimC[-i-1] < fsim[-i-1]: # still better than the reflected
simFinal[-i-1, :] = simC[-i-1,:]
fsimFinal[-i-1] = fsimC[-i-1]
state[i] = -5
actives.remove(i)
else:
simFinal[-i-1, :] = sim[-i-1,:]
fsimFinal[-i-1] = fsim[-i-1]
shrinks.append(i)
state[i] = -10
actives.remove(i)
else:
print 'Weird'
for i in range(nactive):
sim[-i-1,:] = simFinal[-i-1,:]
fsim[-i-1] = fsimFinal[-i-1]
#print 'ST1', state
if len(shrinks) == nactive:
print 'Shrink....'
# no change was positive
for j in one2np1:
sim[j] = sim[0] + sigma * (sim[j] - sim[0])
applicator(j, sim[j, :])
for j in one2np1:
fsim[j] = pool.get(j)
ind = np.argsort(fsim)
sim = np.take(sim, ind, 0)
fsim = np.take(fsim, ind, 0)
if callback is not None:
callback(sim[0])
iterations += 1
if retall:
allvecs.append(sim)
#allvecs = allcalls
x = sim[0]
fval = np.min(fsim)
warnflag = 0
if fcalls[0] >= maxfun:
warnflag = 1
msg = _status_message['maxfev']
if disp:
print('Warning: ' + msg)
elif iterations >= maxiter:
warnflag = 2
msg = _status_message['maxiter']
if disp:
print('Warning: ' + msg)
else:
msg = _status_message['success']
if disp:
print(msg)
print(" Current function value: %f" % fval)
print(" Iterations: %d" % iterations)
print(" Function evaluations: %d" % fcalls[0])
result = OptimizeResult(fun=fval, nit=iterations, nfev=fcalls[0],
status=warnflag, success=(warnflag == 0), message=msg,
x=x)
if retall:
result['allvecs'] = allvecs
return result
def _minimize_anneal(x0, args, schedule, T0, Tf, maxfev, maxaccept, maxiter,
boltzmann, learn_rate, ftol, quench, m, n, lower, upper,
dwell, disp):
"""
Minimization of scalar function of one or more variables using the
simulated annealing algorithm.
Options for the simulated annealing algorithm are:
disp : bool
Set to True to print convergence messages.
schedule : str
Annealing schedule to use. One of: 'fast', 'cauchy' or
'boltzmann'.
T0 : float
Initial Temperature (estimated as 1.2 times the largest
cost-function deviation over random points in the range).
Tf : float
Final goal temperature.
maxfev : int
Maximum number of function evaluations to make.
maxaccept : int
Maximum changes to accept.
maxiter : int
Maximum number of iterations to perform.
boltzmann : float
Boltzmann constant in acceptance test (increase for less
stringent test at each temperature).
learn_rate : float
Scale constant for adjusting guesses.
ftol : float
Relative error in ``fun(x)`` acceptable for convergence.
quench, m, n : float
Parameters to alter fast_sa schedule.
lower, upper : float or ndarray
Lower and upper bounds on `x`.
dwell : int
The number of times to search the space at each temperature.
This function is called by the `minimize` function with
`method=anneal`. It is not supposed to be called directly.
"""
#_check_unknown_options(unknown_options)
maxeval = maxfev
feps = ftol
#x0 = asarray(x0)
lower = float(lower)
upper = float(upper)
#schedule = eval(schedule+'_sa()')
# initialize the schedule
schedule = cauchy_sa(numpy.array([2], dtype=numpy.int64), boltzmann, T0,
learn_rate, lower, upper, m, n, quench, dwell)
current_state, last_state, best_state = _state(), _state(), _state()
if T0 is None:
x0 = schedule.getstart_temp(best_state)
else:
best_state.x = None
best_state.cost = numpy.Inf
last_state.x = asarray(x0).copy()
fval = rosen(asarray(x0))
schedule.feval += 1
last_state.cost = fval
if last_state.cost < best_state.cost:
best_state.cost = fval
best_state.x = asarray(x0).copy()
schedule.T = schedule.T0
fqueue = [100, 300, 500, 700]
iters = 0
while 1:
for n in range(dwell):
current_state.x = schedule.update_guess(last_state.x)
current_state.cost = rosen(current_state.x)
schedule.feval += 1
dE = current_state.cost - last_state.cost
if schedule.accept_test(dE):
last_state.x = current_state.x.copy()
last_state.cost = current_state.cost
if last_state.cost < best_state.cost:
best_state.x = last_state.x.copy()
best_state.cost = last_state.cost
schedule.update_temp()
iters += 1
# Stopping conditions
# 0) last saved values of f from each cooling step
# are all very similar (effectively cooled)
# 1) Tf is set and we are below it
# 2) maxeval is set and we are past it
# 3) maxiter is set and we are past it
# 4) maxaccept is set and we are past it
fqueue.append((last_state.cost))
fqueue.pop(0)
af = asarray(fqueue) * 1.0
if all(abs((af - af[0]) / af[0]) < feps):
retval = 0
if abs(af[-1] - best_state.cost) > feps * 10:
retval = 5
if disp:
print("Warning: Cooled to %f at %s but this is not" %
((last_state.cost), str((last_state.x))) +
" the smallest point found.")
break
if (Tf is not None) and (schedule.T < Tf):
retval = 1
break
if (maxeval is not None) and (schedule.feval > maxeval):
retval = 2
break
if (iters > maxiter):
if disp:
print("Warning: Maximum number of iterations exceeded.")
retval = 3
break
if (maxaccept is not None) and (schedule.accepted > maxaccept):
retval = 4
break
result = OptimizeResult(x=best_state.x,
fun=best_state.cost,
T=schedule.T,
nfev=schedule.feval,
nit=iters,
accept=schedule.accepted,
status=retval,
success=(retval <= 1),
message={
0:
'Points no longer changing',
1:
'Cooled to final temperature',
2:
'Maximum function evaluations',
3:
'Maximum cooling iterations reached',
4:
'Maximum accepted query locations reached',
5:
'Final point not the minimum amongst '
'encountered points'
}[retval])
return result
def rol_minimize(fun,
x0,
method=None,
jac=None,
hess=None,
hessp=None,
bounds=None,
constraints=(),
tol=None,
options={},
x_grad=None):
obj = ROLObj(fun, jac, hess, hessp)
if x_grad is not None:
print("Testing objective", flush=True)
xg = get_rol_numpy_vector(x_grad)
d = get_rol_numpy_vector(np.random.normal(0, 1, (x_grad.shape[0])))
obj.checkGradient(xg, d, 12, 1)
obj.checkHessVec(xg, d, 12, 1)
use_bfgs = False
if hess is None and hessp is None:
use_bfgs = True
if type(hess) == BFGS:
use_bfgs = True
for constr in constraints:
if (type(constr) != LinearConstraint
and (type(constr.hess) == BFGS or constr.hess is None)):
use_bfgs = True
constr.hess = None
assert method == 'rol-trust-constr' or method == None
if 'step-type' in options:
rol_method = options['step-type']
del options['step-type']
else:
rol_method = 'Augmented Lagrangian'
params = get_rol_parameters(rol_method, use_bfgs, options)
x = get_rol_numpy_vector(x0)
bnd, econ, emul, icon, imul, ibnd = get_constraints(
constraints, bounds, x_grad)
optimProblem = ROL.OptimizationProblem(obj,
x,
bnd=bnd,
econs=econ,
emuls=emul,
icons=icon,
imuls=imul,
ibnds=ibnd)
solver = ROL.OptimizationSolver(optimProblem, params)
solver.solve(options.get('verbose', 0))
state = solver.getAlgorithmState()
success = state.statusFlag.name == 'EXITSTATUS_CONVERGED'
res = OptimizeResult(
x=rol_vector_to_numpy(x),
fun=state.value,
cnorm=state.cnorm,
gnorm=state.gnorm,
snorm=state.snorm,
success=success,
nit=state.iter,
nfev=state.nfval,
ngev=state.ngrad,
constr_nfev=state.ncval,
status=state.statusFlag.name,
message=f'Optimization terminated early {state.statusFlag.name}')
return res
def _minimize_slsqp(func,
x0,
args=(),
jac=None,
bounds=None,
constraints=(),
maxiter=100,
ftol=1.0E-6,
iprint=1,
disp=False,
eps=_epsilon,
callback=None,
finite_diff_rel_step=None,
**unknown_options):
"""
Minimize a scalar function of one or more variables using Sequential
Least Squares Programming (SLSQP).
Options
-------
ftol : float
Precision goal for the value of f in the stopping criterion.
eps : float
Step size used for numerical approximation of the Jacobian.
disp : bool
Set to True to print convergence messages. If False,
`verbosity` is ignored and set to 0.
maxiter : int
Maximum number of iterations.
finite_diff_rel_step : None or array_like, optional
If `jac in ['2-point', '3-point', 'cs']` the relative step size to
use for numerical approximation of `jac`. The absolute step
size is computed as ``h = rel_step * sign(x0) * max(1, abs(x0))``,
possibly adjusted to fit into the bounds. For ``method='3-point'``
the sign of `h` is ignored. If None (default) then step is selected
automatically.
"""
_check_unknown_options(unknown_options)
iter = maxiter - 1
acc = ftol
epsilon = eps
if not disp:
iprint = 0
# Constraints are triaged per type into a dictionary of tuples
if isinstance(constraints, dict):
constraints = (constraints, )
cons = {'eq': (), 'ineq': ()}
for ic, con in enumerate(constraints):
# check type
try:
ctype = con['type'].lower()
except KeyError:
raise KeyError('Constraint %d has no type defined.' % ic)
except TypeError:
raise TypeError('Constraints must be defined using a '
'dictionary.')
except AttributeError:
raise TypeError("Constraint's type must be a string.")
else:
if ctype not in ['eq', 'ineq']:
raise ValueError("Unknown constraint type '%s'." % con['type'])
# check function
if 'fun' not in con:
raise ValueError('Constraint %d has no function defined.' % ic)
# check Jacobian
cjac = con.get('jac')
if cjac is None:
# approximate Jacobian function. The factory function is needed
# to keep a reference to `fun`, see gh-4240.
def cjac_factory(fun):
def cjac(x, *args):
if jac in ['2-point', '3-point', 'cs']:
return approx_derivative(fun,
x,
method=jac,
args=args,
rel_step=finite_diff_rel_step)
else:
return approx_derivative(fun,
x,
method='2-point',
abs_step=epsilon,
args=args)
return cjac
cjac = cjac_factory(con['fun'])
# update constraints' dictionary
cons[ctype] += ({
'fun': con['fun'],
'jac': cjac,
'args': con.get('args', ())
}, )
exit_modes = {
-1: "Gradient evaluation required (g & a)",
0: "Optimization terminated successfully",
1: "Function evaluation required (f & c)",
2: "More equality constraints than independent variables",
3: "More than 3*n iterations in LSQ subproblem",
4: "Inequality constraints incompatible",
5: "Singular matrix E in LSQ subproblem",
6: "Singular matrix C in LSQ subproblem",
7: "Rank-deficient equality constraint subproblem HFTI",
8: "Positive directional derivative for linesearch",
9: "Iteration limit reached"
}
# Transform x0 into an array.
x = asfarray(x0).flatten()
# SLSQP is sent 'old-style' bounds, 'new-style' bounds are required by
# ScalarFunction
if bounds is None or len(bounds) == 0:
new_bounds = (-np.inf, np.inf)
else:
new_bounds = old_bound_to_new(bounds)
# clip the initial guess to bounds, otherwise ScalarFunction doesn't work
x = np.clip(x, new_bounds[0], new_bounds[1])
# Set the parameters that SLSQP will need
# meq, mieq: number of equality and inequality constraints
meq = sum(
map(len, [atleast_1d(c['fun'](x, *c['args'])) for c in cons['eq']]))
mieq = sum(
map(len, [atleast_1d(c['fun'](x, *c['args'])) for c in cons['ineq']]))
# m = The total number of constraints
m = meq + mieq
# la = The number of constraints, or 1 if there are no constraints
la = array([1, m]).max()
# n = The number of independent variables
n = len(x)
# Define the workspaces for SLSQP
n1 = n + 1
mineq = m - meq + n1 + n1
len_w = (3*n1+m)*(n1+1)+(n1-meq+1)*(mineq+2) + 2*mineq+(n1+mineq)*(n1-meq) \
+ 2*meq + n1 + ((n+1)*n)//2 + 2*m + 3*n + 3*n1 + 1
len_jw = mineq
w = zeros(len_w)
jw = zeros(len_jw)
# Decompose bounds into xl and xu
if bounds is None or len(bounds) == 0:
xl = np.empty(n, dtype=float)
xu = np.empty(n, dtype=float)
xl.fill(np.nan)
xu.fill(np.nan)
else:
bnds = array([(_arr_to_scalar(l), _arr_to_scalar(u))
for (l, u) in bounds], float)
if bnds.shape[0] != n:
raise IndexError('SLSQP Error: the length of bounds is not '
'compatible with that of x0.')
with np.errstate(invalid='ignore'):
bnderr = bnds[:, 0] > bnds[:, 1]
if bnderr.any():
raise ValueError('SLSQP Error: lb > ub in bounds %s.' %
', '.join(str(b) for b in bnderr))
xl, xu = bnds[:, 0], bnds[:, 1]
# Mark infinite bounds with nans; the Fortran code understands this
infbnd = ~isfinite(bnds)
xl[infbnd[:, 0]] = np.nan
xu[infbnd[:, 1]] = np.nan
# ScalarFunction provides function and gradient evaluation
sf = _prepare_scalar_function(func,
x,
jac=jac,
args=args,
epsilon=eps,
finite_diff_rel_step=finite_diff_rel_step,
bounds=new_bounds)
# Initialize the iteration counter and the mode value
mode = array(0, int)
acc = array(acc, float)
majiter = array(iter, int)
majiter_prev = 0
# Initialize internal SLSQP state variables
alpha = array(0, float)
f0 = array(0, float)
gs = array(0, float)
h1 = array(0, float)
h2 = array(0, float)
h3 = array(0, float)
h4 = array(0, float)
t = array(0, float)
t0 = array(0, float)
tol = array(0, float)
iexact = array(0, int)
incons = array(0, int)
ireset = array(0, int)
itermx = array(0, int)
line = array(0, int)
n1 = array(0, int)
n2 = array(0, int)
n3 = array(0, int)
# Print the header if iprint >= 2
if iprint >= 2:
print("%5s %5s %16s %16s" % ("NIT", "FC", "OBJFUN", "GNORM"))
# mode is zero on entry, so call objective, constraints and gradients
# there should be no func evaluations here because it's cached from
# ScalarFunction
fx = sf.fun(x)
try:
fx = float(np.asarray(fx))
except (TypeError, ValueError):
raise ValueError("Objective function must return a scalar")
g = append(sf.grad(x), 0.0)
c = _eval_constraint(x, cons)
a = _eval_con_normals(x, cons, la, n, m, meq, mieq)
while 1:
# Call SLSQP
slsqp(m, meq, x, xl, xu, fx, c, g, a, acc, majiter, mode, w, jw, alpha,
f0, gs, h1, h2, h3, h4, t, t0, tol, iexact, incons, ireset,
itermx, line, n1, n2, n3)
if mode == 1: # objective and constraint evaluation required
fx = sf.fun(x)
c = _eval_constraint(x, cons)
if mode == -1: # gradient evaluation required
g = append(sf.grad(x), 0.0)
a = _eval_con_normals(x, cons, la, n, m, meq, mieq)
if majiter > majiter_prev:
# call callback if major iteration has incremented
if callback is not None:
callback(np.copy(x))
# Print the status of the current iterate if iprint > 2
if iprint >= 2:
print("%5i %5i % 16.6E % 16.6E" %
(majiter, sf.nfev, fx, linalg.norm(g)))
# If exit mode is not -1 or 1, slsqp has completed
if abs(mode) != 1:
break
majiter_prev = int(majiter)
# Optimization loop complete. Print status if requested
if iprint >= 1:
print(exit_modes[int(mode)] + " (Exit mode " + str(mode) + ')')
print(" Current function value:", fx)
print(" Iterations:", majiter)
print(" Function evaluations:", sf.nfev)
print(" Gradient evaluations:", sf.ngev)
return OptimizeResult(x=x,
fun=fx,
jac=g[:-1],
nit=int(majiter),
nfev=sf.nfev,
njev=sf.ngev,
status=int(mode),
message=exit_modes[int(mode)],
success=(mode == 0))
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If `polish`
was employed, and a lower minimum was obtained by the polishing,
then OptimizeResult also contains the ``jac`` attribute.
"""
nit, warning_flag = 0, False
# dictionary that holds standard status messages of optimizers
status_message = _status_message['success']
# The population may have just been initialized (all entries are
# np.inf). If it has you have to calculate the initial energies.
# Although this is also done in the evolve generator it's possible
# that someone can set maxiter=0, at which point we still want the
# initial energies to be calculated (the following loop isn't run).
#np.all checks that there are no 0's in the array
if self.maxiter == 0:
if np.all(np.isinf(self.population_energies)):
if self.disp:
print("Calculating initial energies when maxiter = 0")
self._calculate_population_energies()
# for i in range(self.num_population_members):
# print(self.population[i,:])
# do the optimisation.
for nit in xrange(1, self.maxiter + 1):
if self.disp:
print("iter: ", nit)
# evolve the population by a generation
try:
next(self)
except StopIteration:
warning_flag = True
status_message = _status_message['maxfev']
break
print("differential_evolution step %d: f(x)= %g" %
(nit, self.population_energies[0]))
#save populations at each iter and rank to analyze after
# np.save("before_rank"+str(self.rank)+"iter"+str(nit), self.population)
#migrate
self.migration()
# np.save("after_rank"+str(self.rank)+"iter"+str(nit), self.population)
# should the solver terminate?
# print("Checking if should converge")
# convergence = self.convergence
#
# if (self.callback and
# self.callback(self._scale_parameters(self.population[0]),
# convergence=self.tol / convergence) is True):
#
# warning_flag = True
# status_message = ('callback function requested stop early '
# 'by returning True')
# break
# print("checking if tolerance level reached")
## intol = (np.std(self.population_energies) <=
## self.atol +
## self.tol * np.abs(np.mean(self.population_energies)))
#
# intol = self.population_energies[0] <= self.mse_thresh
# if warning_flag or intol:
# print("stopping iterations")
# break
print("Starting next iter")
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=self._nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
print("done iters")
if self.polish:
print("performing final polishing")
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
self._nfev += result.nfev
DE_result.nfev = self._nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def result(self): return OptimizeResult(x=self.x, message=self.message, nit=self.nit, status=-1, success=self.success, slow=self.slowness)
def _fit_no_arch_normal_errors(self, cov_type='robust'):
"""
Estimates model parameters
Parameters
----------
cov_type : str, optional
Covariance estimator to use when estimating parameter variances and
covariances. One of 'hetero' or 'heteroskedastic' for Whites's
covariance estimator, or 'mle' for the classic
OLS estimator appropriate for homoskedastic data. 'hetero' is the
the default.
Returns
-------
result : ARCHModelResult
Results class containing parameter estimates, estimated parameter
covariance and related estimates
Notes
-----
See :class:`ARCHModelResult` for details on computed results
"""
nobs = self._fit_y.shape[0]
if nobs < self.num_params:
raise ValueError('Insufficient data, ' + str(self.num_params) +
' regressors, ' + str(nobs) +
' data points available')
x = self._fit_regressors
y = self._fit_y
# Fake convergence results, see GH #87
opt = OptimizeResult({'status': 0, 'message': ''})
if x.shape[1] > 0:
regression_params = np.linalg.pinv(x).dot(y)
xpxi = np.linalg.inv(x.T.dot(x) / nobs)
fitted = x.dot(regression_params)
else:
regression_params = np.empty(0)
xpxi = np.empty((0, 0))
fitted = 0.0
e = y - fitted
sigma2 = e.T.dot(e) / nobs
params = np.hstack((regression_params, sigma2))
hessian = np.zeros((self.num_params + 1, self.num_params + 1))
hessian[:self.num_params, :self.num_params] = -xpxi
hessian[-1, -1] = -1
if cov_type in ('mle', ):
param_cov = sigma2 * -hessian
param_cov[self.num_params, self.num_params] = 2 * sigma2**2.0
param_cov /= nobs
cov_type = COV_TYPES['classic_ols']
elif cov_type in ('robust', ):
scores = np.zeros((nobs, self.num_params + 1))
scores[:, :self.num_params] = x * e[:, None]
scores[:, -1] = e**2.0 - sigma2
score_cov = scores.T.dot(scores) / nobs
param_cov = hessian.dot(score_cov).dot(hessian) / nobs
cov_type = COV_TYPES['white']
else:
raise ValueError('Unknown cov_type')
r2 = self._r2(regression_params)
first_obs, last_obs = self._fit_indices
resids = np.empty_like(self._y, dtype=np.float64)
resids.fill(np.nan)
resids[first_obs:last_obs] = e
vol = np.zeros_like(resids)
vol.fill(np.nan)
vol[first_obs:last_obs] = np.sqrt(sigma2)
names = self._all_parameter_names()
loglikelihood = self._static_gaussian_loglikelihood(e)
# Throw away names in the case of starting values
num_params = params.shape[0]
if len(names) != num_params:
names = ['p' + str(i) for i in range(num_params)]
fit_start, fit_stop = self._fit_indices
return ARCHModelResult(params, param_cov, r2, resids, vol, cov_type,
self._y_series, names, loglikelihood,
self._is_pandas, opt, fit_start, fit_stop,
copy.deepcopy(self))
def minimize_ipopt(fun,
x0,
args=(),
kwargs=None,
method=None,
jac=None,
hess=None,
hessp=None,
bounds=None,
constraints=(),
tol=None,
callback=None,
options=None):
"""
Minimize a function using ipopt. The call signature is exactly like for
`scipy.optimize.mimize`. In options, all options are directly passed to
ipopt. Check [http://www.coin-or.org/Ipopt/documentation/node39.html] for
details.
The options `disp` and `maxiter` are automatically mapped to their
ipopt-equivalents `print_level` and `max_iter`.
"""
_x0 = np.atleast_1d(x0)
problem = IpoptProblemWrapper(fun,
args=args,
kwargs=kwargs,
jac=jac,
hess=hess,
hessp=hessp,
constraints=constraints)
lb, ub = get_bounds(bounds)
cl, cu = get_constraint_bounds(constraints, x0)
if options is None:
options = {}
nlp = cyipopt.problem(n=len(_x0),
m=len(cl),
problem_obj=problem,
lb=lb,
ub=ub,
cl=cl,
cu=cu)
# Rename some default scipy options
replace_option(options, 'disp', 'print_level')
replace_option(options, 'maxiter', 'max_iter')
if b'print_level' not in options:
options[b'print_level'] = 0
if b'tol' not in options:
options[b'tol'] = tol or 1e-8
if b'mu_strategy' not in options:
options[b'mu_strategy'] = b'adaptive'
if b'hessian_approximation' not in options:
if hess is None and hessp is None:
options[b'hessian_approximation'] = b'limited-memory'
for option, value in options.items():
try:
nlp.addOption(option, value)
except TypeError as e:
raise TypeError(
'Invalid option for IPOPT: {0}: {1} (Original message: "{2}")'.
format(option, value, e))
x, info = nlp.solve(_x0)
if np.asarray(x0).shape == ():
x = x[0]
return OptimizeResult(x=x,
success=info['status'] == 0,
status=info['status'],
message=info['status_msg'],
fun=info['obj_val'],
info=info,
nfev=problem.nfev,
njev=problem.njev,
nit=problem.nit)
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If `polish`
was employed, and a lower minimum was obtained by the polishing,
then OptimizeResult also contains the ``jac`` attribute.
"""
nit, warning_flag = 0, False
status_message = _status_message['success']
# The population may have just been initialized (all entries are
# np.inf). If it has you have to calculate the initial energies.
# Although this is also done in the evolve generator it's possible
# that someone can set maxiter=0, at which point we still want the
# initial energies to be calculated (the following loop isn't run).
if np.all(np.isinf(self.population_energies)):
self._calculate_population_energies()
for nmig in xrange(1, self.number_of_migrations + 1):
if nmig != 1:
# Get the host node
host = int(self.island_marker[-1])
# Get all the neighbors list
neighbors = self.topology.neighbors(host)
neighbor_results = {}
neighbor_energy_results = {}
for each_neighbor in neighbors:
replacement = client.get(self.key + str(each_neighbor))
if replacement is None:
for _ in range(int(self.wait_time / self.poll_time)):
replacement = client.get(self.key +
str(each_neighbor))
if replacement is None:
print("POLLING!!!")
time.sleep(self.poll_time)
else:
break
if replacement is not None:
neighbor_results[each_neighbor] = np.array([
float(items)
for items in replacement.split(",")
])
neighbor_energy_results[each_neighbor] = self.func(
neighbor_results[each_neighbor], *self.args)
total_computed_neighbors = len(neighbor_results)
energies = []
for each_neighbor in neighbor_results.keys():
energies.append((neighbor_results[each_neighbor],
neighbor_energy_results[each_neighbor]))
for pop_index in range(1, total_computed_neighbors + 1):
energies.append((self.population[pop_index],
self.population_energies[pop_index]))
energies.sort(key=lambda x: x[-1])
energies = energies[:total_computed_neighbors]
for pop_index in range(1, total_computed_neighbors + 1):
self.population[pop_index] = energies[pop_index - 1][0]
self.population_energies[pop_index] = energies[pop_index -
1][1]
# do the optimisation.
is_optimisation_complete = False
for nit in xrange(1, self.maxiter + 1):
# evolve the population by a generation
try:
next(self)
except StopIteration:
warning_flag = True
status_message = _status_message['maxfev']
#is_optimisation_complete = False
break
if self.disp:
print("differential_evolution step %d: f(x)= %g" %
(nit, self.population_energies[0]))
# should the solver terminate?
convergence = self.convergence
if (self.callback and self.callback(
self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
is_optimisation_complete = False
break
intol = (np.std(self.population_energies) <= self.atol +
self.tol * np.abs(np.mean(self.population_energies)))
if intol:
is_optimisation_complete = False
if warning_flag or intol:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
client.set(self.island_marker,
",".join([str(items) for items in self.x]))
print("MARKED IN MEMCACHE")
print(self.island_marker,
",".join([str(items) for items in self.x]))
if not is_optimisation_complete:
#break
print("Exited due to some break condition above!!",
status_message)
DE_result = OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=self._nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
self._nfev += result.nfev
DE_result.nfev = self._nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def dogbox(fun, jac, x0, f0, J0, lb, ub, ftol, xtol, gtol, max_nfev, x_scale,
loss_function, tr_solver, tr_options, verbose):
f = f0
f_true = f.copy()
nfev = 1
J = J0
njev = 1
if loss_function is not None:
rho = loss_function(f)
cost = 0.5 * np.sum(rho[0])
J, f = scale_for_robust_loss_function(J, f, rho)
else:
cost = 0.5 * np.dot(f, f)
g = compute_grad(J, f)
jac_scale = isinstance(x_scale, string_types) and x_scale == 'jac'
if jac_scale:
scale, scale_inv = compute_jac_scale(J)
else:
scale, scale_inv = x_scale, 1 / x_scale
Delta = norm(x0 * scale_inv, ord=np.inf)
if Delta == 0:
Delta = 1.0
on_bound = np.zeros_like(x0, dtype=int)
on_bound[np.equal(x0, lb)] = -1
on_bound[np.equal(x0, ub)] = 1
x = x0
step = np.empty_like(x0)
if max_nfev is None:
max_nfev = x0.size * 100
termination_status = None
iteration = 0
step_norm = None
actual_reduction = None
if verbose == 2:
print_header_nonlinear()
while True:
active_set = on_bound * g < 0
free_set = ~active_set
g_free = g[free_set]
g_full = g.copy()
g[active_set] = 0
g_norm = norm(g, ord=np.inf)
if g_norm < gtol:
termination_status = 1
if verbose == 2:
print_iteration_nonlinear(iteration, nfev, cost, actual_reduction,
step_norm, g_norm)
if termination_status is not None or nfev == max_nfev:
break
x_free = x[free_set]
lb_free = lb[free_set]
ub_free = ub[free_set]
scale_free = scale[free_set]
# Compute (Gauss-)Newton and build quadratic model for Cauchy step.
if tr_solver == 'exact':
J_free = J[:, free_set]
newton_step = lstsq(J_free, -f)[0]
# Coefficients for the quadratic model along the anti-gradient.
a, b = build_quadratic_1d(J_free, g_free, -g_free)
elif tr_solver == 'lsmr':
Jop = aslinearoperator(J)
# We compute lsmr step in scaled variables and then
# transform back to normal variables, if lsmr would give exact lsq
# solution this would be equivalent to not doing any
# transformations, but from experience it's better this way.
# We pass active_set to make computations as if we selected
# the free subset of J columns, but without actually doing any
# slicing, which is expensive for sparse matrices and impossible
# for LinearOperator.
lsmr_op = lsmr_operator(Jop, scale, active_set)
newton_step = -lsmr(lsmr_op, f, **tr_options)[0][free_set]
newton_step *= scale_free
# Components of g for active variables were zeroed, so this call
# is correct and equivalent to using J_free and g_free.
a, b = build_quadratic_1d(Jop, g, -g)
actual_reduction = -1.0
while actual_reduction <= 0 and nfev < max_nfev:
tr_bounds = Delta * scale_free
step_free, on_bound_free, tr_hit = dogleg_step(
x_free, newton_step, g_free, a, b, tr_bounds, lb_free, ub_free)
step.fill(0.0)
step[free_set] = step_free
if tr_solver == 'exact':
predicted_reduction = -evaluate_quadratic(J_free, g_free,
step_free)
elif tr_solver == 'lsmr':
predicted_reduction = -evaluate_quadratic(Jop, g, step)
x_new = x + step
f_new = fun(x_new)
nfev += 1
step_h_norm = norm(step * scale_inv, ord=np.inf)
if not np.all(np.isfinite(f_new)):
Delta = 0.25 * step_h_norm
continue
# Usual trust-region step quality estimation.
if loss_function is not None:
cost_new = loss_function(f_new, cost_only=True)
else:
cost_new = 0.5 * np.dot(f_new, f_new)
actual_reduction = cost - cost_new
Delta, ratio = update_tr_radius(
Delta, actual_reduction, predicted_reduction,
step_h_norm, tr_hit
)
step_norm = norm(step)
termination_status = check_termination(
actual_reduction, cost, step_norm, norm(x), ratio, ftol, xtol)
if termination_status is not None:
break
if actual_reduction > 0:
on_bound[free_set] = on_bound_free
x = x_new
# Set variables exactly at the boundary.
mask = on_bound == -1
x[mask] = lb[mask]
mask = on_bound == 1
x[mask] = ub[mask]
f = f_new
f_true = f.copy()
cost = cost_new
J = jac(x, f)
njev += 1
if loss_function is not None:
rho = loss_function(f)
J, f = scale_for_robust_loss_function(J, f, rho)
g = compute_grad(J, f)
if jac_scale:
scale, scale_inv = compute_jac_scale(J, scale_inv)
else:
step_norm = 0
actual_reduction = 0
iteration += 1
if termination_status is None:
termination_status = 0
return OptimizeResult(
x=x, cost=cost, fun=f_true, jac=J, grad=g_full, optimality=g_norm,
active_mask=on_bound, nfev=nfev, njev=njev, status=termination_status)
def gp_minimize(func,
dimensions,
base_estimator=None,
alpha=10e-10,
acq="EI",
xi=0.01,
kappa=1.96,
search="auto",
n_calls=100,
n_points=500,
n_random_starts=10,
n_restarts_optimizer=5,
x0=None,
y0=None,
random_state=None):
"""Bayesian optimization using Gaussian Processes.
If every function evaluation is expensive, for instance
when the parameters are the hyperparameters of a neural network
and the function evaluation is the mean cross-validation score across
ten folds, optimizing the hyperparameters by standard optimization
routines would take for ever!
The idea is to approximate the function using a Gaussian process.
In other words the function values are assumed to follow a multivariate
gaussian. The covariance of the function values are given by a
GP kernel between the parameters. Then a smart choice to choose the
next parameter to evaluate can be made by the acquisition function
over the Gaussian prior which is much quicker to evaluate.
The total number of evaluations, `n_calls`, are performed like the
following. If `x0` is provided but not `y0`, then the elements of `x0`
are first evaluated, followed by `n_random_starts` evaluations.
Finally, `n_calls - len(x0) - n_random_starts` evaluations are
made guided by the surrogate model. If `x0` and `y0` are both
provided then `n_random_starts` evaluations are first made then
`n_calls - n_random_starts` subsequent evaluations are made
guided by the surrogate model.
Parameters
----------
* `func` [callable]:
Function to minimize. Should take a array of parameters and
return the function values.
* `dimensions` [list, shape=(n_dims,)]:
List of search space dimensions.
Each search dimension can be defined either as
- a `(upper_bound, lower_bound)` tuple (for `Real` or `Integer`
dimensions),
- a `(upper_bound, lower_bound, "prior")` tuple (for `Real`
dimensions),
- as a list of categories (for `Categorical` dimensions), or
- an instance of a `Dimension` object (`Real`, `Integer` or
`Categorical`).
* `base_estimator` [a Gaussian process estimator]:
The Gaussian process estimator to use for optimization.
* `alpha` [float, default=1e-10]:
Value added to the diagonal of the kernel matrix during fitting.
Larger values correspond to an increased noise level in the
observations and reduce potential numerical issues during fitting.
* `acq` [string, default=`"EI"`]:
Function to minimize over the gaussian prior. Can be either
- `"LCB"` for lower confidence bound,
- `"EI"` for expected improvement,
- `"PI"` for probability of improvement.
* `xi` [float, default=0.01]:
Controls how much improvement one wants over the previous best
values. Used when the acquisition is either `"EI"` or `"PI"`.
* `kappa` [float, default=1.96]:
Controls how much of the variance in the predicted values should be
taken into account. If set to be very high, then we are favouring
exploration over exploitation and vice versa.
Used when the acquisition is `"LCB"`.
* `search` [string, `"auto"`, `"sampling"` or `"lbfgs"`, default=`"auto"`]:
Searching for the next possible candidate to update the Gaussian prior
with.
If search is set to `"auto"`, then it is set to `"lbfgs"`` if
all the search dimensions are Real(continuous). It defaults to
`"sampling"` for all other cases.
If search is set to `"sampling"`, `n_points` are sampled randomly
and the Gaussian Process prior is updated with the point that gives
the best acquisition value over the Gaussian prior.
If search is set to `"lbfgs"`, then a point is sampled randomly, and
lbfgs is run for 10 iterations optimizing the acquisition function
over the Gaussian prior.
* `n_calls` [int, default=100]:
Number of calls to `func`.
* `n_points` [int, default=500]:
Number of points to sample to determine the next "best" point.
Useless if search is set to `"lbfgs"`.
* `n_random_starts` [int, default=10]:
Number of evaluations of `func` with random initialization points
before approximating the `func` with `base_estimator`.
* `n_restarts_optimizer` [int, default=10]:
The number of restarts of the optimizer when `search` is `"lbfgs"`.
* `x0` [list, list of lists or `None`]:
Initial input points.
- If it is a list of lists, use it as a list of input points.
- If it is a list, use it as a single initial input point.
- If it is `None`, no initial input points are used.
* `y0` [list, scalar or `None`]
Evaluation of initial input points.
- If it is a list, then it corresponds to evaluations of the function
at each element of `x0` : the i-th element of `y0` corresponds
to the function evaluated at the i-th element of `x0`.
- If it is a scalar, then it corresponds to the evaluation of the
function at `x0`.
- If it is None and `x0` is provided, then the function is evaluated
at each element of `x0`.
* `random_state` [int, RandomState instance, or None (default)]:
Set random state to something other than None for reproducible
results.
Returns
-------
* `res` [`OptimizeResult`, scipy object]:
The optimization result returned as a OptimizeResult object.
Important attributes are:
- `x` [list]: location of the minimum.
- `fun` [float]: function value at the minimum.
- `models`: surrogate models used for each iteration.
- `x_iters` [list of lists]: location of function evaluation for each
iteration.
- `func_vals` [array]: function value for each iteration.
- `space` [Space]: the optimization space.
- `specs` [dict]`: the call specifications.
- `rng` [RandomState instance]: State of the random state
at the end of minimization.
For more details related to the OptimizeResult object, refer
http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.OptimizeResult.html
"""
# Save call args
specs = {
"args": copy.copy(inspect.currentframe().f_locals),
"function": inspect.currentframe().f_code.co_name
}
# Check params
rng = check_random_state(random_state)
space = Space(dimensions)
# Default GP
if base_estimator is None:
base_estimator = GaussianProcessRegressor(
kernel=(ConstantKernel(1.0, (0.01, 1000.0)) * Matern(
length_scale=np.ones(space.transformed_n_dims),
length_scale_bounds=[(0.01, 100)] * space.transformed_n_dims,
nu=2.5)),
normalize_y=True,
alpha=alpha,
random_state=random_state)
# Initialize with provided points (x0 and y0) and/or random points
if x0 is None:
x0 = []
elif not isinstance(x0[0], list):
x0 = [x0]
if not isinstance(x0, list):
raise ValueError("`x0` should be a list, but got %s" % type(x0))
n_init_func_calls = len(x0) if y0 is not None else 0
n_total_init_calls = n_random_starts + n_init_func_calls
if n_total_init_calls <= 0:
# if x0 is not provided and n_random_starts is 0 then
# it will ask for n_random_starts to be > 0.
raise ValueError("Expected `n_random_starts` > 0, got %d" %
n_random_starts)
if n_calls < n_total_init_calls:
raise ValueError("Expected `n_calls` >= %d, got %d" %
(n_total_init_calls, n_calls))
if y0 is None and x0:
y0 = [func(x) for x in x0]
elif x0:
if isinstance(y0, Iterable):
y0 = list(y0)
elif isinstance(y0, numbers.Number):
y0 = [y0]
else:
raise ValueError("`y0` should be an iterable or a scalar, got %s" %
type(y0))
if len(x0) != len(y0):
raise ValueError("`x0` and `y0` should have the same length")
if not all(map(np.isscalar, y0)):
raise ValueError("`y0` elements should be scalars")
else:
y0 = []
Xi = x0 + space.rvs(n_samples=n_random_starts, random_state=rng)
yi = y0 + [func(x) for x in Xi[len(x0):]]
if np.ndim(yi) != 1:
raise ValueError("`func` should return a scalar")
if search == "auto":
if space.is_real:
search = "lbfgs"
else:
search = "sampling"
elif search not in ["lbfgs", "sampling"]:
raise ValueError(
"Expected search to be 'lbfgs', 'sampling' or 'auto', "
"got %s" % search)
# Bayesian optimization loop
models = []
n_model_iter = n_calls - n_total_init_calls
for i in range(n_model_iter):
gp = clone(base_estimator)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
gp.fit(space.transform(Xi), yi)
models.append(gp)
if search == "sampling":
X = space.transform(space.rvs(n_samples=n_points,
random_state=rng))
values = _gaussian_acquisition(X=X,
model=gp,
y_opt=np.min(yi),
method=acq,
xi=xi,
kappa=kappa)
next_x = X[np.argmin(values)]
elif search == "lbfgs":
best = np.inf
for j in range(n_restarts_optimizer):
x0 = space.transform(space.rvs(n_samples=1,
random_state=rng))[0]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
x, a, _ = fmin_l_bfgs_b(_acquisition,
x0,
args=(gp, np.min(yi), acq, xi,
kappa),
bounds=space.transformed_bounds,
approx_grad=True,
maxiter=20)
if a < best:
next_x, best = x, a
next_x = space.inverse_transform(next_x.reshape((1, -1)))[0]
next_y = func(next_x)
Xi.append(next_x)
yi.append(next_y)
# Pack results
res = OptimizeResult()
best = np.argmin(yi)
res.x = Xi[best]
res.fun = yi[best]
res.func_vals = np.array(yi)
res.x_iters = Xi
res.models = models
res.space = space
res.random_state = rng
res.specs = specs
return res
def trf_linear(A, b, x_lsq, lb, ub, tol, lsq_solver, lsmr_tol, max_iter,
verbose):
m, n = A.shape
x, _ = reflective_transformation(x_lsq, lb, ub)
x = make_strictly_feasible(x, lb, ub, rstep=0.1)
if lsq_solver == 'exact':
QT, R, perm = qr(A, mode='economic', pivoting=True)
QT = QT.T
if m < n:
R = np.vstack((R, np.zeros((n - m, n))))
QTr = np.zeros(n)
k = min(m, n)
elif lsq_solver == 'lsmr':
r_aug = np.zeros(m + n)
auto_lsmr_tol = False
if lsmr_tol is None:
lsmr_tol = 1e-2 * tol
elif lsmr_tol == 'auto':
auto_lsmr_tol = True
r = A.dot(x) - b
g = compute_grad(A, r)
cost = 0.5 * np.dot(r, r)
initial_cost = cost
termination_status = None
step_norm = None
cost_change = None
if max_iter is None:
max_iter = 100
if verbose == 2:
print_header_linear()
for iteration in range(max_iter):
v, dv = CL_scaling_vector(x, g, lb, ub)
g_scaled = g * v
g_norm = norm(g_scaled, ord=np.inf)
if g_norm < tol:
termination_status = 1
if verbose == 2:
print_iteration_linear(iteration, cost, cost_change, step_norm,
g_norm)
if termination_status is not None:
break
diag_h = g * dv
diag_root_h = diag_h**0.5
d = v**0.5
g_h = d * g
A_h = right_multiplied_operator(A, d)
if lsq_solver == 'exact':
QTr[:k] = QT.dot(r)
p_h = -regularized_lsq_with_qr(
m, n, R * d[perm], QTr, perm, diag_root_h, copy_R=False)
elif lsq_solver == 'lsmr':
lsmr_op = regularized_lsq_operator(A_h, diag_root_h)
r_aug[:m] = r
if auto_lsmr_tol:
eta = 1e-2 * min(0.5, g_norm)
lsmr_tol = max(EPS, min(0.1, eta * g_norm))
p_h = -lsmr(lsmr_op, r_aug, atol=lsmr_tol, btol=lsmr_tol)[0]
p = d * p_h
p_dot_g = np.dot(p, g)
if p_dot_g > 0:
termination_status = -1
theta = 1 - min(0.005, g_norm)
step = select_step(x, A_h, g_h, diag_h, p, p_h, d, lb, ub, theta)
cost_change = -evaluate_quadratic(A, g, step)
# Perhaps almost never executed, the idea is that `p` is descent
# direction thus we must find acceptable cost decrease using simple
# "backtracking", otherwise algorithm's logic would break.
if cost_change < 0:
x, step, cost_change = backtracking(A, g, x, p, theta, p_dot_g, lb,
ub)
else:
x = make_strictly_feasible(x + step, lb, ub, rstep=0)
step_norm = norm(step)
r = A.dot(x) - b
g = compute_grad(A, r)
if cost_change < tol * cost:
termination_status = 2
cost = 0.5 * np.dot(r, r)
if termination_status is None:
termination_status = 0
active_mask = find_active_constraints(x, lb, ub, rtol=tol)
return OptimizeResult(x=x,
fun=r,
cost=cost,
optimality=g_norm,
active_mask=active_mask,
nit=iteration + 1,
status=termination_status,
initial_cost=initial_cost)
def trf_no_bounds(fun, jac, x0, f0, J0, ftol, xtol, gtol, max_nfev, x_scale,
loss_function, tr_solver, tr_options, verbose):
x = x0.copy()
f = f0
f_true = f.copy()
nfev = 1
J = J0
njev = 1
m, n = J.shape
if loss_function is not None:
rho = loss_function(f)
cost = 0.5 * np.sum(rho[0])
J, f = scale_for_robust_loss_function(J, f, rho)
else:
cost = 0.5 * np.dot(f, f)
g = compute_grad(J, f)
jac_scale = isinstance(x_scale, string_types) and x_scale == 'jac'
if jac_scale:
scale, scale_inv = compute_jac_scale(J)
else:
scale, scale_inv = x_scale, 1 / x_scale
Delta = norm(x0 * scale_inv)
if Delta == 0:
Delta = 1.0
if tr_solver == 'lsmr':
reg_term = 0
damp = tr_options.pop('damp', 0.0)
regularize = tr_options.pop('regularize', True)
if max_nfev is None:
max_nfev = x0.size * 100
alpha = 0.0 # "Levenberg-Marquardt" parameter
termination_status = None
iteration = 0
step_norm = None
actual_reduction = None
if verbose == 2:
print_header_nonlinear()
while True:
g_norm = norm(g, ord=np.inf)
if g_norm < gtol:
termination_status = 1
if verbose == 2:
print_iteration_nonlinear(iteration, nfev, cost, actual_reduction,
step_norm, g_norm)
if termination_status is not None or nfev == max_nfev:
break
d = scale
g_h = d * g
if tr_solver == 'exact':
J_h = J * d
U, s, V = svd(J_h, full_matrices=False)
V = V.T
uf = U.T.dot(f)
elif tr_solver == 'lsmr':
J_h = right_multiplied_operator(J, d)
if regularize:
a, b = build_quadratic_1d(J_h, g_h, -g_h)
to_tr = Delta / norm(g_h)
ag_value = minimize_quadratic_1d(a, b, 0, to_tr)[1]
reg_term = -ag_value / Delta**2
damp_full = (damp**2 + reg_term)**0.5
gn_h = lsmr(J_h, f, damp=damp_full, **tr_options)[0]
S = np.vstack((g_h, gn_h)).T
S, _ = qr(S, mode='economic')
JS = J_h.dot(S)
B_S = np.dot(JS.T, JS)
g_S = S.T.dot(g_h)
actual_reduction = -1
while actual_reduction <= 0 and nfev < max_nfev:
if tr_solver == 'exact':
step_h, alpha, n_iter = solve_lsq_trust_region(
n, m, uf, s, V, Delta, initial_alpha=alpha)
elif tr_solver == 'lsmr':
p_S, _ = solve_trust_region_2d(B_S, g_S, Delta)
step_h = S.dot(p_S)
predicted_reduction = -evaluate_quadratic(J_h, g_h, step_h)
step = d * step_h
x_new = x + step
f_new = fun(x_new)
nfev += 1
step_h_norm = norm(step_h)
if not np.all(np.isfinite(f_new)):
Delta = 0.25 * step_h_norm
continue
# Usual trust-region step quality estimation.
if loss_function is not None:
cost_new = loss_function(f_new, cost_only=True)
else:
cost_new = 0.5 * np.dot(f_new, f_new)
actual_reduction = cost - cost_new
Delta_new, ratio = update_tr_radius(Delta, actual_reduction,
predicted_reduction,
step_h_norm,
step_h_norm > 0.95 * Delta)
alpha *= Delta / Delta_new
Delta = Delta_new
step_norm = norm(step)
termination_status = check_termination(actual_reduction, cost,
step_norm, norm(x), ratio,
ftol, xtol)
if termination_status is not None:
break
if actual_reduction > 0:
x = x_new
f = f_new
f_true = f.copy()
cost = cost_new
J = jac(x, f)
njev += 1
if loss_function is not None:
rho = loss_function(f)
J, f = scale_for_robust_loss_function(J, f, rho)
g = compute_grad(J, f)
if jac_scale:
scale, scale_inv = compute_jac_scale(J, scale_inv)
else:
step_norm = 0
actual_reduction = 0
iteration += 1
if termination_status is None:
termination_status = 0
active_mask = np.zeros_like(x)
return OptimizeResult(x=x,
cost=cost,
fun=f_true,
jac=J,
grad=g,
optimality=g_norm,
active_mask=active_mask,
nfev=nfev,
njev=njev,
status=termination_status)
