def _root_df_sane(func,
x0,
args=(),
ftol=1e-8,
fatol=1e-300,
maxfev=1000,
fnorm=None,
callback=None,
disp=False,
M=10,
eta_strategy=None,
sigma_eps=1e-10,
sigma_0=1.0,
line_search='cruz',
**unknown_options):
r"""
Solve nonlinear equation with the DF-SANE method
Options
-------
ftol : float, optional
Relative norm tolerance.
fatol : float, optional
Absolute norm tolerance.
Algorithm terminates when ``||func(x)|| < fatol + ftol ||func(x_0)||``.
fnorm : callable, optional
Norm to use in the convergence check. If None, 2-norm is used.
maxfev : int, optional
Maximum number of function evaluations.
disp : bool, optional
Whether to print convergence process to stdout.
eta_strategy : callable, optional
Choice of the ``eta_k`` parameter, which gives slack for growth
of ``||F||**2``. Called as ``eta_k = eta_strategy(k, x, F)`` with
`k` the iteration number, `x` the current iterate and `F` the current
residual. Should satisfy ``eta_k > 0`` and ``sum(eta, k=0..inf) < inf``.
Default: ``||F||**2 / (1 + k)**2``.
sigma_eps : float, optional
The spectral coefficient is constrained to ``sigma_eps < sigma < 1/sigma_eps``.
Default: 1e-10
sigma_0 : float, optional
Initial spectral coefficient.
Default: 1.0
M : int, optional
Number of iterates to include in the nonmonotonic line search.
Default: 10
line_search : {'cruz', 'cheng'}
Type of line search to employ. 'cruz' is the original one defined in
[Martinez & Raydan. Math. Comp. 75, 1429 (2006)], 'cheng' is
a modified search defined in [Cheng & Li. IMA J. Numer. Anal. 29, 814 (2009)].
Default: 'cruz'
References
----------
.. [1] "Spectral residual method without gradient information for solving
large-scale nonlinear systems of equations." W. La Cruz,
J.M. Martinez, M. Raydan. Math. Comp. **75**, 1429 (2006).
.. [2] W. La Cruz, Opt. Meth. Software, 29, 24 (2014).
.. [3] W. Cheng, D.-H. Li. IMA J. Numer. Anal. **29**, 814 (2009).
"""
_check_unknown_options(unknown_options)
if line_search not in ('cheng', 'cruz'):
raise ValueError("Invalid value %r for 'line_search'" %
(line_search, ))
nexp = 2
if eta_strategy is None:
# Different choice from [1], as their eta is not invariant
# vs. scaling of F.
def eta_strategy(k, x, F):
# Obtain squared 2-norm of the initial residual from the outer scope
return f_0 / (1 + k)**2
if fnorm is None:
def fnorm(F):
# Obtain squared 2-norm of the current residual from the outer scope
return f_k**(1.0 / nexp)
def fmerit(F):
return np.linalg.norm(F)**nexp
nfev = [0]
f, x_k, x_shape, f_k, F_k, is_complex = _wrap_func(func, x0, fmerit, nfev,
maxfev, args)
k = 0
f_0 = f_k
sigma_k = sigma_0
F_0_norm = fnorm(F_k)
# For the 'cruz' line search
prev_fs = collections.deque([f_k], M)
# For the 'cheng' line search
Q = 1.0
C = f_0
converged = False
message = "too many function evaluations required"
while True:
F_k_norm = fnorm(F_k)
if disp:
print("iter %d: ||F|| = %g, sigma = %g" % (k, F_k_norm, sigma_k))
if callback is not None:
callback(x_k, F_k)
if F_k_norm < ftol * F_0_norm + fatol:
# Converged!
message = "successful convergence"
converged = True
break
# Control spectral parameter, from [2]
if abs(sigma_k) > 1 / sigma_eps:
sigma_k = 1 / sigma_eps * np.sign(sigma_k)
elif abs(sigma_k) < sigma_eps:
sigma_k = sigma_eps
# Line search direction
d = -sigma_k * F_k
# Nonmonotone line search
eta = eta_strategy(k, x_k, F_k)
try:
if line_search == 'cruz':
alpha, xp, fp, Fp = _nonmonotone_line_search_cruz(f,
x_k,
d,
prev_fs,
eta=eta)
elif line_search == 'cheng':
alpha, xp, fp, Fp, C, Q = _nonmonotone_line_search_cheng(
f, x_k, d, f_k, C, Q, eta=eta)
except _NoConvergence:
break
# Update spectral parameter
s_k = xp - x_k
y_k = Fp - F_k
sigma_k = np.vdot(s_k, s_k) / np.vdot(s_k, y_k)
# Take step
x_k = xp
F_k = Fp
f_k = fp
# Store function value
if line_search == 'cruz':
prev_fs.append(fp)
k += 1
x = _wrap_result(x_k, is_complex, shape=x_shape)
F = _wrap_result(F_k, is_complex)
result = OptimizeResult(x=x,
success=converged,
message=message,
fun=F,
nfev=nfev[0],
nit=k)
return result
def _minimize_slsqp(func,
x0,
args=(),
jac=None,
bounds=None,
constraints=(),
maxiter=100,
ftol=1.0E-6,
iprint=1,
disp=False,
eps=_epsilon,
callback=None,
**unknown_options):
"""
Minimize a scalar function of one or more variables using Sequential
Least SQuares Programming (SLSQP).
Options
-------
ftol : float
Precision goal for the value of f in the stopping criterion.
eps : float
Step size used for numerical approximation of the jacobian.
disp : bool
Set to True to print convergence messages. If False,
`verbosity` is ignored and set to 0.
maxiter : int
Maximum number of iterations.
"""
_check_unknown_options(unknown_options)
fprime = jac
iter = maxiter
acc = ftol
epsilon = eps
if not disp:
iprint = 0
# Constraints are triaged per type into a dictionnary of tuples
if isinstance(constraints, dict):
constraints = (constraints, )
cons = {'eq': (), 'ineq': ()}
for ic, con in enumerate(constraints):
# check type
try:
ctype = con['type'].lower()
except KeyError:
raise KeyError('Constraint %d has no type defined.' % ic)
except TypeError:
raise TypeError('Constraints must be defined using a '
'dictionary.')
except AttributeError:
raise TypeError("Constraint's type must be a string.")
else:
if ctype not in ['eq', 'ineq']:
raise ValueError("Unknown constraint type '%s'." % con['type'])
# check function
if 'fun' not in con:
raise ValueError('Constraint %d has no function defined.' % ic)
# check jacobian
cjac = con.get('jac')
if cjac is None:
# approximate jacobian function. The factory function is needed
# to keep a reference to `fun`, see gh-4240.
def cjac_factory(fun):
def cjac(x, *args):
return approx_jacobian(x, fun, epsilon, *args)
return cjac
cjac = cjac_factory(con['fun'])
# update constraints' dictionary
cons[ctype] += ({
'fun': con['fun'],
'jac': cjac,
'args': con.get('args', ())
}, )
exit_modes = {
-1: "Gradient evaluation required (g & a)",
0: "Optimization terminated successfully.",
1: "Function evaluation required (f & c)",
2: "More equality constraints than independent variables",
3: "More than 3*n iterations in LSQ subproblem",
4: "Inequality constraints incompatible",
5: "Singular matrix E in LSQ subproblem",
6: "Singular matrix C in LSQ subproblem",
7: "Rank-deficient equality constraint subproblem HFTI",
8: "Positive directional derivative for linesearch",
9: "Iteration limit exceeded"
}
# Wrap func
feval, func = wrap_function(func, args)
# Wrap fprime, if provided, or approx_jacobian if not
if fprime:
geval, fprime = wrap_function(fprime, args)
else:
geval, fprime = wrap_function(approx_jacobian, (func, epsilon))
# Transform x0 into an array.
x = asfarray(x0).flatten()
# Set the parameters that SLSQP will need
# meq, mieq: number of equality and inequality constraints
meq = sum(
map(len, [atleast_1d(c['fun'](x, *c['args'])) for c in cons['eq']]))
mieq = sum(
map(len, [atleast_1d(c['fun'](x, *c['args'])) for c in cons['ineq']]))
# m = The total number of constraints
m = meq + mieq
# la = The number of constraints, or 1 if there are no constraints
la = array([1, m]).max()
# n = The number of independent variables
n = len(x)
# Define the workspaces for SLSQP
n1 = n + 1
mineq = m - meq + n1 + n1
len_w = (3*n1+m)*(n1+1)+(n1-meq+1)*(mineq+2) + 2*mineq+(n1+mineq)*(n1-meq) \
+ 2*meq + n1 + ((n+1)*n)//2 + 2*m + 3*n + 3*n1 + 1
len_jw = mineq
w = zeros(len_w)
jw = zeros(len_jw)
# Decompose bounds into xl and xu
if bounds is None or len(bounds) == 0:
xl = np.empty(n, dtype=float)
xu = np.empty(n, dtype=float)
xl.fill(np.nan)
xu.fill(np.nan)
else:
bnds = array(bounds, float)
if bnds.shape[0] != n:
raise IndexError('SLSQP Error: the length of bounds is not '
'compatible with that of x0.')
with np.errstate(invalid='ignore'):
bnderr = bnds[:, 0] > bnds[:, 1]
if bnderr.any():
raise ValueError('SLSQP Error: lb > ub in bounds %s.' %
', '.join(str(b) for b in bnderr))
xl, xu = bnds[:, 0], bnds[:, 1]
# Mark infinite bounds with nans; the Fortran code understands this
infbnd = ~isfinite(bnds)
xl[infbnd[:, 0]] = np.nan
xu[infbnd[:, 1]] = np.nan
# Clip initial guess to bounds (SLSQP may fail with bounds-infeasible initial point)
have_bound = np.isfinite(xl)
x[have_bound] = np.clip(x[have_bound], xl[have_bound], np.inf)
have_bound = np.isfinite(xu)
x[have_bound] = np.clip(x[have_bound], -np.inf, xu[have_bound])
# Initialize the iteration counter and the mode value
mode = array(0, int)
acc = array(acc, float)
majiter = array(iter, int)
majiter_prev = 0
# Print the header if iprint >= 2
if iprint >= 2:
print("%5s %5s %16s %16s" % ("NIT", "FC", "OBJFUN", "GNORM"))
while 1:
if mode == 0 or mode == 1: # objective and constraint evaluation requird
# Compute objective function
fx = func(x)
try:
fx = float(np.asarray(fx))
except (TypeError, ValueError):
raise ValueError("Objective function must return a scalar")
# Compute the constraints
if cons['eq']:
c_eq = concatenate([
atleast_1d(con['fun'](x, *con['args']))
for con in cons['eq']
])
else:
c_eq = zeros(0)
if cons['ineq']:
c_ieq = concatenate([
atleast_1d(con['fun'](x, *con['args']))
for con in cons['ineq']
])
else:
c_ieq = zeros(0)
# Now combine c_eq and c_ieq into a single matrix
c = concatenate((c_eq, c_ieq))
if mode == 0 or mode == -1: # gradient evaluation required
# Compute the derivatives of the objective function
# For some reason SLSQP wants g dimensioned to n+1
g = append(fprime(x), 0.0)
# Compute the normals of the constraints
if cons['eq']:
a_eq = vstack(
[con['jac'](x, *con['args']) for con in cons['eq']])
else: # no equality constraint
a_eq = zeros((meq, n))
if cons['ineq']:
a_ieq = vstack(
[con['jac'](x, *con['args']) for con in cons['ineq']])
else: # no inequality constraint
a_ieq = zeros((mieq, n))
# Now combine a_eq and a_ieq into a single a matrix
if m == 0: # no constraints
a = zeros((la, n))
else:
a = vstack((a_eq, a_ieq))
a = concatenate((a, zeros([la, 1])), 1)
# Call SLSQP
slsqp(m, meq, x, xl, xu, fx, c, g, a, acc, majiter, mode, w, jw)
# call callback if major iteration has incremented
if callback is not None and majiter > majiter_prev:
callback(x)
# Print the status of the current iterate if iprint > 2 and the
# major iteration has incremented
if iprint >= 2 and majiter > majiter_prev:
print("%5i %5i % 16.6E % 16.6E" %
(majiter, feval[0], fx, linalg.norm(g)))
# If exit mode is not -1 or 1, slsqp has completed
if abs(mode) != 1:
break
majiter_prev = int(majiter)
# Optimization loop complete. Print status if requested
if iprint >= 1:
print(exit_modes[int(mode)] + " (Exit mode " + str(mode) + ')')
print(" Current function value:", fx)
print(" Iterations:", majiter)
print(" Function evaluations:", feval[0])
print(" Gradient evaluations:", geval[0])
return OptimizeResult(x=x,
fun=fx,
jac=g[:-1],
nit=int(majiter),
nfev=feval[0],
njev=geval[0],
status=int(mode),
message=exit_modes[int(mode)],
success=(mode == 0))
def _root_df_sane(
func,
x0,
args=(),
ftol=1e-8,
fatol=1e-300,
maxfev=1000,
fnorm=None,
callback=None,
disp=False,
M=10,
eta_strategy=None,
sigma_eps=1e-10,
sigma_0=1.0,
line_search="cruz",
**unknown_options
):
r"""
Solve nonlinear equation with the DF-SANE method
Options
-------
ftol : float, optional
Relative norm tolerance.
fatol : float, optional
Absolute norm tolerance.
Algorithm terminates when ``||func(x)|| < fatol + ftol ||func(x_0)||``.
fnorm : callable, optional
Norm to use in the convergence check. If None, 2-norm is used.
maxfev : int, optional
Maximum number of function evaluations.
disp : bool, optional
Whether to print convergence process to stdout.
eta_strategy : callable, optional
Choice of the ``eta_k`` parameter, which gives slack for growth
of ``||F||**2``. Called as ``eta_k = eta_strategy(k, x, F)`` with
`k` the iteration number, `x` the current iterate and `F` the current
residual. Should satisfy ``eta_k > 0`` and ``sum(eta, k=0..inf) < inf``.
Default: ``||F||**2 / (1 + k)**2``.
sigma_eps : float, optional
The spectral coefficient is constrained to ``sigma_eps < sigma < 1/sigma_eps``.
Default: 1e-10
sigma_0 : float, optional
Initial spectral coefficient.
Default: 1.0
M : int, optional
Number of iterates to include in the nonmonotonic line search.
Default: 10
line_search : {'cruz', 'cheng'}
Type of line search to employ. 'cruz' is the original one defined in
[Martinez & Raydan. Math. Comp. 75, 1429 (2006)], 'cheng' is
a modified search defined in [Cheng & Li. IMA J. Numer. Anal. 29, 814 (2009)].
Default: 'cruz'
References
----------
.. [1] "Spectral residual method without gradient information for solving
large-scale nonlinear systems of equations." W. La Cruz,
J.M. Martinez, M. Raydan. Math. Comp. **75**, 1429 (2006).
.. [2] W. La Cruz, Opt. Meth. Software, 29, 24 (2014).
.. [3] W. Cheng, D.-H. Li. IMA J. Numer. Anal. **29**, 814 (2009).
"""
_check_unknown_options(unknown_options)
if line_search not in ("cheng", "cruz"):
raise ValueError("Invalid value %r for 'line_search'" % (line_search,))
nexp = 2
if eta_strategy is None:
# Different choice from [1], as their eta is not invariant
# vs. scaling of F.
def eta_strategy(k, x, F):
# Obtain squared 2-norm of the initial residual from the outer scope
return f_0 / (1 + k) ** 2
if fnorm is None:
def fnorm(F):
# Obtain squared 2-norm of the current residual from the outer scope
return f_k ** (1.0 / nexp)
def fmerit(F):
return np.linalg.norm(F) ** nexp
nfev = [0]
f, x_k, x_shape, f_k, F_k, is_complex = _wrap_func(func, x0, fmerit, nfev, maxfev, args)
k = 0
f_0 = f_k
sigma_k = sigma_0
F_0_norm = fnorm(F_k)
# For the 'cruz' line search
prev_fs = collections.deque([f_k], M)
# For the 'cheng' line search
Q = 1.0
C = f_0
converged = False
message = "too many function evaluations required"
while True:
F_k_norm = fnorm(F_k)
if disp:
print("iter %d: ||F|| = %g, sigma = %g" % (k, F_k_norm, sigma_k))
if callback is not None:
callback(x_k, F_k)
if F_k_norm < ftol * F_0_norm + fatol:
# Converged!
message = "successful convergence"
converged = True
break
# Control spectral parameter, from [2]
if abs(sigma_k) > 1 / sigma_eps:
sigma_k = 1 / sigma_eps * np.sign(sigma_k)
elif abs(sigma_k) < sigma_eps:
sigma_k = sigma_eps
# Line search direction
d = -sigma_k * F_k
# Nonmonotone line search
eta = eta_strategy(k, x_k, F_k)
try:
if line_search == "cruz":
alpha, xp, fp, Fp = _nonmonotone_line_search_cruz(f, x_k, d, prev_fs, eta=eta)
elif line_search == "cheng":
alpha, xp, fp, Fp, C, Q = _nonmonotone_line_search_cheng(f, x_k, d, f_k, C, Q, eta=eta)
except _NoConvergence:
break
# Update spectral parameter
s_k = xp - x_k
y_k = Fp - F_k
sigma_k = np.vdot(s_k, s_k) / np.vdot(s_k, y_k)
# Take step
x_k = xp
F_k = Fp
f_k = fp
# Store function value
if line_search == "cruz":
prev_fs.append(fp)
k += 1
x = _wrap_result(x_k, is_complex, shape=x_shape)
F = _wrap_result(F_k, is_complex)
result = OptimizeResult(x=x, success=converged, message=message, fun=F, nfev=nfev[0], nit=k)
return result
def _minimize_neldermead(func, x0, args=(), callback=None,
xtol=1e-4, ftol=1e-4, maxiter=None, maxfev=None,
disp=False, return_all=False, return_simplex=False,
**unknown_options):
"""
Minimization of scalar function of one or more variables using the
Nelder-Mead algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
xtol : float
Relative error in solution `xopt` acceptable for convergence.
ftol : float
Relative error in ``fun(xopt)`` acceptable for convergence.
maxiter : int
Maximum number of iterations to perform.
maxfev : int
Maximum number of function evaluations to make.
return_simplex : bool
Set to True to return all nodes of final simplex and their function
values.
"""
_check_unknown_options(unknown_options)
maxfun = maxfev
retall = return_all
fcalls, func = wrap_function(func, args)
x0 = asfarray(x0).flatten()
N = len(x0)
if maxiter is None:
maxiter = N * 200
if maxfun is None:
maxfun = N * 200
rho = 1
chi = 2
psi = 0.5
sigma = 0.5
one2np1 = list(range(1, N + 1))
sim = numpy.zeros((N + 1, N), dtype=x0.dtype)
fsim = numpy.zeros((N + 1,), float)
sim[0] = x0
if retall:
allvecs = [sim[0]]
fsim[0] = func(x0)
nonzdelt = 0.05
zdelt = 0.00025
for k in range(0, N):
y = numpy.array(x0, copy=True)
if y[k] != 0:
y[k] = (1 + nonzdelt) * y[k]
else:
y[k] = zdelt
sim[k + 1] = y
f = func(y)
fsim[k + 1] = f
ind = numpy.argsort(fsim)
fsim = numpy.take(fsim, ind, 0)
# sort so sim[0,:] has the lowest function value
sim = numpy.take(sim, ind, 0)
iterations = 1
while (fcalls[0] < maxfun and iterations < maxiter):
if (numpy.max(numpy.ravel(numpy.abs(sim[1:] - sim[0]))) <= xtol and
numpy.max(numpy.abs(fsim[0] - fsim[1:])) <= ftol):
break
xbar = numpy.add.reduce(sim[:-1], 0) / N
xr = (1 + rho) * xbar - rho * sim[-1]
fxr = func(xr)
doshrink = 0
if fxr < fsim[0]:
xe = (1 + rho * chi) * xbar - rho * chi * sim[-1]
fxe = func(xe)
if fxe < fxr:
sim[-1] = xe
fsim[-1] = fxe
else:
sim[-1] = xr
fsim[-1] = fxr
else: # fsim[0] <= fxr
if fxr < fsim[-2]:
sim[-1] = xr
fsim[-1] = fxr
else: # fxr >= fsim[-2]
# Perform contraction
if fxr < fsim[-1]:
xc = (1 + psi * rho) * xbar - psi * rho * sim[-1]
fxc = func(xc)
if fxc <= fxr:
sim[-1] = xc
fsim[-1] = fxc
else:
doshrink = 1
else:
# Perform an inside contraction
xcc = (1 - psi) * xbar + psi * sim[-1]
fxcc = func(xcc)
if fxcc < fsim[-1]:
sim[-1] = xcc
fsim[-1] = fxcc
else:
doshrink = 1
if doshrink:
for j in one2np1:
sim[j] = sim[0] + sigma * (sim[j] - sim[0])
fsim[j] = func(sim[j])
ind = numpy.argsort(fsim)
sim = numpy.take(sim, ind, 0)
fsim = numpy.take(fsim, ind, 0)
if callback is not None:
callback(sim[0])
iterations += 1
if retall:
allvecs.append(sim[0])
x = sim[0]
fval = numpy.min(fsim)
warnflag = 0
if fcalls[0] >= maxfun:
warnflag = 1
msg = _status_message['maxfev']
if disp:
print('Warning: ' + msg)
elif iterations >= maxiter:
warnflag = 2
msg = _status_message['maxiter']
if disp:
print('Warning: ' + msg)
else:
msg = _status_message['success']
if disp:
print(msg)
print(" Current function value: %f" % fval)
print(" Iterations: %d" % iterations)
print(" Function evaluations: %d" % fcalls[0])
result = OptimizeResult(fun=fval, nit=iterations, nfev=fcalls[0],
status=warnflag, success=(warnflag == 0),
message=msg, x=x)
if retall:
result['allvecs'] = allvecs
if return_simplex:
result['sim'] = sim
result['fsim'] = fsim
return result
def _root_hybr(func,
x0,
args=(),
jac=None,
col_deriv=0,
xtol=1.49012e-08,
maxfev=0,
band=None,
eps=None,
factor=100,
diag=None,
**unknown_options):
"""
Find the roots of a multivariate function using MINPACK's hybrd and
hybrj routines (modified Powell method).
Options
-------
col_deriv : bool
Specify whether the Jacobian function computes derivatives down
the columns (faster, because there is no transpose operation).
xtol : float
The calculation will terminate if the relative error between two
consecutive iterates is at most `xtol`.
maxfev : int
The maximum number of calls to the function. If zero, then
``100*(N+1)`` is the maximum where N is the number of elements
in `x0`.
band : tuple
If set to a two-sequence containing the number of sub- and
super-diagonals within the band of the Jacobi matrix, the
Jacobi matrix is considered banded (only for ``fprime=None``).
eps : float
A suitable step length for the forward-difference
approximation of the Jacobian (for ``fprime=None``). If
`eps` is less than the machine precision, it is assumed
that the relative errors in the functions are of the order of
the machine precision.
factor : float
A parameter determining the initial step bound
(``factor * || diag * x||``). Should be in the interval
``(0.1, 100)``.
diag : sequence
N positive entries that serve as a scale factors for the
variables.
"""
_check_unknown_options(unknown_options)
epsfcn = eps
x0 = asarray(x0).flatten()
n = len(x0)
if not isinstance(args, tuple):
args = (args, )
shape, dtype = _check_func('fsolve', 'func', func, x0, args, n, (n, ))
if epsfcn is None:
epsfcn = finfo(dtype).eps
Dfun = jac
if Dfun is None:
if band is None:
ml, mu = -10, -10
else:
ml, mu = band[:2]
if maxfev == 0:
maxfev = 200 * (n + 1)
retval = _minpack._hybrd(func, x0, args, 1, xtol, maxfev, ml, mu,
epsfcn, factor, diag)
else:
_check_func('fsolve', 'fprime', Dfun, x0, args, n, (n, n))
if (maxfev == 0):
maxfev = 100 * (n + 1)
retval = _minpack._hybrj(func, Dfun, x0, args, 1, col_deriv, xtol,
maxfev, factor, diag)
x, status = retval[0], retval[-1]
errors = {
0:
"Improper input parameters were entered.",
1:
"The solution converged.",
2:
"The number of calls to function has "
"reached maxfev = %d." % maxfev,
3:
"xtol=%f is too small, no further improvement "
"in the approximate\n solution "
"is possible." % xtol,
4:
"The iteration is not making good progress, as measured "
"by the \n improvement from the last five "
"Jacobian evaluations.",
5:
"The iteration is not making good progress, "
"as measured by the \n improvement from the last "
"ten iterations.",
'unknown':
"An error occurred."
}
info = retval[1]
info['fun'] = info.pop('fvec')
sol = OptimizeResult(x=x, success=(status == 1), status=status)
sol.update(info)
try:
sol['message'] = errors[status]
except KeyError:
sol['message'] = errors['unknown']
return sol
def _minimize(fun, x0, args=(), jac=None, callback=None,
gtol=1e-5, fxtol=1e-09, xtol=1e-09, norm=Inf,
eps=_epsilon, maxiter=None, disp=False,
return_all=False, **unknown_options):
_check_unknown_options(unknown_options)
f = fun
fprime = jac
epsilon = eps
retall = return_all
x0 = asarray(x0).flatten()
if x0.ndim == 0:
x0.shape = (1,)
if maxiter is None:
maxiter = len(x0) * 200
func_calls, f = wrap_function(f, args)
grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
gfk = myfprime(x0)
k = 0
N = len(x0)
I = numpy.eye(N, dtype=int)
Hk = I
old_fval = f(x0)
old_old_fval = None
xk = x0
if retall:
allvecs = [x0]
sk = [2 * gtol]
warnflag = 0
gnorm = vecnorm(gfk, ord=norm)
xnorm = np.Inf
fx = np.Inf
print_lst = []
while (gnorm > gtol) and (xnorm > xtol) and (fx > fxtol) and (k < maxiter):
pk = -numpy.dot(Hk, gfk)
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
_line_search_wolfe12(f, myfprime, xk, pk, gfk,
old_fval, old_old_fval)
except _LineSearchError:
# search failed to find a better solution.
print_lst.append("Przeszukiwanie liniowe zawiodlo lub nie moze osiagnac lepszego rozwiazania")
warnflag = 2
break
xkp1 = xk + alpha_k * pk
fx = np.absolute(old_old_fval - old_fval)
xnorm = vecnorm(xkp1 - xk)
if retall:
allvecs.append(xkp1)
sk = xkp1 - xk
xk = xkp1
if gfkp1 is None:
gfkp1 = myfprime(xkp1)
yk = gfkp1 - gfk
gfk = gfkp1
if callback is not None:
callback(xk)
k += 1
if disp:
print_ = ('Iter: ' + str(k) + '\n')
print_ += ('x: ' + str(xk) + '\n')
print_ += ('f(x): ' + str(f(xk)) + '\n') #zmiana na fx
print_ +=('gtol: ' + str(gnorm) + '\n')
print_ +=('xtol: ' + str(xnorm) + '\n')
print_ +=('fxtol: ' + str(fx) + '\n')
print_lst.append(print_)
gnorm = vecnorm(gfk, ord=norm)
if (gnorm <= gtol):
break
if not numpy.isfinite(old_fval):
# We correctly found +-Inf as optimal value, or something went
# wrong.
print_lst.append("Zlaneziono +-Inf za optymalna wartosc... lub cos poszlo zle.")
warnflag = 2
break
try: # this was handled in numeric, let it remaines for more safety
rhok = 1.0 / (numpy.dot(yk, sk))
except ZeroDivisionError:
rhok = 1000.0
if disp:
print_lst.append("Dzielenie przez zero!!")
if isinf(rhok): # this is patch for numpy
rhok = 1000.0
if disp:
print_lst.appedn("Dzielenie przez zero!!")
A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok
A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok
Hk = numpy.dot(A1, numpy.dot(Hk, A2)) + (rhok * sk[:, numpy.newaxis] *
sk[numpy.newaxis, :])
fval = old_fval
if np.isnan(fval):
# This can happen if the first call to f returned NaN;
# the loop is then never entered.
print_lst.append("Osiagnieto Nan w pierwszym wywolaniem algorytmu.")
warnflag = 2
if warnflag == 2:
msg = _status_message['pr_loss']
if disp:
print_ = ("Ostrzezenie: " + msg)
print_ += (" Wartosc funkcji celu: %f" % fval)
print_ += (" Iteracje: %d" % k)
print_ += (" Wywolania funkcji: %d" % func_calls[0])
print_ += (" Wywolania gradientu: %d" % grad_calls[0])
elif k >= maxiter:
warnflag = 1
msg = _status_message['maxiter']
if disp:
print_ = ("Ostrzerzenie: " + msg)
print_ += (" Wartosc funkcji celu: %f" % fval)
print_ += (" Iteracje: %d" % k)
print_ += (" Wywolania funkcji: %d" % func_calls[0])
print_ += (" Wywolania gradientu: %d" % grad_calls[0])
print_lst.append(print_)
else:
msg = _status_message['success']
if disp:
print_ = (msg + '\n')
print_ += (" Wartosc funkcji celu: %f" % fval)
print_ += (" Iteracje: %d" % k)
print_ += (" Wywolania funkcji: %d" % func_calls[0])
print_ += (" Wywolania gradientu: %d" % grad_calls[0])
print_lst.append(print_)
[print(line) for line in print_lst]
result = OptimizeResult(fun=fval,lst=print_lst, jac=gfk, hess_inv=Hk, nfev=func_calls[0],
njev=grad_calls[0], status=warnflag,
success=(warnflag == 0), message=msg, x=xk,
nit=k)
if retall:
result['allvecs'] = allvecs
return result
def _minimize_lbfgsb_timeup(
fun, x0, args=(), jac=None, bounds=None,
disp=None, maxcor=10, ftol=2.2204460492503131e-09,
gtol=1e-5, eps=1e-8, maxfun=15000, maxiter=15000,
iprint=-1, callback=None, maxls=20,
t0=None, timeup=float("inf"),
**unknown_options): # JFF: added time-up check
"""
Minimize a scalar function of one or more variables using the L-BFGS-B
algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxcor : int
The maximum number of variable metric corrections used to
define the limited memory matrix. (The limited memory BFGS
method does not store the full hessian but uses this many terms
in an approximation to it.)
factr : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr * eps``, where ``eps``
is the machine precision, which is automatically generated by
the code. Typical values for `factr` are: 1e12 for low
accuracy; 1e7 for moderate accuracy; 10.0 for extremely high
accuracy.
ftol : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol``.
gtol : float
The iteration will stop when ``max{|proj g_i | i = 1, ..., n}
<= gtol`` where ``pg_i`` is the i-th component of the
projected gradient.
eps : float
Step size used for numerical approximation of the jacobian.
disp : int
Set to True to print convergence messages.
maxfun : int
Maximum number of function evaluations.
maxiter : int
Maximum number of iterations.
maxls : int, optional
Maximum number of line search steps (per iteration). Default is 20.
"""
_check_unknown_options(unknown_options)
m = maxcor
epsilon = eps
pgtol = gtol
factr = ftol / np.finfo(float).eps
x0 = asarray(x0).ravel()
n, = x0.shape
if bounds is None:
bounds = [(None, None)] * n
if len(bounds) != n:
raise ValueError('length of x0 != length of bounds')
# unbounded variables must use None, not +-inf, for optimizer to work properly
bounds = [(None if l == -np.inf else l, None if u == np.inf else u) for l, u in bounds]
if disp is not None:
if disp == 0:
iprint = -1
else:
iprint = disp
n_function_evals, fun = wrap_function(fun, ())
if jac is None:
def func_and_grad(x):
f = fun(x, *args)
g = _approx_fprime_helper(x, fun, epsilon, args=args, f0=f)
return f, g
else:
def func_and_grad(x):
f = fun(x, *args)
g = jac(x, *args)
return f, g
nbd = zeros(n, int32)
low_bnd = zeros(n, float64)
upper_bnd = zeros(n, float64)
bounds_map = {(None, None): 0,
(1, None): 1,
(1, 1): 2,
(None, 1): 3}
for i in range(0, n):
l, u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
if not maxls > 0:
raise ValueError('maxls must be positive.')
x = array(x0, float64)
f = array(0.0, float64)
g = zeros((n,), float64)
wa = zeros(2*m*n + 5*n + 11*m*m + 8*m, float64)
iwa = zeros(3*n, int32)
task = zeros(1, 'S60')
csave = zeros(1, 'S60')
lsave = zeros(4, int32)
isave = zeros(44, int32)
dsave = zeros(29, float64)
task[:] = 'START'
n_iterations = 0
if t0 is None:
t0 = time.time()
time_profile.predicted_inner_loop_func2_duration = 0.0
while 1:
# x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
_lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
pgtol, wa, iwa, task, iprint, csave, lsave,
isave, dsave, maxls)
task_str = task.tostring()
# begin EB
curr_time = time.time()
predicted_inner_loop_func2_duration = (curr_time +
time_profile.maximize_inner_time_profile.mean +
time_profile.func2_time_profile.mean - t0)
if predicted_inner_loop_func2_duration > timeup: # JFF: added time-up check
task[:] = ('STOP: PREDICTED COMPUTATION TIME EXCEEDS LIMIT')
break
# end EB
if task_str.startswith(b'FG'):
# The minimization routine wants f and g at the current x.
# Note that interruptions due to maxfun are postponed
# until the completion of the current minimization iteration.
# Overwrite f and g:
f, g = func_and_grad(x)
elif task_str.startswith(b'NEW_X'):
# new iteration
if n_iterations > maxiter:
task[:] = 'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT'
elif n_function_evals[0] > maxfun:
task[:] = ('STOP: TOTAL NO. of f AND g EVALUATIONS '
'EXCEEDS LIMIT')
else:
n_iterations += 1
if callback is not None:
callback(x)
else:
break
time_profile.predicted_inner_loop_func2_duration = predicted_inner_loop_func2_duration
task_str = task.tostring().strip(b'\x00').strip()
if task_str.startswith(b'CONV'):
warnflag = 0
elif n_function_evals[0] > maxfun:
warnflag = 1
elif n_iterations > maxiter:
warnflag = 1
else:
warnflag = 2
# These two portions of the workspace are described in the mainlb
# subroutine in lbfgsb.f. See line 363.
s = wa[0: m*n].reshape(m, n)
y = wa[m*n: 2*m*n].reshape(m, n)
# See lbfgsb.f line 160 for this portion of the workspace.
# isave(31) = the total number of BFGS updates prior the current iteration;
n_bfgs_updates = isave[30]
n_corrs = min(n_bfgs_updates, maxcor)
hess_inv = LbfgsInvHessProduct(s[:n_corrs], y[:n_corrs])
return OptimizeResult(fun=f, jac=g, nfev=n_function_evals[0],
nit=n_iterations, status=warnflag, message=task_str,
x=x, success=(warnflag == 0), hess_inv=hess_inv)
def _minimize_cg(fun, x0, args=(), jac=None, callback=None,
gtol=1e-5, norm=Inf, eps=_epsilon, maxiter=None,
disp=False, return_all=False,
xtol= 1e-6,
**unknown_options):
"""
Minimization of scalar function of one or more variables using the
conjugate gradient algorithm.
Options for the conjugate gradient algorithm are:
disp : bool
Set to True to print convergence messages.
maxiter : int
Maximum number of iterations to perform.
gtol : float
Gradient norm must be less than `gtol` before successful
termination.
norm : float
Order of norm (Inf is max, -Inf is min).
eps : float or ndarray
If `jac` is approximated, use this value for the step size.
This function is called by the `minimize` function with `method=CG`. It
is not supposed to be called directly.
"""
_check_unknown_options(unknown_options)
f = fun
fprime = jac
epsilon = eps
retall = return_all
x0 = asarray(x0).flatten()
if maxiter is None:
maxiter = len(x0) * 200
func_calls, f = wrap_function(f, args)
if fprime is None:
grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
else:
grad_calls, myfprime = wrap_function(fprime, args)
gfk = myfprime(x0)
k = 0
xk = x0
old_fval = f(xk)
old_old_fval = None
if retall:
allvecs = [xk]
warnflag = 0
pk = -gfk
gnorm = vecnorm(gfk, ord=norm)
while (gnorm > gtol) and (k < maxiter):
deltak = numpy.dot(gfk, gfk)
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
_line_search_wolfe12(f, myfprime, xk, pk, gfk, old_fval,
old_old_fval, c2=0.4, xtol=xtol)
except _LineSearchError:
# Line search failed to find a better solution.
warnflag = 2
break
xk = xk + alpha_k * pk
if retall:
allvecs.append(xk)
if gfkp1 is None:
gfkp1 = myfprime(xk)
yk = gfkp1 - gfk
beta_k = max(0, numpy.dot(yk, gfkp1) / deltak)
pk = -gfkp1 + beta_k * pk
gfk = gfkp1
gnorm = vecnorm(gfk, ord=norm)
if callback is not None:
callback(xk)
k += 1
fval = old_fval
if warnflag == 2:
msg = _status_message['pr_loss']
if disp:
print("Warning: " + msg)
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
print(" Function evaluations: %d" % func_calls[0])
print(" Gradient evaluations: %d" % grad_calls[0])
elif k >= maxiter:
warnflag = 1
msg = _status_message['maxiter']
if disp:
print("Warning: " + msg)
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
print(" Function evaluations: %d" % func_calls[0])
print(" Gradient evaluations: %d" % grad_calls[0])
else:
msg = _status_message['success']
if disp:
print(msg)
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
print(" Function evaluations: %d" % func_calls[0])
print(" Gradient evaluations: %d" % grad_calls[0])
result = OptimizeResult(fun=fval, jac=gfk, nfev=func_calls[0],
njev=grad_calls[0], status=warnflag,
success=(warnflag == 0), message=msg, x=xk)
if retall:
result['allvecs'] = allvecs
return result
def _minimize_lbfgsb_multi(fun, x0, args=(), jac=None, bounds=None,
disp=None, maxcor=10, ftol=2.2204460492503131e-09,
gtol=1e-5, eps=1e-8, maxfun=15000, maxiter=15000,
iprint=-1, callback=None, maxls=20, **unknown_options):
"""
Minimize a scalar function of one or more variables using the L-BFGS-B
algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxcor : int
The maximum number of variable metric corrections used to
define the limited memory matrix. (The limited memory BFGS
method does not store the full hessian but uses this many terms
in an approximation to it.)
ftol : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol``.
gtol : float
The iteration will stop when ``max{|proj g_i | i = 1, ..., n}
<= gtol`` where ``pg_i`` is the i-th component of the
projected gradient.
eps : float
Step size used for numerical approximation of the jacobian.
disp : int
Set to True to print convergence messages.
maxfun : int
Maximum number of function evaluations.
maxiter : int
Maximum number of iterations.
maxls : int, optional
Maximum number of line search steps (per iteration). Default is 20.
Notes
-----
The option `ftol` is exposed via the `scipy.optimize.minimize` interface,
but calling `scipy.optimize.fmin_l_bfgs_b` directly exposes `factr`. The
relationship between the two is ``ftol = factr * numpy.finfo(float).eps``.
I.e., `factr` multiplies the default machine floating-point precision to
arrive at `ftol`.
"""
_check_unknown_options(unknown_options)
m = maxcor
epsilon = eps
pgtol = gtol
factr = ftol / np.finfo(float).eps
k,n = x0.shape
# x0 = [asarray(x).ravel() for x in x0]
# n, = x0.shape
if bounds is None:
bounds = [(None, None)] * n
if len(bounds) != n:
raise ValueError('length of x0 != length of bounds')
# unbounded variables must use None, not +-inf, for optimizer to work properly
bounds = [(None if l == -np.inf else l, None if u == np.inf else u) for l, u in bounds]
if disp is not None:
if disp == 0:
iprint = -1
else:
iprint = disp
if jac is None:
def func_and_grad(x):
# f = fun(x, *args)
f, g = _approx_fprime_helper(x, fun, epsilon, args=args)
return f, g
else:
def func_and_grad(x):
f = fun(x, *args)
g = jac(x, *args)
return f, g
nbd = zeros(n, int32)
low_bnd = zeros(n, float64)
upper_bnd = zeros(n, float64)
bounds_map = {(None, None): 0,
(1, None): 1,
(1, 1): 2,
(None, 1): 3}
for i in range(0, n):
l, u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
if not maxls > 0:
raise ValueError('maxls must be positive.')
# x = [array(x0_, float64) for x0_ in x0]
X = x0.copy()
f = [array(0.0, float64) for _ in range(k)]
g = [zeros((n,), float64) for _ in range(k)]
wa = [zeros(2*m*n + 5*n + 11*m*m + 8*m, float64) for _ in range(k)]
iwa = [zeros(3*n, int32) for _ in range(k)]
task = [zeros(1, 'S60') for _ in range(k)]
csave = [zeros(1, 'S60') for _ in range(k)]
lsave = [zeros(4, int32) for _ in range(k)]
isave = [zeros(44, int32) for _ in range(k)]
dsave = [zeros(29, float64) for _ in range(k)]
n_function_evals = 0
for i in range(k):
task[i][:] = 'START'
n_iterations = [0 for _ in range(k)]
k_running = [i for i in range(k)]
while 1:
# x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
request_index = []
# run all instance until they request a new point
# X contains only points for the remaining instances
for i in k_running:
while 1:
_lbfgsb.setulb(m, X[i], low_bnd, upper_bnd, nbd, f[i], g[i], factr,
pgtol, wa[i], iwa[i], task[i], iprint, csave[i], lsave[i],
isave[i], dsave[i], maxls)
task_str = task[i].tostring()
if task_str.startswith(b'FG'):
# The minimization routine wants f and g at the current x.
# Note that interruptions due to maxfun are postponed
# until the completion of the current minimization iteration.
# Overwrite f and g:
request_index.append(i)
break
elif task_str.startswith(b'NEW_X'):
# new iteration
if n_iterations[i] > maxiter:
task[i][:] = 'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT'
# break
elif n_function_evals > maxfun:
task[i][:] = ('STOP: TOTAL NO. of f AND g EVALUATIONS '
'EXCEEDS LIMIT')
# break
else:
n_iterations[i] += 1
# if callback is not None:
# callback(x)
else:
break
k_running = request_index
if len(k_running) == 0:
break
F,G = func_and_grad(X[request_index])
n_function_evals += 1
for ff,gg,i in zip(F,G,k_running):
f[i] = ff
g[i] = gg
warnflag = []
task_str = [t.tostring().strip(b'\x00').strip() for t in task]
for i,t in enumerate(task_str):
if t.startswith(b'CONV'):
warnflag.append(0)
elif n_function_evals > maxfun:
warnflag.append(1)
elif n_iterations[i] > maxiter:
warnflag.append(2)
else:
warnflag.append(3)
# These two portions of the workspace are described in the mainlb
# subroutine in lbfgsb.f. See line 363.
# s = [wa[i][0: m*n].reshape(m, n) for i in range(k)]
# y = [wa[i][m*n: 2*m*n].reshape(m, n) for i in range(k)]
# See lbfgsb.f line 160 for this portion of the workspace.
# isave(31) = the total number of BFGS updates prior the current iteration;
# n_bfgs_updates = isave[30]
# n_corrs = min(n_bfgs_updates, maxcor)
# hess_inv = LbfgsInvHessProduct(s[:n_corrs], y[:n_corrs])
return OptimizeResult(fun=f, jac=g, nfev=n_function_evals,
nit=n_iterations, status=warnflag, message=task_str,
x=X, success=(sum(warnflag) == 0))
def _minimize_lbfgsb(
fun,
x0,
bounds=None,
args=(),
kwargs={},
jac=None,
callback=None,
tol={"abs": 1e-05, "rel": 1e-08},
norm=np.Inf,
maxiter=None,
disp=False,
return_all=False,
**unknown_options
):
"""
Minimization of scalar function of one or more variables using the
BHHH algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxiter : int
Maximum number of iterations to perform.
tol : dict
Absolute and relative tolerance values.
norm : float
Order of norm (Inf is max, -Inf is min).
"""
_check_unknown_options(unknown_options)
def f(x0):
return fun(x0, *args, **kwargs)
fprime = jac
# epsilon = eps Add functionality
retall = return_all
k = 0
ns = 0
nsmax = 5
N = len(x0)
x0 = np.asarray(x0).flatten()
if x0.ndim == 0:
x0.shape = (1,)
if bounds is None:
bounds = np.array([np.inf] * N * 2).reshape((2, N))
bounds[0, :] = -bounds[0, :]
if bounds.shape[1] != N:
raise ValueError("length of x0 != length of bounds")
low = bounds[0, :]
up = bounds[1, :]
x0 = np.clip(x0, low, up)
if maxiter is None:
maxiter = len(x0) * 200
if not callable(fprime):
def myfprime(x0):
return approx_derivative(f, x0, args=args, kwargs=kwargs)
else:
myfprime = fprime
# Setup for iteration
old_fval = f(x0)
gf0 = myfprime(x0)
gfk = gf0
norm_pg0 = vecnorm(x0 - np.clip(x0 - gf0, low, up), ord=norm)
xk = x0
norm_pgk = norm_pg0
sstore = np.zeros((maxiter, N))
ystore = sstore.copy()
if retall:
allvecs = [x0]
warnflag = 0
# Calculate indices ofactive and inative set using projected gradient
epsilon = min(np.min(up - low) / 2, norm_pgk)
activeset = np.logical_or(xk - low <= epsilon, up - xk <= epsilon)
inactiveset = np.logical_not(activeset)
for _ in range(maxiter): # for loop instead.
# Check tolerance of gradient norm
if norm_pgk <= tol["abs"] + tol["rel"] * norm_pg0:
break
pk = -gfk
pk = bfgsrecb(ns, sstore, ystore, pk, activeset)
gfk_active = gfk.copy()
gfk_active[inactiveset] = 0
pk = -gfk_active + pk
# Sets the initial step guess to dx ~ 1
old_old_fval = old_fval + np.linalg.norm(gfk) / 2
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = _line_search_wolfe12(
f,
myfprime,
xk,
pk,
gfk,
old_fval,
old_old_fval,
amin=1e-100,
amax=1e100,
)
except _LineSearchError:
# Line search failed to find a better solution.
warnflag = 2
break
xkp1 = np.clip(xk + alpha_k * pk, low, up)
if retall:
allvecs.append(xkp1)
yk = myfprime(xkp1) - gfk
sk = xkp1 - xk
xk = xkp1
gfk = myfprime(xkp1)
norm_pgk = vecnorm(xk - np.clip(xk - gfk, low, up), ord=norm)
# Calculate indices ofactive and inative set using projected gradient
epsilon = min(np.min(up - low) / 2, norm_pgk)
activeset = np.logical_or(xk - low <= epsilon, up - xk <= epsilon)
inactiveset = np.logical_not(activeset)
yk[activeset] = 0
sk[activeset] = 0
# reset storage
ytsk = yk.dot(sk)
if ytsk <= 0:
ns = 0
if ns == nsmax:
print("ns reached maximum size")
ns = 0
elif ytsk > 0:
ns += 1
alpha0 = ytsk ** 0.5
sstore[ns - 1, :] = sk / alpha0
ystore[ns - 1, :] = yk / alpha0
k += 1
if callback is not None:
callback(xk)
if np.isinf(old_fval):
# We correctly found +-Inf as optimal value, or something went
# wrong.
warnflag = 2
break
if np.isnan(xk).any():
warnflag = 3
break
fval = old_fval
if warnflag == 2:
msg = _status_message["pr_loss"]
elif k >= maxiter:
warnflag = 1
msg = _status_message["maxiter"]
elif np.isnan(fval) or np.isnan(xk).any():
warnflag = 3
msg = _status_message["nan"]
else:
msg = _status_message["success"]
if disp:
print("{}{}".format("Warning: " if warnflag != 0 else "", msg))
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
result = OptimizeResult(
fun=fval,
jac=gfk,
status=warnflag,
success=(warnflag == 0),
message=msg,
x=xk,
nit=k,
)
if retall:
result["allvecs"] = allvecs
return result
def _linprog_simplex(c, A_ub=None, b_ub=None, A_eq=None, b_eq=None,
bounds=None, maxiter=1000, disp=False, callback=None,
tol=1.0E-12, bland=False, **unknown_options):
"""
Solve the following linear programming problem via a two-phase
simplex algorithm.
maximize: c^T * x
subject to: A_ub * x <= b_ub
A_eq * x == b_eq
Parameters
----------
c : array_like
Coefficients of the linear objective function to be maximized.
A_ub :
2-D array which, when matrix-multiplied by x, gives the values of the
upper-bound inequality constraints at x.
b_ub : array_like
1-D array of values representing the upper-bound of each inequality
constraint (row) in A_ub.
A_eq : array_like
2-D array which, when matrix-multiplied by x, gives the values of the
equality constraints at x.
b_eq : array_like
1-D array of values representing the RHS of each equality constraint
(row) in A_eq.
bounds : array_like
The bounds for each independent variable in the solution, which can take
one of three forms::
None : The default bounds, all variables are non-negative.
(lb, ub) : If a 2-element sequence is provided, the same
lower bound (lb) and upper bound (ub) will be applied
to all variables.
[(lb_0, ub_0), (lb_1, ub_1), ...] : If an n x 2 sequence is provided,
each variable x_i will be bounded by lb[i] and ub[i].
Infinite bounds are specified using -np.inf (negative)
or np.inf (positive).
maxiter : int
The maximum number of iterations to perform.
disp : bool
If True, print exit status message to sys.stdout
callback : callable
If a callback function is provide, it will be called within each
iteration of the simplex algorithm. The callback must have the
signature `callback(xk, **kwargs)` where xk is the current solution
vector and kwargs is a dictionary containing the following::
"tableau" : The current Simplex algorithm tableau
"nit" : The current iteration.
"pivot" : The pivot (row, column) used for the next iteration.
"phase" : Whether the algorithm is in Phase 1 or Phase 2.
"bv" : A structured array containing a string representation of each
basic variable and its current value.
tol : float
The tolerance which determines when a solution is "close enough" to zero
in Phase 1 to be considered a basic feasible solution or close enough
to positive to to serve as an optimal solution.
bland : bool
If True, use Bland's anti-cycling rule [3] to choose pivots to
prevent cycling. If False, choose pivots which should lead to a
converged solution more quickly. The latter method is subject to
cycling (non-convergence) in rare instances.
Returns
-------
A scipy.optimize.OptimizeResult consisting of the following fields::
x : ndarray
The independent variable vector which optimizes the linear
programming problem.
slack : ndarray
The values of the slack variables. Each slack variable corresponds
to an inequality constraint. If the slack is zero, then the
corresponding constraint is active.
success : bool
Returns True if the algorithm succeeded in finding an optimal
solution.
status : int
An integer representing the exit status of the optimization::
0 : Optimization terminated successfully
1 : Iteration limit reached
2 : Problem appears to be infeasible
3 : Problem appears to be unbounded
nit : int
The number of iterations performed.
message : str
A string descriptor of the exit status of the optimization.
Examples
--------
Consider the following problem:
Minimize: f = -1*x[0] + 4*x[1]
Subject to: -3*x[0] + 1*x[1] <= 6
1*x[0] + 2*x[1] <= 4
x[1] >= -3
where: -inf <= x[0] <= inf
This problem deviates from the standard linear programming problem. In
standard form, linear programming problems assume the variables x are
non-negative. Since the variables don't have standard bounds where
0 <= x <= inf, the bounds of the variables must be explicitly set.
There are two upper-bound constraints, which can be expressed as
dot(A_ub, x) <= b_ub
The input for this problem is as follows:
>>> c = [-1, 4]
>>> A = [[-3, 1], [1, 2]]
>>> b = [6, 4]
>>> x0_bnds = (None, None)
>>> x1_bnds = (-3, None)
>>> res = linprog(c, A, b, bounds=(x0_bnds, x1_bnds), options={"disp":True})
>>> print(res)
Optimization terminated successfully.
Current function value: 11.428571
Iterations: 2
status: 0
success: True
fun: 11.428571428571429
x: array([-1.14285714, 2.57142857])
slack: array([], dtype=np.float64)
message: 'Optimization terminated successfully.'
nit: 2
References
----------
.. [1] Dantzig, George B., Linear programming and extensions. Rand
Corporation Research Study Princeton Univ. Press, Princeton, NJ, 1963
.. [2] Hillier, S.H. and Lieberman, G.J. (1995), "Introduction to
Mathematical Programming", McGraw-Hill, Chapter 4.
.. [3] Bland, Robert G. New finite pivoting rules for the simplex method.
Mathematics of Operations Research (2), 1977: pp. 103-107.
"""
_check_unknown_options(unknown_options)
status = 0
messages = {0: "Optimization terminated successfully.",
1: "Iteration limit reached.",
2: "Optimzation failed. Unable to find a feasible"
" starting point.",
3: "Optimization failed. The problem appears to be unbounded.",
4: "Optimization failed. Singular matrix encountered."}
have_floor_variable = False
cc = np.asarray(c)
# The initial value of the objective function element in the tableau
f0 = 0
# The number of variables as given by c
n = len(c)
# Convert the input arguments to arrays (sized to zero if not provided)
Aeq = np.asarray(A_eq) if A_eq is not None else np.empty([0, len(cc)])
Aub = np.asarray(A_ub) if A_ub is not None else np.empty([0, len(cc)])
beq = np.ravel(np.asarray(b_eq)) if b_eq is not None else np.empty([0])
bub = np.ravel(np.asarray(b_ub)) if b_ub is not None else np.empty([0])
# Analyze the bounds and determine what modifications to me made to
# the constraints in order to accommodate them.
L = np.zeros(n, dtype=np.float64)
U = np.ones(n, dtype=np.float64) * np.inf
if bounds is None or len(bounds) == 0:
pass
elif len(bounds) == 2 and not hasattr(bounds[0], '__len__'):
# All bounds are the same
L = np.asarray(n * [bounds[0]], dtype=np.float64)
U = np.asarray(n * [bounds[1]], dtype=np.float64)
else:
if len(bounds) != n:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Length of bounds is inconsistent with the length of c")
else:
try:
for i in range(n):
if len(bounds[i]) != 2:
raise IndexError()
L[i] = bounds[i][0] if bounds[i][0] is not None else -np.inf
U[i] = bounds[i][1] if bounds[i][1] is not None else np.inf
except IndexError:
status = -1
message = ("Invalid input for linprog with "
"method = 'simplex'. bounds must be a n x 2 "
"sequence/array where n = len(c).")
if np.any(L == -np.inf):
# If any lower-bound constraint is a free variable
# add the first column variable as the "floor" variable which
# accommodates the most negative variable in the problem.
n = n + 1
L = np.concatenate([np.array([0]), L])
U = np.concatenate([np.array([np.inf]), U])
cc = np.concatenate([np.array([0]), cc])
Aeq = np.hstack([np.zeros([Aeq.shape[0], 1]), Aeq])
Aub = np.hstack([np.zeros([Aub.shape[0], 1]), Aub])
have_floor_variable = True
# Now before we deal with any variables with lower bounds < 0,
# deal with finite bounds which can be simply added as new constraints.
# Also validate bounds inputs here.
for i in range(n):
if(L[i] > U[i]):
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Lower bound %d is greater than upper bound %d" % (i, i))
if np.isinf(L[i]) and L[i] > 0:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Lower bound may not be +infinity")
if np.isinf(U[i]) and U[i] < 0:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Upper bound may not be -infinity")
if np.isfinite(L[i]) and L[i] > 0:
# Add a new lower-bound (negative upper-bound) constraint
Aub = np.vstack([Aub, np.zeros(n)])
Aub[-1, i] = -1
bub = np.concatenate([bub, np.array([-L[i]])])
L[i] = 0
if np.isfinite(U[i]):
# Add a new upper-bound constraint
Aub = np.vstack([Aub, np.zeros(n)])
Aub[-1, i] = 1
bub = np.concatenate([bub, np.array([U[i]])])
U[i] = np.inf
# Now find negative lower bounds (finite or infinite) which require a
# change of variables or free variables and handle them appropriately
for i in range(0, n):
if L[i] < 0:
if np.isfinite(L[i]) and L[i] < 0:
# Add a change of variables for x[i]
# For each row in the constraint matrices, we take the
# coefficient from column i in A,
# and subtract the product of that and L[i] to the RHS b
beq[:] = beq[:] - Aeq[:, i] * L[i]
bub[:] = bub[:] - Aub[:, i] * L[i]
# We now have a nonzero initial value for the objective
# function as well.
f0 = f0 - cc[i] * L[i]
else:
# This is an unrestricted variable, let x[i] = u[i] - v[0]
# where v is the first column in all matrices.
Aeq[:, 0] = Aeq[:, 0] - Aeq[:, i]
Aub[:, 0] = Aub[:, 0] - Aub[:, i]
cc[0] = cc[0] - cc[i]
if np.isinf(U[i]):
if U[i] < 0:
status = -1
message = ("Invalid input for linprog with "
"method = 'simplex'. Upper bound may not be -inf.")
# The number of upper bound constraints (rows in A_ub and elements in b_ub)
mub = len(bub)
# The number of equality constraints (rows in A_eq and elements in b_eq)
meq = len(beq)
# The total number of constraints
m = mub + meq
# The number of slack variables (one for each of the upper-bound constraints)
n_slack = mub
# The number of artificial variables (one for each lower-bound and equality
# constraint)
n_artificial = meq + _count_nonzero(bub < 0)
try:
Aub_rows, Aub_cols = Aub.shape
except ValueError:
raise ValueError("Invalid input. A_ub must be two-dimensional")
try:
Aeq_rows, Aeq_cols = Aeq.shape
except ValueError:
raise ValueError("Invalid input. A_eq must be two-dimensional")
if Aeq_rows != meq:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"The number of rows in A_eq must be equal "
"to the number of values in b_eq")
if Aub_rows != mub:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"The number of rows in A_ub must be equal "
"to the number of values in b_ub")
if Aeq_cols > 0 and Aeq_cols != n:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Number of columns in A_eq must be equal "
"to the size of c")
if Aub_cols > 0 and Aub_cols != n:
status = -1
message = ("Invalid input for linprog with method = 'simplex'. "
"Number of columns in A_ub must be equal to the size of c")
if status != 0:
# Invalid inputs provided
raise ValueError(message)
# Create the tableau
T = np.zeros([m + 2, n + n_slack + n_artificial + 1])
# Insert objective into tableau
T[-2, :n] = cc
T[-2, -1] = f0
b = T[:-2, -1]
if meq > 0:
# Add Aeq to the tableau
T[:meq, :n] = Aeq
# Add beq to the tableau
b[:meq] = beq
if mub > 0:
# Add Aub to the tableau
T[meq:meq + mub, :n] = Aub
# At bub to the tableau
b[meq:meq + mub] = bub
# Add the slack variables to the tableau
np.fill_diagonal(T[meq:m, n:n + n_slack], 1)
# Further setup the tableau
# If a row corresponds to an equality constraint or a negative b (a lower
# bound constraint), then an artificial variable is added for that row.
# Also, if b is negative, first flip the signs in that constraint.
slcount = 0
avcount = 0
basis = np.zeros(m, dtype=int)
r_artificial = np.zeros(n_artificial, dtype=int)
for i in range(m):
if i < meq or b[i] < 0:
# basic variable i is in column n+n_slack+avcount
basis[i] = n + n_slack + avcount
r_artificial[avcount] = i
avcount += 1
if b[i] < 0:
b[i] *= -1
T[i, :-1] *= -1
T[i, basis[i]] = 1
T[-1, basis[i]] = 1
else:
# basic variable i is in column n+slcount
basis[i] = n + slcount
slcount += 1
# Make the artificial variables basic feasible variables by subtracting
# each row with an artificial variable from the Phase 1 objective
for r in r_artificial:
T[-1, :] = T[-1, :] - T[r, :]
nit1, status = _solve_simplex(T, n, basis, phase=1, callback=callback,
maxiter=maxiter, tol=tol, bland=bland)
# if pseudo objective is zero, remove the last row from the tableau and
# proceed to phase 2
if abs(T[-1, -1]) < tol:
# Remove the pseudo-objective row from the tableau
T = T[:-1, :]
# Remove the artificial variable columns from the tableau
T = np.delete(T, np.s_[n + n_slack:n + n_slack + n_artificial], 1)
else:
# Failure to find a feasible starting point
status = 2
if status != 0:
message = messages[status]
if disp:
print(message)
return OptimizeResult(x=np.nan, fun=-T[-1, -1], nit=nit1, status=status,
message=message, success=False)
# Phase 2
nit2, status = _solve_simplex(T, n, basis, maxiter=maxiter - nit1, phase=2,
callback=callback, tol=tol, nit0=nit1,
bland=bland)
solution = np.zeros(n + n_slack + n_artificial)
solution[basis[:m]] = T[:m, -1]
x = solution[:n]
slack = solution[n:n + n_slack]
# For those variables with finite negative lower bounds,
# reverse the change of variables
masked_L = np.ma.array(L, mask=np.isinf(L), fill_value=0.0).filled()
x = x + masked_L
# For those variables with infinite negative lower bounds,
# take x[i] as the difference between x[i] and the floor variable.
if have_floor_variable:
for i in range(1, n):
if np.isinf(L[i]):
x[i] -= x[0]
x = x[1:]
# Optimization complete at this point
obj = -T[-1, -1]
if status in (0, 1):
if disp:
print(messages[status])
print(" Current function value: {: <12.6f}".format(obj))
print(" Iterations: {:d}".format(nit2))
else:
if disp:
print(messages[status])
print(" Iterations: {:d}".format(nit2))
return OptimizeResult(x=x, fun=obj, nit=int(nit2), status=status, slack=slack,
message=messages[status], success=(status == 0))
def _minimize_bhhh(fun,
x0,
bounds=None,
args=(),
jac=None,
callback=None,
tol={
"abs": 1e-05,
"rel": 1e-08
},
norm=np.Inf,
maxiter=None,
disp=False,
return_all=False,
**unknown_options):
"""
Minimization of scalar function of one or more variables using the
BHHH algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxiter : int
Maximum number of iterations to perform.
tol : dict
Absolute and relative tolerance values.
norm : float
Order of norm (Inf is max, -Inf is min).
"""
_check_unknown_options(unknown_options)
f = fun
fprime = jac
retall = return_all
k = 0
N = len(x0)
x0 = np.asarray(x0).flatten()
if x0.ndim == 0:
x0.shape = (1, )
if bounds is None:
bounds = np.array([np.inf] * N * 2).reshape((2, N))
bounds[0, :] = -bounds[0, :]
if bounds.shape[1] != N:
raise ValueError("length of x0 != length of bounds")
low = bounds[0, :]
up = bounds[1, :]
x0 = np.clip(x0, low, up)
if maxiter is None:
maxiter = len(x0) * 200
# Need the aggregate functions to take only x0 as an argument
func_calls, agg_fun = wrap_function_agg(f, args)
if not callable(fprime):
grad_calls, myfprime = wrap_function_num_dev(f, args)
else:
grad_calls, myfprime = wrap_function(fprime, args)
def agg_fprime(x0):
return myfprime(x0).sum(axis=0)
# Setup for iteration
old_fval = agg_fun(x0)
gf0 = agg_fprime(x0)
norm_pg0 = vecnorm(x0 - np.clip(x0 - gf0, low, up), ord=norm)
xk = x0
norm_pgk = norm_pg0
if retall:
allvecs = [x0]
warnflag = 0
for _ in range(maxiter): # for loop instead.
# Individual
gfk_obs = myfprime(xk)
# Aggregate fprime. Might replace by simply summing up gfk_obs
gfk = gfk_obs.sum(axis=0)
norm_pgk = vecnorm(xk - np.clip(xk - gfk, low, up), ord=norm)
# Check tolerance of gradient norm
if norm_pgk <= tol["abs"] + tol["rel"] * norm_pg0:
break
# Sets the initial step guess to dx ~ 1
old_old_fval = old_fval + np.linalg.norm(gfk) / 2
# Calculate BHHH hessian and step
Hk = np.dot(gfk_obs.T, gfk_obs)
Bk = np.linalg.inv(Hk)
pk = np.empty(N)
pk = -np.dot(Bk, gfk)
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = _line_search_wolfe12(
agg_fun,
agg_fprime,
xk,
pk,
gfk,
old_fval,
old_old_fval,
amin=1e-100,
amax=1e100,
)
except _LineSearchError:
# Line search failed to find a better solution.
warnflag = 2
break
xkp1 = np.clip(xk + alpha_k * pk, low, up)
if retall:
allvecs.append(xkp1)
xk = xkp1
if callback is not None:
callback(xk)
k += 1
if np.isinf(old_fval):
# We correctly found +-Inf as optimal value, or something went
# wrong.
warnflag = 2
break
fval = old_fval
if warnflag == 2:
msg = _status_message["pr_loss"]
elif k >= maxiter:
warnflag = 1
msg = _status_message["maxiter"]
elif np.isnan(fval) or np.isnan(xk).any():
warnflag = 3
msg = _status_message["nan"]
else:
msg = _status_message["success"]
if disp:
print("{}{}".format("Warning: " if warnflag != 0 else "", msg))
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
result = OptimizeResult(
fun=fval,
jac=gfk,
hess_inv=Bk,
nfev=func_calls[0],
njev=grad_calls[0],
status=warnflag,
success=(warnflag == 0),
message=msg,
x=xk,
nit=k,
)
if retall:
result["allvecs"] = allvecs
return result
