def _root_df_sane(func, x0, args=(), ftol=1e-8, fatol=1e-300, maxfev=1000, fnorm=None, callback=None, disp=False, M=10, eta_strategy=None, sigma_eps=1e-10, sigma_0=1.0, line_search='cruz', **unknown_options): r""" Solve nonlinear equation with the DF-SANE method Options ------- ftol : float, optional Relative norm tolerance. fatol : float, optional Absolute norm tolerance. Algorithm terminates when ``||func(x)|| < fatol + ftol ||func(x_0)||``. fnorm : callable, optional Norm to use in the convergence check. If None, 2-norm is used. maxfev : int, optional Maximum number of function evaluations. disp : bool, optional Whether to print convergence process to stdout. eta_strategy : callable, optional Choice of the ``eta_k`` parameter, which gives slack for growth of ``||F||**2``. Called as ``eta_k = eta_strategy(k, x, F)`` with `k` the iteration number, `x` the current iterate and `F` the current residual. Should satisfy ``eta_k > 0`` and ``sum(eta, k=0..inf) < inf``. Default: ``||F||**2 / (1 + k)**2``. sigma_eps : float, optional The spectral coefficient is constrained to ``sigma_eps < sigma < 1/sigma_eps``. Default: 1e-10 sigma_0 : float, optional Initial spectral coefficient. Default: 1.0 M : int, optional Number of iterates to include in the nonmonotonic line search. Default: 10 line_search : {'cruz', 'cheng'} Type of line search to employ. 'cruz' is the original one defined in [Martinez & Raydan. Math. Comp. 75, 1429 (2006)], 'cheng' is a modified search defined in [Cheng & Li. IMA J. Numer. Anal. 29, 814 (2009)]. Default: 'cruz' References ---------- .. [1] "Spectral residual method without gradient information for solving large-scale nonlinear systems of equations." W. La Cruz, J.M. Martinez, M. Raydan. Math. Comp. **75**, 1429 (2006). .. [2] W. La Cruz, Opt. Meth. Software, 29, 24 (2014). .. [3] W. Cheng, D.-H. Li. IMA J. Numer. Anal. **29**, 814 (2009). """ _check_unknown_options(unknown_options) if line_search not in ('cheng', 'cruz'): raise ValueError("Invalid value %r for 'line_search'" % (line_search, )) nexp = 2 if eta_strategy is None: # Different choice from [1], as their eta is not invariant # vs. scaling of F. def eta_strategy(k, x, F): # Obtain squared 2-norm of the initial residual from the outer scope return f_0 / (1 + k)**2 if fnorm is None: def fnorm(F): # Obtain squared 2-norm of the current residual from the outer scope return f_k**(1.0 / nexp) def fmerit(F): return np.linalg.norm(F)**nexp nfev = [0] f, x_k, x_shape, f_k, F_k, is_complex = _wrap_func(func, x0, fmerit, nfev, maxfev, args) k = 0 f_0 = f_k sigma_k = sigma_0 F_0_norm = fnorm(F_k) # For the 'cruz' line search prev_fs = collections.deque([f_k], M) # For the 'cheng' line search Q = 1.0 C = f_0 converged = False message = "too many function evaluations required" while True: F_k_norm = fnorm(F_k) if disp: print("iter %d: ||F|| = %g, sigma = %g" % (k, F_k_norm, sigma_k)) if callback is not None: callback(x_k, F_k) if F_k_norm < ftol * F_0_norm + fatol: # Converged! message = "successful convergence" converged = True break # Control spectral parameter, from [2] if abs(sigma_k) > 1 / sigma_eps: sigma_k = 1 / sigma_eps * np.sign(sigma_k) elif abs(sigma_k) < sigma_eps: sigma_k = sigma_eps # Line search direction d = -sigma_k * F_k # Nonmonotone line search eta = eta_strategy(k, x_k, F_k) try: if line_search == 'cruz': alpha, xp, fp, Fp = _nonmonotone_line_search_cruz(f, x_k, d, prev_fs, eta=eta) elif line_search == 'cheng': alpha, xp, fp, Fp, C, Q = _nonmonotone_line_search_cheng( f, x_k, d, f_k, C, Q, eta=eta) except _NoConvergence: break # Update spectral parameter s_k = xp - x_k y_k = Fp - F_k sigma_k = np.vdot(s_k, s_k) / np.vdot(s_k, y_k) # Take step x_k = xp F_k = Fp f_k = fp # Store function value if line_search == 'cruz': prev_fs.append(fp) k += 1 x = _wrap_result(x_k, is_complex, shape=x_shape) F = _wrap_result(F_k, is_complex) result = OptimizeResult(x=x, success=converged, message=message, fun=F, nfev=nfev[0], nit=k) return result
def _minimize_slsqp(func, x0, args=(), jac=None, bounds=None, constraints=(), maxiter=100, ftol=1.0E-6, iprint=1, disp=False, eps=_epsilon, callback=None, **unknown_options): """ Minimize a scalar function of one or more variables using Sequential Least SQuares Programming (SLSQP). Options ------- ftol : float Precision goal for the value of f in the stopping criterion. eps : float Step size used for numerical approximation of the jacobian. disp : bool Set to True to print convergence messages. If False, `verbosity` is ignored and set to 0. maxiter : int Maximum number of iterations. """ _check_unknown_options(unknown_options) fprime = jac iter = maxiter acc = ftol epsilon = eps if not disp: iprint = 0 # Constraints are triaged per type into a dictionnary of tuples if isinstance(constraints, dict): constraints = (constraints, ) cons = {'eq': (), 'ineq': ()} for ic, con in enumerate(constraints): # check type try: ctype = con['type'].lower() except KeyError: raise KeyError('Constraint %d has no type defined.' % ic) except TypeError: raise TypeError('Constraints must be defined using a ' 'dictionary.') except AttributeError: raise TypeError("Constraint's type must be a string.") else: if ctype not in ['eq', 'ineq']: raise ValueError("Unknown constraint type '%s'." % con['type']) # check function if 'fun' not in con: raise ValueError('Constraint %d has no function defined.' % ic) # check jacobian cjac = con.get('jac') if cjac is None: # approximate jacobian function. The factory function is needed # to keep a reference to `fun`, see gh-4240. def cjac_factory(fun): def cjac(x, *args): return approx_jacobian(x, fun, epsilon, *args) return cjac cjac = cjac_factory(con['fun']) # update constraints' dictionary cons[ctype] += ({ 'fun': con['fun'], 'jac': cjac, 'args': con.get('args', ()) }, ) exit_modes = { -1: "Gradient evaluation required (g & a)", 0: "Optimization terminated successfully.", 1: "Function evaluation required (f & c)", 2: "More equality constraints than independent variables", 3: "More than 3*n iterations in LSQ subproblem", 4: "Inequality constraints incompatible", 5: "Singular matrix E in LSQ subproblem", 6: "Singular matrix C in LSQ subproblem", 7: "Rank-deficient equality constraint subproblem HFTI", 8: "Positive directional derivative for linesearch", 9: "Iteration limit exceeded" } # Wrap func feval, func = wrap_function(func, args) # Wrap fprime, if provided, or approx_jacobian if not if fprime: geval, fprime = wrap_function(fprime, args) else: geval, fprime = wrap_function(approx_jacobian, (func, epsilon)) # Transform x0 into an array. x = asfarray(x0).flatten() # Set the parameters that SLSQP will need # meq, mieq: number of equality and inequality constraints meq = sum( map(len, [atleast_1d(c['fun'](x, *c['args'])) for c in cons['eq']])) mieq = sum( map(len, [atleast_1d(c['fun'](x, *c['args'])) for c in cons['ineq']])) # m = The total number of constraints m = meq + mieq # la = The number of constraints, or 1 if there are no constraints la = array([1, m]).max() # n = The number of independent variables n = len(x) # Define the workspaces for SLSQP n1 = n + 1 mineq = m - meq + n1 + n1 len_w = (3*n1+m)*(n1+1)+(n1-meq+1)*(mineq+2) + 2*mineq+(n1+mineq)*(n1-meq) \ + 2*meq + n1 + ((n+1)*n)//2 + 2*m + 3*n + 3*n1 + 1 len_jw = mineq w = zeros(len_w) jw = zeros(len_jw) # Decompose bounds into xl and xu if bounds is None or len(bounds) == 0: xl = np.empty(n, dtype=float) xu = np.empty(n, dtype=float) xl.fill(np.nan) xu.fill(np.nan) else: bnds = array(bounds, float) if bnds.shape[0] != n: raise IndexError('SLSQP Error: the length of bounds is not ' 'compatible with that of x0.') with np.errstate(invalid='ignore'): bnderr = bnds[:, 0] > bnds[:, 1] if bnderr.any(): raise ValueError('SLSQP Error: lb > ub in bounds %s.' % ', '.join(str(b) for b in bnderr)) xl, xu = bnds[:, 0], bnds[:, 1] # Mark infinite bounds with nans; the Fortran code understands this infbnd = ~isfinite(bnds) xl[infbnd[:, 0]] = np.nan xu[infbnd[:, 1]] = np.nan # Clip initial guess to bounds (SLSQP may fail with bounds-infeasible initial point) have_bound = np.isfinite(xl) x[have_bound] = np.clip(x[have_bound], xl[have_bound], np.inf) have_bound = np.isfinite(xu) x[have_bound] = np.clip(x[have_bound], -np.inf, xu[have_bound]) # Initialize the iteration counter and the mode value mode = array(0, int) acc = array(acc, float) majiter = array(iter, int) majiter_prev = 0 # Print the header if iprint >= 2 if iprint >= 2: print("%5s %5s %16s %16s" % ("NIT", "FC", "OBJFUN", "GNORM")) while 1: if mode == 0 or mode == 1: # objective and constraint evaluation requird # Compute objective function fx = func(x) try: fx = float(np.asarray(fx)) except (TypeError, ValueError): raise ValueError("Objective function must return a scalar") # Compute the constraints if cons['eq']: c_eq = concatenate([ atleast_1d(con['fun'](x, *con['args'])) for con in cons['eq'] ]) else: c_eq = zeros(0) if cons['ineq']: c_ieq = concatenate([ atleast_1d(con['fun'](x, *con['args'])) for con in cons['ineq'] ]) else: c_ieq = zeros(0) # Now combine c_eq and c_ieq into a single matrix c = concatenate((c_eq, c_ieq)) if mode == 0 or mode == -1: # gradient evaluation required # Compute the derivatives of the objective function # For some reason SLSQP wants g dimensioned to n+1 g = append(fprime(x), 0.0) # Compute the normals of the constraints if cons['eq']: a_eq = vstack( [con['jac'](x, *con['args']) for con in cons['eq']]) else: # no equality constraint a_eq = zeros((meq, n)) if cons['ineq']: a_ieq = vstack( [con['jac'](x, *con['args']) for con in cons['ineq']]) else: # no inequality constraint a_ieq = zeros((mieq, n)) # Now combine a_eq and a_ieq into a single a matrix if m == 0: # no constraints a = zeros((la, n)) else: a = vstack((a_eq, a_ieq)) a = concatenate((a, zeros([la, 1])), 1) # Call SLSQP slsqp(m, meq, x, xl, xu, fx, c, g, a, acc, majiter, mode, w, jw) # call callback if major iteration has incremented if callback is not None and majiter > majiter_prev: callback(x) # Print the status of the current iterate if iprint > 2 and the # major iteration has incremented if iprint >= 2 and majiter > majiter_prev: print("%5i %5i % 16.6E % 16.6E" % (majiter, feval[0], fx, linalg.norm(g))) # If exit mode is not -1 or 1, slsqp has completed if abs(mode) != 1: break majiter_prev = int(majiter) # Optimization loop complete. Print status if requested if iprint >= 1: print(exit_modes[int(mode)] + " (Exit mode " + str(mode) + ')') print(" Current function value:", fx) print(" Iterations:", majiter) print(" Function evaluations:", feval[0]) print(" Gradient evaluations:", geval[0]) return OptimizeResult(x=x, fun=fx, jac=g[:-1], nit=int(majiter), nfev=feval[0], njev=geval[0], status=int(mode), message=exit_modes[int(mode)], success=(mode == 0))
def _root_df_sane( func, x0, args=(), ftol=1e-8, fatol=1e-300, maxfev=1000, fnorm=None, callback=None, disp=False, M=10, eta_strategy=None, sigma_eps=1e-10, sigma_0=1.0, line_search="cruz", **unknown_options ): r""" Solve nonlinear equation with the DF-SANE method Options ------- ftol : float, optional Relative norm tolerance. fatol : float, optional Absolute norm tolerance. Algorithm terminates when ``||func(x)|| < fatol + ftol ||func(x_0)||``. fnorm : callable, optional Norm to use in the convergence check. If None, 2-norm is used. maxfev : int, optional Maximum number of function evaluations. disp : bool, optional Whether to print convergence process to stdout. eta_strategy : callable, optional Choice of the ``eta_k`` parameter, which gives slack for growth of ``||F||**2``. Called as ``eta_k = eta_strategy(k, x, F)`` with `k` the iteration number, `x` the current iterate and `F` the current residual. Should satisfy ``eta_k > 0`` and ``sum(eta, k=0..inf) < inf``. Default: ``||F||**2 / (1 + k)**2``. sigma_eps : float, optional The spectral coefficient is constrained to ``sigma_eps < sigma < 1/sigma_eps``. Default: 1e-10 sigma_0 : float, optional Initial spectral coefficient. Default: 1.0 M : int, optional Number of iterates to include in the nonmonotonic line search. Default: 10 line_search : {'cruz', 'cheng'} Type of line search to employ. 'cruz' is the original one defined in [Martinez & Raydan. Math. Comp. 75, 1429 (2006)], 'cheng' is a modified search defined in [Cheng & Li. IMA J. Numer. Anal. 29, 814 (2009)]. Default: 'cruz' References ---------- .. [1] "Spectral residual method without gradient information for solving large-scale nonlinear systems of equations." W. La Cruz, J.M. Martinez, M. Raydan. Math. Comp. **75**, 1429 (2006). .. [2] W. La Cruz, Opt. Meth. Software, 29, 24 (2014). .. [3] W. Cheng, D.-H. Li. IMA J. Numer. Anal. **29**, 814 (2009). """ _check_unknown_options(unknown_options) if line_search not in ("cheng", "cruz"): raise ValueError("Invalid value %r for 'line_search'" % (line_search,)) nexp = 2 if eta_strategy is None: # Different choice from [1], as their eta is not invariant # vs. scaling of F. def eta_strategy(k, x, F): # Obtain squared 2-norm of the initial residual from the outer scope return f_0 / (1 + k) ** 2 if fnorm is None: def fnorm(F): # Obtain squared 2-norm of the current residual from the outer scope return f_k ** (1.0 / nexp) def fmerit(F): return np.linalg.norm(F) ** nexp nfev = [0] f, x_k, x_shape, f_k, F_k, is_complex = _wrap_func(func, x0, fmerit, nfev, maxfev, args) k = 0 f_0 = f_k sigma_k = sigma_0 F_0_norm = fnorm(F_k) # For the 'cruz' line search prev_fs = collections.deque([f_k], M) # For the 'cheng' line search Q = 1.0 C = f_0 converged = False message = "too many function evaluations required" while True: F_k_norm = fnorm(F_k) if disp: print("iter %d: ||F|| = %g, sigma = %g" % (k, F_k_norm, sigma_k)) if callback is not None: callback(x_k, F_k) if F_k_norm < ftol * F_0_norm + fatol: # Converged! message = "successful convergence" converged = True break # Control spectral parameter, from [2] if abs(sigma_k) > 1 / sigma_eps: sigma_k = 1 / sigma_eps * np.sign(sigma_k) elif abs(sigma_k) < sigma_eps: sigma_k = sigma_eps # Line search direction d = -sigma_k * F_k # Nonmonotone line search eta = eta_strategy(k, x_k, F_k) try: if line_search == "cruz": alpha, xp, fp, Fp = _nonmonotone_line_search_cruz(f, x_k, d, prev_fs, eta=eta) elif line_search == "cheng": alpha, xp, fp, Fp, C, Q = _nonmonotone_line_search_cheng(f, x_k, d, f_k, C, Q, eta=eta) except _NoConvergence: break # Update spectral parameter s_k = xp - x_k y_k = Fp - F_k sigma_k = np.vdot(s_k, s_k) / np.vdot(s_k, y_k) # Take step x_k = xp F_k = Fp f_k = fp # Store function value if line_search == "cruz": prev_fs.append(fp) k += 1 x = _wrap_result(x_k, is_complex, shape=x_shape) F = _wrap_result(F_k, is_complex) result = OptimizeResult(x=x, success=converged, message=message, fun=F, nfev=nfev[0], nit=k) return result
def _minimize_neldermead(func, x0, args=(), callback=None, xtol=1e-4, ftol=1e-4, maxiter=None, maxfev=None, disp=False, return_all=False, return_simplex=False, **unknown_options): """ Minimization of scalar function of one or more variables using the Nelder-Mead algorithm. Options ------- disp : bool Set to True to print convergence messages. xtol : float Relative error in solution `xopt` acceptable for convergence. ftol : float Relative error in ``fun(xopt)`` acceptable for convergence. maxiter : int Maximum number of iterations to perform. maxfev : int Maximum number of function evaluations to make. return_simplex : bool Set to True to return all nodes of final simplex and their function values. """ _check_unknown_options(unknown_options) maxfun = maxfev retall = return_all fcalls, func = wrap_function(func, args) x0 = asfarray(x0).flatten() N = len(x0) if maxiter is None: maxiter = N * 200 if maxfun is None: maxfun = N * 200 rho = 1 chi = 2 psi = 0.5 sigma = 0.5 one2np1 = list(range(1, N + 1)) sim = numpy.zeros((N + 1, N), dtype=x0.dtype) fsim = numpy.zeros((N + 1,), float) sim[0] = x0 if retall: allvecs = [sim[0]] fsim[0] = func(x0) nonzdelt = 0.05 zdelt = 0.00025 for k in range(0, N): y = numpy.array(x0, copy=True) if y[k] != 0: y[k] = (1 + nonzdelt) * y[k] else: y[k] = zdelt sim[k + 1] = y f = func(y) fsim[k + 1] = f ind = numpy.argsort(fsim) fsim = numpy.take(fsim, ind, 0) # sort so sim[0,:] has the lowest function value sim = numpy.take(sim, ind, 0) iterations = 1 while (fcalls[0] < maxfun and iterations < maxiter): if (numpy.max(numpy.ravel(numpy.abs(sim[1:] - sim[0]))) <= xtol and numpy.max(numpy.abs(fsim[0] - fsim[1:])) <= ftol): break xbar = numpy.add.reduce(sim[:-1], 0) / N xr = (1 + rho) * xbar - rho * sim[-1] fxr = func(xr) doshrink = 0 if fxr < fsim[0]: xe = (1 + rho * chi) * xbar - rho * chi * sim[-1] fxe = func(xe) if fxe < fxr: sim[-1] = xe fsim[-1] = fxe else: sim[-1] = xr fsim[-1] = fxr else: # fsim[0] <= fxr if fxr < fsim[-2]: sim[-1] = xr fsim[-1] = fxr else: # fxr >= fsim[-2] # Perform contraction if fxr < fsim[-1]: xc = (1 + psi * rho) * xbar - psi * rho * sim[-1] fxc = func(xc) if fxc <= fxr: sim[-1] = xc fsim[-1] = fxc else: doshrink = 1 else: # Perform an inside contraction xcc = (1 - psi) * xbar + psi * sim[-1] fxcc = func(xcc) if fxcc < fsim[-1]: sim[-1] = xcc fsim[-1] = fxcc else: doshrink = 1 if doshrink: for j in one2np1: sim[j] = sim[0] + sigma * (sim[j] - sim[0]) fsim[j] = func(sim[j]) ind = numpy.argsort(fsim) sim = numpy.take(sim, ind, 0) fsim = numpy.take(fsim, ind, 0) if callback is not None: callback(sim[0]) iterations += 1 if retall: allvecs.append(sim[0]) x = sim[0] fval = numpy.min(fsim) warnflag = 0 if fcalls[0] >= maxfun: warnflag = 1 msg = _status_message['maxfev'] if disp: print('Warning: ' + msg) elif iterations >= maxiter: warnflag = 2 msg = _status_message['maxiter'] if disp: print('Warning: ' + msg) else: msg = _status_message['success'] if disp: print(msg) print(" Current function value: %f" % fval) print(" Iterations: %d" % iterations) print(" Function evaluations: %d" % fcalls[0]) result = OptimizeResult(fun=fval, nit=iterations, nfev=fcalls[0], status=warnflag, success=(warnflag == 0), message=msg, x=x) if retall: result['allvecs'] = allvecs if return_simplex: result['sim'] = sim result['fsim'] = fsim return result
def _root_hybr(func, x0, args=(), jac=None, col_deriv=0, xtol=1.49012e-08, maxfev=0, band=None, eps=None, factor=100, diag=None, **unknown_options): """ Find the roots of a multivariate function using MINPACK's hybrd and hybrj routines (modified Powell method). Options ------- col_deriv : bool Specify whether the Jacobian function computes derivatives down the columns (faster, because there is no transpose operation). xtol : float The calculation will terminate if the relative error between two consecutive iterates is at most `xtol`. maxfev : int The maximum number of calls to the function. If zero, then ``100*(N+1)`` is the maximum where N is the number of elements in `x0`. band : tuple If set to a two-sequence containing the number of sub- and super-diagonals within the band of the Jacobi matrix, the Jacobi matrix is considered banded (only for ``fprime=None``). eps : float A suitable step length for the forward-difference approximation of the Jacobian (for ``fprime=None``). If `eps` is less than the machine precision, it is assumed that the relative errors in the functions are of the order of the machine precision. factor : float A parameter determining the initial step bound (``factor * || diag * x||``). Should be in the interval ``(0.1, 100)``. diag : sequence N positive entries that serve as a scale factors for the variables. """ _check_unknown_options(unknown_options) epsfcn = eps x0 = asarray(x0).flatten() n = len(x0) if not isinstance(args, tuple): args = (args, ) shape, dtype = _check_func('fsolve', 'func', func, x0, args, n, (n, )) if epsfcn is None: epsfcn = finfo(dtype).eps Dfun = jac if Dfun is None: if band is None: ml, mu = -10, -10 else: ml, mu = band[:2] if maxfev == 0: maxfev = 200 * (n + 1) retval = _minpack._hybrd(func, x0, args, 1, xtol, maxfev, ml, mu, epsfcn, factor, diag) else: _check_func('fsolve', 'fprime', Dfun, x0, args, n, (n, n)) if (maxfev == 0): maxfev = 100 * (n + 1) retval = _minpack._hybrj(func, Dfun, x0, args, 1, col_deriv, xtol, maxfev, factor, diag) x, status = retval[0], retval[-1] errors = { 0: "Improper input parameters were entered.", 1: "The solution converged.", 2: "The number of calls to function has " "reached maxfev = %d." % maxfev, 3: "xtol=%f is too small, no further improvement " "in the approximate\n solution " "is possible." % xtol, 4: "The iteration is not making good progress, as measured " "by the \n improvement from the last five " "Jacobian evaluations.", 5: "The iteration is not making good progress, " "as measured by the \n improvement from the last " "ten iterations.", 'unknown': "An error occurred." } info = retval[1] info['fun'] = info.pop('fvec') sol = OptimizeResult(x=x, success=(status == 1), status=status) sol.update(info) try: sol['message'] = errors[status] except KeyError: sol['message'] = errors['unknown'] return sol
def _minimize(fun, x0, args=(), jac=None, callback=None, gtol=1e-5, fxtol=1e-09, xtol=1e-09, norm=Inf, eps=_epsilon, maxiter=None, disp=False, return_all=False, **unknown_options): _check_unknown_options(unknown_options) f = fun fprime = jac epsilon = eps retall = return_all x0 = asarray(x0).flatten() if x0.ndim == 0: x0.shape = (1,) if maxiter is None: maxiter = len(x0) * 200 func_calls, f = wrap_function(f, args) grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon)) gfk = myfprime(x0) k = 0 N = len(x0) I = numpy.eye(N, dtype=int) Hk = I old_fval = f(x0) old_old_fval = None xk = x0 if retall: allvecs = [x0] sk = [2 * gtol] warnflag = 0 gnorm = vecnorm(gfk, ord=norm) xnorm = np.Inf fx = np.Inf print_lst = [] while (gnorm > gtol) and (xnorm > xtol) and (fx > fxtol) and (k < maxiter): pk = -numpy.dot(Hk, gfk) try: alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ _line_search_wolfe12(f, myfprime, xk, pk, gfk, old_fval, old_old_fval) except _LineSearchError: # search failed to find a better solution. print_lst.append("Przeszukiwanie liniowe zawiodlo lub nie moze osiagnac lepszego rozwiazania") warnflag = 2 break xkp1 = xk + alpha_k * pk fx = np.absolute(old_old_fval - old_fval) xnorm = vecnorm(xkp1 - xk) if retall: allvecs.append(xkp1) sk = xkp1 - xk xk = xkp1 if gfkp1 is None: gfkp1 = myfprime(xkp1) yk = gfkp1 - gfk gfk = gfkp1 if callback is not None: callback(xk) k += 1 if disp: print_ = ('Iter: ' + str(k) + '\n') print_ += ('x: ' + str(xk) + '\n') print_ += ('f(x): ' + str(f(xk)) + '\n') #zmiana na fx print_ +=('gtol: ' + str(gnorm) + '\n') print_ +=('xtol: ' + str(xnorm) + '\n') print_ +=('fxtol: ' + str(fx) + '\n') print_lst.append(print_) gnorm = vecnorm(gfk, ord=norm) if (gnorm <= gtol): break if not numpy.isfinite(old_fval): # We correctly found +-Inf as optimal value, or something went # wrong. print_lst.append("Zlaneziono +-Inf za optymalna wartosc... lub cos poszlo zle.") warnflag = 2 break try: # this was handled in numeric, let it remaines for more safety rhok = 1.0 / (numpy.dot(yk, sk)) except ZeroDivisionError: rhok = 1000.0 if disp: print_lst.append("Dzielenie przez zero!!") if isinf(rhok): # this is patch for numpy rhok = 1000.0 if disp: print_lst.appedn("Dzielenie przez zero!!") A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok Hk = numpy.dot(A1, numpy.dot(Hk, A2)) + (rhok * sk[:, numpy.newaxis] * sk[numpy.newaxis, :]) fval = old_fval if np.isnan(fval): # This can happen if the first call to f returned NaN; # the loop is then never entered. print_lst.append("Osiagnieto Nan w pierwszym wywolaniem algorytmu.") warnflag = 2 if warnflag == 2: msg = _status_message['pr_loss'] if disp: print_ = ("Ostrzezenie: " + msg) print_ += (" Wartosc funkcji celu: %f" % fval) print_ += (" Iteracje: %d" % k) print_ += (" Wywolania funkcji: %d" % func_calls[0]) print_ += (" Wywolania gradientu: %d" % grad_calls[0]) elif k >= maxiter: warnflag = 1 msg = _status_message['maxiter'] if disp: print_ = ("Ostrzerzenie: " + msg) print_ += (" Wartosc funkcji celu: %f" % fval) print_ += (" Iteracje: %d" % k) print_ += (" Wywolania funkcji: %d" % func_calls[0]) print_ += (" Wywolania gradientu: %d" % grad_calls[0]) print_lst.append(print_) else: msg = _status_message['success'] if disp: print_ = (msg + '\n') print_ += (" Wartosc funkcji celu: %f" % fval) print_ += (" Iteracje: %d" % k) print_ += (" Wywolania funkcji: %d" % func_calls[0]) print_ += (" Wywolania gradientu: %d" % grad_calls[0]) print_lst.append(print_) [print(line) for line in print_lst] result = OptimizeResult(fun=fval,lst=print_lst, jac=gfk, hess_inv=Hk, nfev=func_calls[0], njev=grad_calls[0], status=warnflag, success=(warnflag == 0), message=msg, x=xk, nit=k) if retall: result['allvecs'] = allvecs return result
def _minimize_lbfgsb_timeup( fun, x0, args=(), jac=None, bounds=None, disp=None, maxcor=10, ftol=2.2204460492503131e-09, gtol=1e-5, eps=1e-8, maxfun=15000, maxiter=15000, iprint=-1, callback=None, maxls=20, t0=None, timeup=float("inf"), **unknown_options): # JFF: added time-up check """ Minimize a scalar function of one or more variables using the L-BFGS-B algorithm. Options ------- disp : bool Set to True to print convergence messages. maxcor : int The maximum number of variable metric corrections used to define the limited memory matrix. (The limited memory BFGS method does not store the full hessian but uses this many terms in an approximation to it.) factr : float The iteration stops when ``(f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr * eps``, where ``eps`` is the machine precision, which is automatically generated by the code. Typical values for `factr` are: 1e12 for low accuracy; 1e7 for moderate accuracy; 10.0 for extremely high accuracy. ftol : float The iteration stops when ``(f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol``. gtol : float The iteration will stop when ``max{|proj g_i | i = 1, ..., n} <= gtol`` where ``pg_i`` is the i-th component of the projected gradient. eps : float Step size used for numerical approximation of the jacobian. disp : int Set to True to print convergence messages. maxfun : int Maximum number of function evaluations. maxiter : int Maximum number of iterations. maxls : int, optional Maximum number of line search steps (per iteration). Default is 20. """ _check_unknown_options(unknown_options) m = maxcor epsilon = eps pgtol = gtol factr = ftol / np.finfo(float).eps x0 = asarray(x0).ravel() n, = x0.shape if bounds is None: bounds = [(None, None)] * n if len(bounds) != n: raise ValueError('length of x0 != length of bounds') # unbounded variables must use None, not +-inf, for optimizer to work properly bounds = [(None if l == -np.inf else l, None if u == np.inf else u) for l, u in bounds] if disp is not None: if disp == 0: iprint = -1 else: iprint = disp n_function_evals, fun = wrap_function(fun, ()) if jac is None: def func_and_grad(x): f = fun(x, *args) g = _approx_fprime_helper(x, fun, epsilon, args=args, f0=f) return f, g else: def func_and_grad(x): f = fun(x, *args) g = jac(x, *args) return f, g nbd = zeros(n, int32) low_bnd = zeros(n, float64) upper_bnd = zeros(n, float64) bounds_map = {(None, None): 0, (1, None): 1, (1, 1): 2, (None, 1): 3} for i in range(0, n): l, u = bounds[i] if l is not None: low_bnd[i] = l l = 1 if u is not None: upper_bnd[i] = u u = 1 nbd[i] = bounds_map[l, u] if not maxls > 0: raise ValueError('maxls must be positive.') x = array(x0, float64) f = array(0.0, float64) g = zeros((n,), float64) wa = zeros(2*m*n + 5*n + 11*m*m + 8*m, float64) iwa = zeros(3*n, int32) task = zeros(1, 'S60') csave = zeros(1, 'S60') lsave = zeros(4, int32) isave = zeros(44, int32) dsave = zeros(29, float64) task[:] = 'START' n_iterations = 0 if t0 is None: t0 = time.time() time_profile.predicted_inner_loop_func2_duration = 0.0 while 1: # x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \ _lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr, pgtol, wa, iwa, task, iprint, csave, lsave, isave, dsave, maxls) task_str = task.tostring() # begin EB curr_time = time.time() predicted_inner_loop_func2_duration = (curr_time + time_profile.maximize_inner_time_profile.mean + time_profile.func2_time_profile.mean - t0) if predicted_inner_loop_func2_duration > timeup: # JFF: added time-up check task[:] = ('STOP: PREDICTED COMPUTATION TIME EXCEEDS LIMIT') break # end EB if task_str.startswith(b'FG'): # The minimization routine wants f and g at the current x. # Note that interruptions due to maxfun are postponed # until the completion of the current minimization iteration. # Overwrite f and g: f, g = func_and_grad(x) elif task_str.startswith(b'NEW_X'): # new iteration if n_iterations > maxiter: task[:] = 'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT' elif n_function_evals[0] > maxfun: task[:] = ('STOP: TOTAL NO. of f AND g EVALUATIONS ' 'EXCEEDS LIMIT') else: n_iterations += 1 if callback is not None: callback(x) else: break time_profile.predicted_inner_loop_func2_duration = predicted_inner_loop_func2_duration task_str = task.tostring().strip(b'\x00').strip() if task_str.startswith(b'CONV'): warnflag = 0 elif n_function_evals[0] > maxfun: warnflag = 1 elif n_iterations > maxiter: warnflag = 1 else: warnflag = 2 # These two portions of the workspace are described in the mainlb # subroutine in lbfgsb.f. See line 363. s = wa[0: m*n].reshape(m, n) y = wa[m*n: 2*m*n].reshape(m, n) # See lbfgsb.f line 160 for this portion of the workspace. # isave(31) = the total number of BFGS updates prior the current iteration; n_bfgs_updates = isave[30] n_corrs = min(n_bfgs_updates, maxcor) hess_inv = LbfgsInvHessProduct(s[:n_corrs], y[:n_corrs]) return OptimizeResult(fun=f, jac=g, nfev=n_function_evals[0], nit=n_iterations, status=warnflag, message=task_str, x=x, success=(warnflag == 0), hess_inv=hess_inv)
def _minimize_cg(fun, x0, args=(), jac=None, callback=None, gtol=1e-5, norm=Inf, eps=_epsilon, maxiter=None, disp=False, return_all=False, xtol= 1e-6, **unknown_options): """ Minimization of scalar function of one or more variables using the conjugate gradient algorithm. Options for the conjugate gradient algorithm are: disp : bool Set to True to print convergence messages. maxiter : int Maximum number of iterations to perform. gtol : float Gradient norm must be less than `gtol` before successful termination. norm : float Order of norm (Inf is max, -Inf is min). eps : float or ndarray If `jac` is approximated, use this value for the step size. This function is called by the `minimize` function with `method=CG`. It is not supposed to be called directly. """ _check_unknown_options(unknown_options) f = fun fprime = jac epsilon = eps retall = return_all x0 = asarray(x0).flatten() if maxiter is None: maxiter = len(x0) * 200 func_calls, f = wrap_function(f, args) if fprime is None: grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon)) else: grad_calls, myfprime = wrap_function(fprime, args) gfk = myfprime(x0) k = 0 xk = x0 old_fval = f(xk) old_old_fval = None if retall: allvecs = [xk] warnflag = 0 pk = -gfk gnorm = vecnorm(gfk, ord=norm) while (gnorm > gtol) and (k < maxiter): deltak = numpy.dot(gfk, gfk) try: alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ _line_search_wolfe12(f, myfprime, xk, pk, gfk, old_fval, old_old_fval, c2=0.4, xtol=xtol) except _LineSearchError: # Line search failed to find a better solution. warnflag = 2 break xk = xk + alpha_k * pk if retall: allvecs.append(xk) if gfkp1 is None: gfkp1 = myfprime(xk) yk = gfkp1 - gfk beta_k = max(0, numpy.dot(yk, gfkp1) / deltak) pk = -gfkp1 + beta_k * pk gfk = gfkp1 gnorm = vecnorm(gfk, ord=norm) if callback is not None: callback(xk) k += 1 fval = old_fval if warnflag == 2: msg = _status_message['pr_loss'] if disp: print("Warning: " + msg) print(" Current function value: %f" % fval) print(" Iterations: %d" % k) print(" Function evaluations: %d" % func_calls[0]) print(" Gradient evaluations: %d" % grad_calls[0]) elif k >= maxiter: warnflag = 1 msg = _status_message['maxiter'] if disp: print("Warning: " + msg) print(" Current function value: %f" % fval) print(" Iterations: %d" % k) print(" Function evaluations: %d" % func_calls[0]) print(" Gradient evaluations: %d" % grad_calls[0]) else: msg = _status_message['success'] if disp: print(msg) print(" Current function value: %f" % fval) print(" Iterations: %d" % k) print(" Function evaluations: %d" % func_calls[0]) print(" Gradient evaluations: %d" % grad_calls[0]) result = OptimizeResult(fun=fval, jac=gfk, nfev=func_calls[0], njev=grad_calls[0], status=warnflag, success=(warnflag == 0), message=msg, x=xk) if retall: result['allvecs'] = allvecs return result
def _minimize_lbfgsb_multi(fun, x0, args=(), jac=None, bounds=None, disp=None, maxcor=10, ftol=2.2204460492503131e-09, gtol=1e-5, eps=1e-8, maxfun=15000, maxiter=15000, iprint=-1, callback=None, maxls=20, **unknown_options): """ Minimize a scalar function of one or more variables using the L-BFGS-B algorithm. Options ------- disp : bool Set to True to print convergence messages. maxcor : int The maximum number of variable metric corrections used to define the limited memory matrix. (The limited memory BFGS method does not store the full hessian but uses this many terms in an approximation to it.) ftol : float The iteration stops when ``(f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol``. gtol : float The iteration will stop when ``max{|proj g_i | i = 1, ..., n} <= gtol`` where ``pg_i`` is the i-th component of the projected gradient. eps : float Step size used for numerical approximation of the jacobian. disp : int Set to True to print convergence messages. maxfun : int Maximum number of function evaluations. maxiter : int Maximum number of iterations. maxls : int, optional Maximum number of line search steps (per iteration). Default is 20. Notes ----- The option `ftol` is exposed via the `scipy.optimize.minimize` interface, but calling `scipy.optimize.fmin_l_bfgs_b` directly exposes `factr`. The relationship between the two is ``ftol = factr * numpy.finfo(float).eps``. I.e., `factr` multiplies the default machine floating-point precision to arrive at `ftol`. """ _check_unknown_options(unknown_options) m = maxcor epsilon = eps pgtol = gtol factr = ftol / np.finfo(float).eps k,n = x0.shape # x0 = [asarray(x).ravel() for x in x0] # n, = x0.shape if bounds is None: bounds = [(None, None)] * n if len(bounds) != n: raise ValueError('length of x0 != length of bounds') # unbounded variables must use None, not +-inf, for optimizer to work properly bounds = [(None if l == -np.inf else l, None if u == np.inf else u) for l, u in bounds] if disp is not None: if disp == 0: iprint = -1 else: iprint = disp if jac is None: def func_and_grad(x): # f = fun(x, *args) f, g = _approx_fprime_helper(x, fun, epsilon, args=args) return f, g else: def func_and_grad(x): f = fun(x, *args) g = jac(x, *args) return f, g nbd = zeros(n, int32) low_bnd = zeros(n, float64) upper_bnd = zeros(n, float64) bounds_map = {(None, None): 0, (1, None): 1, (1, 1): 2, (None, 1): 3} for i in range(0, n): l, u = bounds[i] if l is not None: low_bnd[i] = l l = 1 if u is not None: upper_bnd[i] = u u = 1 nbd[i] = bounds_map[l, u] if not maxls > 0: raise ValueError('maxls must be positive.') # x = [array(x0_, float64) for x0_ in x0] X = x0.copy() f = [array(0.0, float64) for _ in range(k)] g = [zeros((n,), float64) for _ in range(k)] wa = [zeros(2*m*n + 5*n + 11*m*m + 8*m, float64) for _ in range(k)] iwa = [zeros(3*n, int32) for _ in range(k)] task = [zeros(1, 'S60') for _ in range(k)] csave = [zeros(1, 'S60') for _ in range(k)] lsave = [zeros(4, int32) for _ in range(k)] isave = [zeros(44, int32) for _ in range(k)] dsave = [zeros(29, float64) for _ in range(k)] n_function_evals = 0 for i in range(k): task[i][:] = 'START' n_iterations = [0 for _ in range(k)] k_running = [i for i in range(k)] while 1: # x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \ request_index = [] # run all instance until they request a new point # X contains only points for the remaining instances for i in k_running: while 1: _lbfgsb.setulb(m, X[i], low_bnd, upper_bnd, nbd, f[i], g[i], factr, pgtol, wa[i], iwa[i], task[i], iprint, csave[i], lsave[i], isave[i], dsave[i], maxls) task_str = task[i].tostring() if task_str.startswith(b'FG'): # The minimization routine wants f and g at the current x. # Note that interruptions due to maxfun are postponed # until the completion of the current minimization iteration. # Overwrite f and g: request_index.append(i) break elif task_str.startswith(b'NEW_X'): # new iteration if n_iterations[i] > maxiter: task[i][:] = 'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT' # break elif n_function_evals > maxfun: task[i][:] = ('STOP: TOTAL NO. of f AND g EVALUATIONS ' 'EXCEEDS LIMIT') # break else: n_iterations[i] += 1 # if callback is not None: # callback(x) else: break k_running = request_index if len(k_running) == 0: break F,G = func_and_grad(X[request_index]) n_function_evals += 1 for ff,gg,i in zip(F,G,k_running): f[i] = ff g[i] = gg warnflag = [] task_str = [t.tostring().strip(b'\x00').strip() for t in task] for i,t in enumerate(task_str): if t.startswith(b'CONV'): warnflag.append(0) elif n_function_evals > maxfun: warnflag.append(1) elif n_iterations[i] > maxiter: warnflag.append(2) else: warnflag.append(3) # These two portions of the workspace are described in the mainlb # subroutine in lbfgsb.f. See line 363. # s = [wa[i][0: m*n].reshape(m, n) for i in range(k)] # y = [wa[i][m*n: 2*m*n].reshape(m, n) for i in range(k)] # See lbfgsb.f line 160 for this portion of the workspace. # isave(31) = the total number of BFGS updates prior the current iteration; # n_bfgs_updates = isave[30] # n_corrs = min(n_bfgs_updates, maxcor) # hess_inv = LbfgsInvHessProduct(s[:n_corrs], y[:n_corrs]) return OptimizeResult(fun=f, jac=g, nfev=n_function_evals, nit=n_iterations, status=warnflag, message=task_str, x=X, success=(sum(warnflag) == 0))
def _minimize_lbfgsb( fun, x0, bounds=None, args=(), kwargs={}, jac=None, callback=None, tol={"abs": 1e-05, "rel": 1e-08}, norm=np.Inf, maxiter=None, disp=False, return_all=False, **unknown_options ): """ Minimization of scalar function of one or more variables using the BHHH algorithm. Options ------- disp : bool Set to True to print convergence messages. maxiter : int Maximum number of iterations to perform. tol : dict Absolute and relative tolerance values. norm : float Order of norm (Inf is max, -Inf is min). """ _check_unknown_options(unknown_options) def f(x0): return fun(x0, *args, **kwargs) fprime = jac # epsilon = eps Add functionality retall = return_all k = 0 ns = 0 nsmax = 5 N = len(x0) x0 = np.asarray(x0).flatten() if x0.ndim == 0: x0.shape = (1,) if bounds is None: bounds = np.array([np.inf] * N * 2).reshape((2, N)) bounds[0, :] = -bounds[0, :] if bounds.shape[1] != N: raise ValueError("length of x0 != length of bounds") low = bounds[0, :] up = bounds[1, :] x0 = np.clip(x0, low, up) if maxiter is None: maxiter = len(x0) * 200 if not callable(fprime): def myfprime(x0): return approx_derivative(f, x0, args=args, kwargs=kwargs) else: myfprime = fprime # Setup for iteration old_fval = f(x0) gf0 = myfprime(x0) gfk = gf0 norm_pg0 = vecnorm(x0 - np.clip(x0 - gf0, low, up), ord=norm) xk = x0 norm_pgk = norm_pg0 sstore = np.zeros((maxiter, N)) ystore = sstore.copy() if retall: allvecs = [x0] warnflag = 0 # Calculate indices ofactive and inative set using projected gradient epsilon = min(np.min(up - low) / 2, norm_pgk) activeset = np.logical_or(xk - low <= epsilon, up - xk <= epsilon) inactiveset = np.logical_not(activeset) for _ in range(maxiter): # for loop instead. # Check tolerance of gradient norm if norm_pgk <= tol["abs"] + tol["rel"] * norm_pg0: break pk = -gfk pk = bfgsrecb(ns, sstore, ystore, pk, activeset) gfk_active = gfk.copy() gfk_active[inactiveset] = 0 pk = -gfk_active + pk # Sets the initial step guess to dx ~ 1 old_old_fval = old_fval + np.linalg.norm(gfk) / 2 try: alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = _line_search_wolfe12( f, myfprime, xk, pk, gfk, old_fval, old_old_fval, amin=1e-100, amax=1e100, ) except _LineSearchError: # Line search failed to find a better solution. warnflag = 2 break xkp1 = np.clip(xk + alpha_k * pk, low, up) if retall: allvecs.append(xkp1) yk = myfprime(xkp1) - gfk sk = xkp1 - xk xk = xkp1 gfk = myfprime(xkp1) norm_pgk = vecnorm(xk - np.clip(xk - gfk, low, up), ord=norm) # Calculate indices ofactive and inative set using projected gradient epsilon = min(np.min(up - low) / 2, norm_pgk) activeset = np.logical_or(xk - low <= epsilon, up - xk <= epsilon) inactiveset = np.logical_not(activeset) yk[activeset] = 0 sk[activeset] = 0 # reset storage ytsk = yk.dot(sk) if ytsk <= 0: ns = 0 if ns == nsmax: print("ns reached maximum size") ns = 0 elif ytsk > 0: ns += 1 alpha0 = ytsk ** 0.5 sstore[ns - 1, :] = sk / alpha0 ystore[ns - 1, :] = yk / alpha0 k += 1 if callback is not None: callback(xk) if np.isinf(old_fval): # We correctly found +-Inf as optimal value, or something went # wrong. warnflag = 2 break if np.isnan(xk).any(): warnflag = 3 break fval = old_fval if warnflag == 2: msg = _status_message["pr_loss"] elif k >= maxiter: warnflag = 1 msg = _status_message["maxiter"] elif np.isnan(fval) or np.isnan(xk).any(): warnflag = 3 msg = _status_message["nan"] else: msg = _status_message["success"] if disp: print("{}{}".format("Warning: " if warnflag != 0 else "", msg)) print(" Current function value: %f" % fval) print(" Iterations: %d" % k) result = OptimizeResult( fun=fval, jac=gfk, status=warnflag, success=(warnflag == 0), message=msg, x=xk, nit=k, ) if retall: result["allvecs"] = allvecs return result
def _linprog_simplex(c, A_ub=None, b_ub=None, A_eq=None, b_eq=None, bounds=None, maxiter=1000, disp=False, callback=None, tol=1.0E-12, bland=False, **unknown_options): """ Solve the following linear programming problem via a two-phase simplex algorithm. maximize: c^T * x subject to: A_ub * x <= b_ub A_eq * x == b_eq Parameters ---------- c : array_like Coefficients of the linear objective function to be maximized. A_ub : 2-D array which, when matrix-multiplied by x, gives the values of the upper-bound inequality constraints at x. b_ub : array_like 1-D array of values representing the upper-bound of each inequality constraint (row) in A_ub. A_eq : array_like 2-D array which, when matrix-multiplied by x, gives the values of the equality constraints at x. b_eq : array_like 1-D array of values representing the RHS of each equality constraint (row) in A_eq. bounds : array_like The bounds for each independent variable in the solution, which can take one of three forms:: None : The default bounds, all variables are non-negative. (lb, ub) : If a 2-element sequence is provided, the same lower bound (lb) and upper bound (ub) will be applied to all variables. [(lb_0, ub_0), (lb_1, ub_1), ...] : If an n x 2 sequence is provided, each variable x_i will be bounded by lb[i] and ub[i]. Infinite bounds are specified using -np.inf (negative) or np.inf (positive). maxiter : int The maximum number of iterations to perform. disp : bool If True, print exit status message to sys.stdout callback : callable If a callback function is provide, it will be called within each iteration of the simplex algorithm. The callback must have the signature `callback(xk, **kwargs)` where xk is the current solution vector and kwargs is a dictionary containing the following:: "tableau" : The current Simplex algorithm tableau "nit" : The current iteration. "pivot" : The pivot (row, column) used for the next iteration. "phase" : Whether the algorithm is in Phase 1 or Phase 2. "bv" : A structured array containing a string representation of each basic variable and its current value. tol : float The tolerance which determines when a solution is "close enough" to zero in Phase 1 to be considered a basic feasible solution or close enough to positive to to serve as an optimal solution. bland : bool If True, use Bland's anti-cycling rule [3] to choose pivots to prevent cycling. If False, choose pivots which should lead to a converged solution more quickly. The latter method is subject to cycling (non-convergence) in rare instances. Returns ------- A scipy.optimize.OptimizeResult consisting of the following fields:: x : ndarray The independent variable vector which optimizes the linear programming problem. slack : ndarray The values of the slack variables. Each slack variable corresponds to an inequality constraint. If the slack is zero, then the corresponding constraint is active. success : bool Returns True if the algorithm succeeded in finding an optimal solution. status : int An integer representing the exit status of the optimization:: 0 : Optimization terminated successfully 1 : Iteration limit reached 2 : Problem appears to be infeasible 3 : Problem appears to be unbounded nit : int The number of iterations performed. message : str A string descriptor of the exit status of the optimization. Examples -------- Consider the following problem: Minimize: f = -1*x[0] + 4*x[1] Subject to: -3*x[0] + 1*x[1] <= 6 1*x[0] + 2*x[1] <= 4 x[1] >= -3 where: -inf <= x[0] <= inf This problem deviates from the standard linear programming problem. In standard form, linear programming problems assume the variables x are non-negative. Since the variables don't have standard bounds where 0 <= x <= inf, the bounds of the variables must be explicitly set. There are two upper-bound constraints, which can be expressed as dot(A_ub, x) <= b_ub The input for this problem is as follows: >>> c = [-1, 4] >>> A = [[-3, 1], [1, 2]] >>> b = [6, 4] >>> x0_bnds = (None, None) >>> x1_bnds = (-3, None) >>> res = linprog(c, A, b, bounds=(x0_bnds, x1_bnds), options={"disp":True}) >>> print(res) Optimization terminated successfully. Current function value: 11.428571 Iterations: 2 status: 0 success: True fun: 11.428571428571429 x: array([-1.14285714, 2.57142857]) slack: array([], dtype=np.float64) message: 'Optimization terminated successfully.' nit: 2 References ---------- .. [1] Dantzig, George B., Linear programming and extensions. Rand Corporation Research Study Princeton Univ. Press, Princeton, NJ, 1963 .. [2] Hillier, S.H. and Lieberman, G.J. (1995), "Introduction to Mathematical Programming", McGraw-Hill, Chapter 4. .. [3] Bland, Robert G. New finite pivoting rules for the simplex method. Mathematics of Operations Research (2), 1977: pp. 103-107. """ _check_unknown_options(unknown_options) status = 0 messages = {0: "Optimization terminated successfully.", 1: "Iteration limit reached.", 2: "Optimzation failed. Unable to find a feasible" " starting point.", 3: "Optimization failed. The problem appears to be unbounded.", 4: "Optimization failed. Singular matrix encountered."} have_floor_variable = False cc = np.asarray(c) # The initial value of the objective function element in the tableau f0 = 0 # The number of variables as given by c n = len(c) # Convert the input arguments to arrays (sized to zero if not provided) Aeq = np.asarray(A_eq) if A_eq is not None else np.empty([0, len(cc)]) Aub = np.asarray(A_ub) if A_ub is not None else np.empty([0, len(cc)]) beq = np.ravel(np.asarray(b_eq)) if b_eq is not None else np.empty([0]) bub = np.ravel(np.asarray(b_ub)) if b_ub is not None else np.empty([0]) # Analyze the bounds and determine what modifications to me made to # the constraints in order to accommodate them. L = np.zeros(n, dtype=np.float64) U = np.ones(n, dtype=np.float64) * np.inf if bounds is None or len(bounds) == 0: pass elif len(bounds) == 2 and not hasattr(bounds[0], '__len__'): # All bounds are the same L = np.asarray(n * [bounds[0]], dtype=np.float64) U = np.asarray(n * [bounds[1]], dtype=np.float64) else: if len(bounds) != n: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Length of bounds is inconsistent with the length of c") else: try: for i in range(n): if len(bounds[i]) != 2: raise IndexError() L[i] = bounds[i][0] if bounds[i][0] is not None else -np.inf U[i] = bounds[i][1] if bounds[i][1] is not None else np.inf except IndexError: status = -1 message = ("Invalid input for linprog with " "method = 'simplex'. bounds must be a n x 2 " "sequence/array where n = len(c).") if np.any(L == -np.inf): # If any lower-bound constraint is a free variable # add the first column variable as the "floor" variable which # accommodates the most negative variable in the problem. n = n + 1 L = np.concatenate([np.array([0]), L]) U = np.concatenate([np.array([np.inf]), U]) cc = np.concatenate([np.array([0]), cc]) Aeq = np.hstack([np.zeros([Aeq.shape[0], 1]), Aeq]) Aub = np.hstack([np.zeros([Aub.shape[0], 1]), Aub]) have_floor_variable = True # Now before we deal with any variables with lower bounds < 0, # deal with finite bounds which can be simply added as new constraints. # Also validate bounds inputs here. for i in range(n): if(L[i] > U[i]): status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Lower bound %d is greater than upper bound %d" % (i, i)) if np.isinf(L[i]) and L[i] > 0: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Lower bound may not be +infinity") if np.isinf(U[i]) and U[i] < 0: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Upper bound may not be -infinity") if np.isfinite(L[i]) and L[i] > 0: # Add a new lower-bound (negative upper-bound) constraint Aub = np.vstack([Aub, np.zeros(n)]) Aub[-1, i] = -1 bub = np.concatenate([bub, np.array([-L[i]])]) L[i] = 0 if np.isfinite(U[i]): # Add a new upper-bound constraint Aub = np.vstack([Aub, np.zeros(n)]) Aub[-1, i] = 1 bub = np.concatenate([bub, np.array([U[i]])]) U[i] = np.inf # Now find negative lower bounds (finite or infinite) which require a # change of variables or free variables and handle them appropriately for i in range(0, n): if L[i] < 0: if np.isfinite(L[i]) and L[i] < 0: # Add a change of variables for x[i] # For each row in the constraint matrices, we take the # coefficient from column i in A, # and subtract the product of that and L[i] to the RHS b beq[:] = beq[:] - Aeq[:, i] * L[i] bub[:] = bub[:] - Aub[:, i] * L[i] # We now have a nonzero initial value for the objective # function as well. f0 = f0 - cc[i] * L[i] else: # This is an unrestricted variable, let x[i] = u[i] - v[0] # where v is the first column in all matrices. Aeq[:, 0] = Aeq[:, 0] - Aeq[:, i] Aub[:, 0] = Aub[:, 0] - Aub[:, i] cc[0] = cc[0] - cc[i] if np.isinf(U[i]): if U[i] < 0: status = -1 message = ("Invalid input for linprog with " "method = 'simplex'. Upper bound may not be -inf.") # The number of upper bound constraints (rows in A_ub and elements in b_ub) mub = len(bub) # The number of equality constraints (rows in A_eq and elements in b_eq) meq = len(beq) # The total number of constraints m = mub + meq # The number of slack variables (one for each of the upper-bound constraints) n_slack = mub # The number of artificial variables (one for each lower-bound and equality # constraint) n_artificial = meq + _count_nonzero(bub < 0) try: Aub_rows, Aub_cols = Aub.shape except ValueError: raise ValueError("Invalid input. A_ub must be two-dimensional") try: Aeq_rows, Aeq_cols = Aeq.shape except ValueError: raise ValueError("Invalid input. A_eq must be two-dimensional") if Aeq_rows != meq: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "The number of rows in A_eq must be equal " "to the number of values in b_eq") if Aub_rows != mub: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "The number of rows in A_ub must be equal " "to the number of values in b_ub") if Aeq_cols > 0 and Aeq_cols != n: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Number of columns in A_eq must be equal " "to the size of c") if Aub_cols > 0 and Aub_cols != n: status = -1 message = ("Invalid input for linprog with method = 'simplex'. " "Number of columns in A_ub must be equal to the size of c") if status != 0: # Invalid inputs provided raise ValueError(message) # Create the tableau T = np.zeros([m + 2, n + n_slack + n_artificial + 1]) # Insert objective into tableau T[-2, :n] = cc T[-2, -1] = f0 b = T[:-2, -1] if meq > 0: # Add Aeq to the tableau T[:meq, :n] = Aeq # Add beq to the tableau b[:meq] = beq if mub > 0: # Add Aub to the tableau T[meq:meq + mub, :n] = Aub # At bub to the tableau b[meq:meq + mub] = bub # Add the slack variables to the tableau np.fill_diagonal(T[meq:m, n:n + n_slack], 1) # Further setup the tableau # If a row corresponds to an equality constraint or a negative b (a lower # bound constraint), then an artificial variable is added for that row. # Also, if b is negative, first flip the signs in that constraint. slcount = 0 avcount = 0 basis = np.zeros(m, dtype=int) r_artificial = np.zeros(n_artificial, dtype=int) for i in range(m): if i < meq or b[i] < 0: # basic variable i is in column n+n_slack+avcount basis[i] = n + n_slack + avcount r_artificial[avcount] = i avcount += 1 if b[i] < 0: b[i] *= -1 T[i, :-1] *= -1 T[i, basis[i]] = 1 T[-1, basis[i]] = 1 else: # basic variable i is in column n+slcount basis[i] = n + slcount slcount += 1 # Make the artificial variables basic feasible variables by subtracting # each row with an artificial variable from the Phase 1 objective for r in r_artificial: T[-1, :] = T[-1, :] - T[r, :] nit1, status = _solve_simplex(T, n, basis, phase=1, callback=callback, maxiter=maxiter, tol=tol, bland=bland) # if pseudo objective is zero, remove the last row from the tableau and # proceed to phase 2 if abs(T[-1, -1]) < tol: # Remove the pseudo-objective row from the tableau T = T[:-1, :] # Remove the artificial variable columns from the tableau T = np.delete(T, np.s_[n + n_slack:n + n_slack + n_artificial], 1) else: # Failure to find a feasible starting point status = 2 if status != 0: message = messages[status] if disp: print(message) return OptimizeResult(x=np.nan, fun=-T[-1, -1], nit=nit1, status=status, message=message, success=False) # Phase 2 nit2, status = _solve_simplex(T, n, basis, maxiter=maxiter - nit1, phase=2, callback=callback, tol=tol, nit0=nit1, bland=bland) solution = np.zeros(n + n_slack + n_artificial) solution[basis[:m]] = T[:m, -1] x = solution[:n] slack = solution[n:n + n_slack] # For those variables with finite negative lower bounds, # reverse the change of variables masked_L = np.ma.array(L, mask=np.isinf(L), fill_value=0.0).filled() x = x + masked_L # For those variables with infinite negative lower bounds, # take x[i] as the difference between x[i] and the floor variable. if have_floor_variable: for i in range(1, n): if np.isinf(L[i]): x[i] -= x[0] x = x[1:] # Optimization complete at this point obj = -T[-1, -1] if status in (0, 1): if disp: print(messages[status]) print(" Current function value: {: <12.6f}".format(obj)) print(" Iterations: {:d}".format(nit2)) else: if disp: print(messages[status]) print(" Iterations: {:d}".format(nit2)) return OptimizeResult(x=x, fun=obj, nit=int(nit2), status=status, slack=slack, message=messages[status], success=(status == 0))
def _minimize_bhhh(fun, x0, bounds=None, args=(), jac=None, callback=None, tol={ "abs": 1e-05, "rel": 1e-08 }, norm=np.Inf, maxiter=None, disp=False, return_all=False, **unknown_options): """ Minimization of scalar function of one or more variables using the BHHH algorithm. Options ------- disp : bool Set to True to print convergence messages. maxiter : int Maximum number of iterations to perform. tol : dict Absolute and relative tolerance values. norm : float Order of norm (Inf is max, -Inf is min). """ _check_unknown_options(unknown_options) f = fun fprime = jac retall = return_all k = 0 N = len(x0) x0 = np.asarray(x0).flatten() if x0.ndim == 0: x0.shape = (1, ) if bounds is None: bounds = np.array([np.inf] * N * 2).reshape((2, N)) bounds[0, :] = -bounds[0, :] if bounds.shape[1] != N: raise ValueError("length of x0 != length of bounds") low = bounds[0, :] up = bounds[1, :] x0 = np.clip(x0, low, up) if maxiter is None: maxiter = len(x0) * 200 # Need the aggregate functions to take only x0 as an argument func_calls, agg_fun = wrap_function_agg(f, args) if not callable(fprime): grad_calls, myfprime = wrap_function_num_dev(f, args) else: grad_calls, myfprime = wrap_function(fprime, args) def agg_fprime(x0): return myfprime(x0).sum(axis=0) # Setup for iteration old_fval = agg_fun(x0) gf0 = agg_fprime(x0) norm_pg0 = vecnorm(x0 - np.clip(x0 - gf0, low, up), ord=norm) xk = x0 norm_pgk = norm_pg0 if retall: allvecs = [x0] warnflag = 0 for _ in range(maxiter): # for loop instead. # Individual gfk_obs = myfprime(xk) # Aggregate fprime. Might replace by simply summing up gfk_obs gfk = gfk_obs.sum(axis=0) norm_pgk = vecnorm(xk - np.clip(xk - gfk, low, up), ord=norm) # Check tolerance of gradient norm if norm_pgk <= tol["abs"] + tol["rel"] * norm_pg0: break # Sets the initial step guess to dx ~ 1 old_old_fval = old_fval + np.linalg.norm(gfk) / 2 # Calculate BHHH hessian and step Hk = np.dot(gfk_obs.T, gfk_obs) Bk = np.linalg.inv(Hk) pk = np.empty(N) pk = -np.dot(Bk, gfk) try: alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = _line_search_wolfe12( agg_fun, agg_fprime, xk, pk, gfk, old_fval, old_old_fval, amin=1e-100, amax=1e100, ) except _LineSearchError: # Line search failed to find a better solution. warnflag = 2 break xkp1 = np.clip(xk + alpha_k * pk, low, up) if retall: allvecs.append(xkp1) xk = xkp1 if callback is not None: callback(xk) k += 1 if np.isinf(old_fval): # We correctly found +-Inf as optimal value, or something went # wrong. warnflag = 2 break fval = old_fval if warnflag == 2: msg = _status_message["pr_loss"] elif k >= maxiter: warnflag = 1 msg = _status_message["maxiter"] elif np.isnan(fval) or np.isnan(xk).any(): warnflag = 3 msg = _status_message["nan"] else: msg = _status_message["success"] if disp: print("{}{}".format("Warning: " if warnflag != 0 else "", msg)) print(" Current function value: %f" % fval) print(" Iterations: %d" % k) result = OptimizeResult( fun=fval, jac=gfk, hess_inv=Bk, nfev=func_calls[0], njev=grad_calls[0], status=warnflag, success=(warnflag == 0), message=msg, x=xk, nit=k, ) if retall: result["allvecs"] = allvecs return result