def reverse_info(ds):
dsn = []
for dp in ds:
dsn.append(
OptimizeResult(f=np.flipud(dp.f),
x=np.flipud(dp.x),
g=np.flipud(dp.g)))
return dsn
def optimize_objective_on_Line_segment(self, start_point, end_point, iteration, blended_point, previous_end,
ratio=0.5, limit=1000):
f = self.objective
if self.blend and not np.allclose(blended_point, end_point):
start_point = blended_point
vertex_ratio = self.vertex_ratio
if vertex_ratio is not None and previous_end is not None:
# mix the current and previous end points
mixed_end = end_point + vertex_ratio * (previous_end - end_point)
if not np.allclose(mixed_end, start_point) and f(mixed_end) < f(end_point):
end_point = mixed_end
left = start_point
right = end_point
f_left = f(left)
f_right = f(right)
count = 0
done = False
convex_ok = True
while convex_ok and not done:
count += 1
middle = left + ratio * (right - left)
f_middle = f(middle)
if f_middle <= f_right:
right = middle
f_right = f_middle
elif f_middle > f_left:
convex_ok = False
done = np.allclose(left, right) or f_right < f_left
if count > limit:
break
if not done and self.scalar_fallback:
return self.optimize_objective_on_Line_segment0(start_point, end_point, iteration, blended_point, previous_end)
res = OptimizeResult()
res.success = done
res.start = start_point
res.end = end_point
res.x = right
if not done:
if not convex_ok:
res.message = "objective is not convex on line segment"
else:
res.message = "limit reached seeking improvement on line segment"
else:
res.message = "improvement found"
return (right, res, done)
def _minimize_neldermead(func,
x0,
args=(),
callback=None,
xtol=1e-4,
ftol=1e-4,
maxiter=None,
maxfev=None,
disp=False,
return_all=False,
**unknown_options):
"""
Minimization of scalar function of one or more variables using the
Nelder-Mead algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
xtol : float
Relative error in solution `xopt` acceptable for convergence.
ftol : float
Relative error in ``fun(xopt)`` acceptable for convergence.
maxiter : int
Maximum number of iterations to perform.
maxfev : int
Maximum number of function evaluations to make.
"""
# _check_unknown_options(unknown_options)
maxfun = maxfev
retall = return_all
fcalls, func = wrap_function(func, args)
x0 = asfarray(x0).flatten()
N = len(x0)
if maxiter is None:
maxiter = N * 200
if maxfun is None:
maxfun = N * 200
rho = 1
chi = 2
psi = 0.5
sigma = 0.5
one2np1 = list(range(1, N + 1))
sim = numpy.zeros((N + 1, N), dtype=x0.dtype)
fsim = numpy.zeros((N + 1, ), float)
sim[0] = x0
if retall:
allvecs = [sim[0]]
fsim[0] = func(x0)
nonzdelt = 0.05
zdelt = 0.00025
for k in range(0, N):
y = numpy.array(x0, copy=True)
if y[k] != 0:
y[k] = (1 + nonzdelt) * y[k]
else:
y[k] = zdelt
sim[k + 1] = y
f = func(y)
fsim[k + 1] = f
ind = numpy.argsort(fsim)
fsim = numpy.take(fsim, ind, 0)
# sort so sim[0,:] has the lowest function value
sim = numpy.take(sim, ind, 0)
iterations = 1
while (fcalls[0] < maxfun and iterations < maxiter):
if (numpy.max(numpy.ravel(numpy.abs(sim[1:] - sim[0]))) <= xtol
and numpy.max(numpy.abs(fsim[0] - fsim[1:])) <= ftol):
break
xbar = numpy.add.reduce(sim[:-1], 0) / N
xr = (1 + rho) * xbar - rho * sim[-1]
fxr = func(xr)
doshrink = 0
if fxr < fsim[0]:
xe = (1 + rho * chi) * xbar - rho * chi * sim[-1]
fxe = func(xe)
if fxe < fxr:
sim[-1] = xe
fsim[-1] = fxe
else:
sim[-1] = xr
fsim[-1] = fxr
else: # fsim[0] <= fxr
if fxr < fsim[-2]:
sim[-1] = xr
fsim[-1] = fxr
else: # fxr >= fsim[-2]
# Perform contraction
if fxr < fsim[-1]:
xc = (1 + psi * rho) * xbar - psi * rho * sim[-1]
fxc = func(xc)
if fxc <= fxr:
sim[-1] = xc
fsim[-1] = fxc
else:
doshrink = 1
else:
# Perform an inside contraction
xcc = (1 - psi) * xbar + psi * sim[-1]
fxcc = func(xcc)
if fxcc < fsim[-1]:
sim[-1] = xcc
fsim[-1] = fxcc
else:
doshrink = 1
if doshrink:
for j in one2np1:
sim[j] = sim[0] + sigma * (sim[j] - sim[0])
fsim[j] = func(sim[j])
ind = numpy.argsort(fsim)
sim = numpy.take(sim, ind, 0)
fsim = numpy.take(fsim, ind, 0)
if callback is not None:
callback(sim[0])
iterations += 1
if retall:
allvecs.append(sim[0])
x = sim[0]
fval = numpy.min(fsim)
warnflag = 0
if fcalls[0] >= maxfun:
warnflag = 1
msg = _status_message['maxfev']
if disp:
print('Warning: ' + msg)
elif iterations >= maxiter:
warnflag = 2
msg = _status_message['maxiter']
if disp:
print('Warning: ' + msg)
else:
msg = _status_message['success']
if disp:
print(msg)
print(" Current function value: %f" % fval)
print(" Iterations: %d" % iterations)
print(" Function evaluations: %d" % fcalls[0])
result = OptimizeResult(fun=fval,
nit=iterations,
nfev=fcalls[0],
status=warnflag,
success=(warnflag == 0),
message=msg,
x=x,
final_simplex=(sim, fsim))
if retall:
result['allvecs'] = allvecs
return result
def _minimize_slsqp(func,
x0,
args=(),
jac=None,
bounds=None,
constraints=(),
maxiter=100,
ftol=1.0E-6,
iprint=1,
disp=False,
eps=_epsilon,
callback=None,
**unknown_options):
"""
Minimize a scalar function of one or more variables using Sequential
Least SQuares Programming (SLSQP).
Options
-------
ftol : float
Precision goal for the value of f in the stopping criterion.
eps : float
Step size used for numerical approximation of the jacobian.
disp : bool
Set to True to print convergence messages. If False,
`verbosity` is ignored and set to 0.
maxiter : int
Maximum number of iterations.
"""
_check_unknown_options(unknown_options)
fprime = jac
iter = maxiter
acc = ftol
epsilon = eps
if not disp:
iprint = 0
# Constraints are triaged per type into a dictionnary of tuples
if isinstance(constraints, dict):
constraints = (constraints, )
cons = {'eq': (), 'ineq': ()}
for ic, con in enumerate(constraints):
# check type
try:
ctype = con['type'].lower()
except KeyError:
raise KeyError('Constraint %d has no type defined.' % ic)
except TypeError:
raise TypeError('Constraints must be defined using a '
'dictionary.')
except AttributeError:
raise TypeError("Constraint's type must be a string.")
else:
if ctype not in ['eq', 'ineq']:
raise ValueError("Unknown constraint type '%s'." % con['type'])
# check function
if 'fun' not in con:
raise ValueError('Constraint %d has no function defined.' % ic)
# check jacobian
cjac = con.get('jac')
if cjac is None:
# approximate jacobian function. The factory function is needed
# to keep a reference to `fun`, see gh-4240.
def cjac_factory(fun):
def cjac(x, *args):
return approx_jacobian(x, fun, epsilon, *args)
return cjac
cjac = cjac_factory(con['fun'])
# update constraints' dictionary
cons[ctype] += ({
'fun': con['fun'],
'jac': cjac,
'args': con.get('args', ())
}, )
exit_modes = {
-1: "Gradient evaluation required (g & a)",
0: "Optimization terminated successfully.",
1: "Function evaluation required (f & c)",
2: "More equality constraints than independent variables",
3: "More than 3*n iterations in LSQ subproblem",
4: "Inequality constraints incompatible",
5: "Singular matrix E in LSQ subproblem",
6: "Singular matrix C in LSQ subproblem",
7: "Rank-deficient equality constraint subproblem HFTI",
8: "Positive directional derivative for linesearch",
9: "Iteration limit exceeded"
}
# Wrap func
feval, func = wrap_function(func, args)
# Wrap fprime, if provided, or approx_jacobian if not
if fprime:
geval, fprime = wrap_function(fprime, args)
else:
geval, fprime = wrap_function(approx_jacobian, (func, epsilon))
# Transform x0 into an array.
x = asfarray(x0).flatten()
# Set the parameters that SLSQP will need
# meq, mieq: number of equality and inequality constraints
meq = sum(
map(len, [atleast_1d(c['fun'](x, *c['args'])) for c in cons['eq']]))
mieq = sum(
map(len, [atleast_1d(c['fun'](x, *c['args'])) for c in cons['ineq']]))
# m = The total number of constraints
m = meq + mieq
# la = The number of constraints, or 1 if there are no constraints
la = array([1, m]).max()
# n = The number of independent variables
n = len(x)
# Define the workspaces for SLSQP
n1 = n + 1
mineq = m - meq + n1 + n1
len_w = (3*n1+m)*(n1+1)+(n1-meq+1)*(mineq+2) + 2*mineq+(n1+mineq)*(n1-meq) \
+ 2*meq + n1 + ((n+1)*n)//2 + 2*m + 3*n + 3*n1 + 1
len_jw = mineq
w = zeros(len_w)
jw = zeros(len_jw)
# Decompose bounds into xl and xu
if bounds is None or len(bounds) == 0:
xl = np.empty(n, dtype=float)
xu = np.empty(n, dtype=float)
xl.fill(np.nan)
xu.fill(np.nan)
else:
bnds = array(bounds, float)
if bnds.shape[0] != n:
raise IndexError('SLSQP Error: the length of bounds is not '
'compatible with that of x0.')
with np.errstate(invalid='ignore'):
bnderr = bnds[:, 0] > bnds[:, 1]
if bnderr.any():
raise ValueError('SLSQP Error: lb > ub in bounds %s.' %
', '.join(str(b) for b in bnderr))
xl, xu = bnds[:, 0], bnds[:, 1]
# Mark infinite bounds with nans; the Fortran code understands this
infbnd = ~isfinite(bnds)
xl[infbnd[:, 0]] = np.nan
xu[infbnd[:, 1]] = np.nan
# Clip initial guess to bounds (SLSQP may fail with bounds-infeasible initial point)
have_bound = np.isfinite(xl)
x[have_bound] = np.clip(x[have_bound], xl[have_bound], np.inf)
have_bound = np.isfinite(xu)
x[have_bound] = np.clip(x[have_bound], -np.inf, xu[have_bound])
# Initialize the iteration counter and the mode value
mode = array(0, int)
acc = array(acc, float)
majiter = array(iter, int)
majiter_prev = 0
# Print the header if iprint >= 2
if iprint >= 2:
print("%5s %5s %16s %16s" % ("NIT", "FC", "OBJFUN", "GNORM"))
while 1:
if mode == 0 or mode == 1: # objective and constraint evaluation requird
# Compute objective function
fx = func(x)
try:
fx = float(np.asarray(fx))
except (TypeError, ValueError):
raise ValueError("Objective function must return a scalar")
# Compute the constraints
if cons['eq']:
c_eq = concatenate([
atleast_1d(con['fun'](x, *con['args']))
for con in cons['eq']
])
else:
c_eq = zeros(0)
if cons['ineq']:
c_ieq = concatenate([
atleast_1d(con['fun'](x, *con['args']))
for con in cons['ineq']
])
else:
c_ieq = zeros(0)
# Now combine c_eq and c_ieq into a single matrix
c = concatenate((c_eq, c_ieq))
if mode == 0 or mode == -1: # gradient evaluation required
# Compute the derivatives of the objective function
# For some reason SLSQP wants g dimensioned to n+1
g = append(fprime(x), 0.0)
# Compute the normals of the constraints
if cons['eq']:
a_eq = vstack(
[con['jac'](x, *con['args']) for con in cons['eq']])
else: # no equality constraint
a_eq = zeros((meq, n))
if cons['ineq']:
a_ieq = vstack(
[con['jac'](x, *con['args']) for con in cons['ineq']])
else: # no inequality constraint
a_ieq = zeros((mieq, n))
# Now combine a_eq and a_ieq into a single a matrix
if m == 0: # no constraints
a = zeros((la, n))
else:
a = vstack((a_eq, a_ieq))
a = concatenate((a, zeros([la, 1])), 1)
# Call SLSQP
slsqp(m, meq, x, xl, xu, fx, c, g, a, acc, majiter, mode, w, jw)
# call callback if major iteration has incremented
if callback is not None and majiter > majiter_prev:
callback(x)
# Print the status of the current iterate if iprint > 2 and the
# major iteration has incremented
if iprint >= 2 and majiter > majiter_prev:
print("%5i %5i % 16.6E % 16.6E" %
(majiter, feval[0], fx, linalg.norm(g)))
# If exit mode is not -1 or 1, slsqp has completed
if abs(mode) != 1:
break
majiter_prev = int(majiter)
# Optimization loop complete. Print status if requested
if iprint >= 1:
print(exit_modes[int(mode)] + " (Exit mode " + str(mode) + ')')
print(" Current function value:", fx)
print(" Iterations:", majiter)
print(" Function evaluations:", feval[0])
print(" Gradient evaluations:", geval[0])
return OptimizeResult(x=x,
fun=fx,
jac=g[:-1],
nit=int(majiter),
nfev=feval[0],
njev=geval[0],
status=int(mode),
message=exit_modes[int(mode)],
success=(mode == 0))
def _root_hybr(func,
x0,
args=(),
jac=None,
col_deriv=0,
xtol=1.49012e-08,
maxfev=0,
band=None,
eps=None,
factor=100,
diag=None,
**unknown_options):
"""
Find the roots of a multivariate function using MINPACK's hybrd and
hybrj routines (modified Powell method).
Options
-------
col_deriv : bool
Specify whether the Jacobian function computes derivatives down
the columns (faster, because there is no transpose operation).
xtol : float
The calculation will terminate if the relative error between two
consecutive iterates is at most `xtol`.
maxfev : int
The maximum number of calls to the function. If zero, then
``100*(N+1)`` is the maximum where N is the number of elements
in `x0`.
band : tuple
If set to a two-sequence containing the number of sub- and
super-diagonals within the band of the Jacobi matrix, the
Jacobi matrix is considered banded (only for ``fprime=None``).
eps : float
A suitable step length for the forward-difference
approximation of the Jacobian (for ``fprime=None``). If
`eps` is less than the machine precision, it is assumed
that the relative errors in the functions are of the order of
the machine precision.
factor : float
A parameter determining the initial step bound
(``factor * || diag * x||``). Should be in the interval
``(0.1, 100)``.
diag : sequence
N positive entries that serve as a scale factors for the
variables.
"""
_check_unknown_options(unknown_options)
epsfcn = eps
x0 = asarray(x0).flatten()
n = len(x0)
if not isinstance(args, tuple):
args = (args, )
shape, dtype = _check_func('fsolve', 'func', func, x0, args, n, (n, ))
if epsfcn is None:
epsfcn = finfo(dtype).eps
Dfun = jac
if Dfun is None:
if band is None:
ml, mu = -10, -10
else:
ml, mu = band[:2]
if maxfev == 0:
maxfev = 200 * (n + 1)
retval = _minpack._hybrd(func, x0, args, 1, xtol, maxfev, ml, mu,
epsfcn, factor, diag)
else:
_check_func('fsolve', 'fprime', Dfun, x0, args, n, (n, n))
if (maxfev == 0):
maxfev = 100 * (n + 1)
retval = _minpack._hybrj(func, Dfun, x0, args, 1, col_deriv, xtol,
maxfev, factor, diag)
x, status = retval[0], retval[-1]
errors = {
0:
"Improper input parameters were entered.",
1:
"The solution converged.",
2:
"The number of calls to function has "
"reached maxfev = %d." % maxfev,
3:
"xtol=%f is too small, no further improvement "
"in the approximate\n solution "
"is possible." % xtol,
4:
"The iteration is not making good progress, as measured "
"by the \n improvement from the last five "
"Jacobian evaluations.",
5:
"The iteration is not making good progress, "
"as measured by the \n improvement from the last "
"ten iterations.",
'unknown':
"An error occurred."
}
info = retval[1]
info['fun'] = info.pop('fvec')
sol = OptimizeResult(x=x, success=(status == 1), status=status)
sol.update(info)
try:
sol['message'] = errors[status]
except KeyError:
sol['message'] = errors['unknown']
return sol
def solve(self, maximize=False):
"""
Minimize the objective function f starting from x0.
:return: x: the optimized parameters
"""
start_time = time.time()
if self.verbose:
if maximize:
solving = 'Maximizing'
else:
solving = 'Minimizing'
self._write(
"{} the problem using a {} IOA with the {} direction.\n".
format(solving, self.alg_type.to_str(), self.dir.to_str()))
xk = np.array(self.x0)
self.optimized = False
# Return values set to Nothing
self.xs = []
self.epochs = []
self.fs = []
self.fs_full = []
self.batches = []
if self.seed != -1:
np.random.seed(self.seed)
self.ep = 0
self.it = 0
# Initialize the multiplicative factor for the direction
self.dir.prep_mult_factor(maximize)
# Initialize the algorithm
self.alg_type.init_solve()
# Initialize the Hessian to an identity matrix for BFGS
# Nothing for the other directions
Bk = self.dir.init_hessian(self.x0)
while self.ep < self.max_epochs:
# Get the function, its gradient and the Hessian
f, fprime, grad_hess = self.dir.compute_func_and_derivatives(
self.alg_type.batch, self.alg_type.full_size)
fk = f(xk)
gk, Bk = grad_hess(xk, Bk)
# Add the return values to the arrays
self.xs.append(xk)
self.fs.append(fk)
self.epochs.append(self.ep)
self.batches.append(self.alg_type.batch)
sc = self.stop_crit(xk, fk, gk)
if 0 < sc <= self.thresh and self.alg_type.batch == self.alg_type.full_size:
if self.verbose:
self._write("Algorithm Optimized!\n")
self._write(" x* = [{}]\n".format(", ".join(
format(x, ".3f") for x in xk)))
self._write(" f(x*) = {:.3f}\n".format(fk))
self.optimized = True
break
if self.verbose:
self._write("Epoch {}:\n".format(self.ep))
self._write(" xk = [{}]\n".format(", ".join(
format(x, ".3f") for x in xk)))
self._write(" f(xk) = {:.3f}\n".format(fk))
self._write(" ||gk|| = {:.3E}\n".format(np.linalg.norm(gk)))
self._write(" stop_crit = {:.3E}\n".format(sc))
# Get the new value for x_k using either a LineSearch or a TrustRegion algorithm
xk_new = self.alg_type.update_xk(xk, fk, gk, Bk, f, fprime,
self.dir, self.fs)
# Update the hessian; only used by BFGS.
# Return the same Bk for the other directions
Bk = self.dir.upd_hessian(xk, xk_new, f, fprime, Bk, gk)
# Make sure xk is still in bounds
xk = back_to_bounds(xk_new, self.bounds)
# Update the batch size if we're using an ABS algorithm
self.dir.batch_changed = self.alg_type.update_batch(
self.it, fk / self.alg_type.batch)
# Update the Hybrid direction if needed (does nothing for all other directions)
self.dir.update_dir(self.alg_type.batch, self.alg_type.full_size)
if self.verbose:
self._write('\n')
# Update the number of epochs
self.ep += self.alg_type.batch / self.alg_type.full_size
self.it += 1
status = 'Algorithm optimized'
if not self.optimized:
status = 'Optimum not reached'
if self.verbose and not self.optimized:
self._write("Algorithm not fully optimized!\n")
self._write(" x_n = [{}]\n".format(", ".join(
format(x, ".3f") for x in xk)))
self._write(" f(x_n) = {:.3f}\n".format(fk))
self.opti_time = time.time() - start_time
# Compute the function value, the gradient and the Hessian one last time.
fk, gk, Bk = self.dir.compute_final_LL_and_derivatives(
xk, hessian=self.compute_final_hessian)
dct = {
'x': xk,
'success': self.optimized,
'status': status,
'fun': fk,
'jac': gk,
'nit': self.it,
'nep': self.ep,
'stop_crit': sc,
'opti_time': self.opti_time
}
if self.compute_final_hessian:
dct['hess'] = Bk
return OptimizeResult(dct)
def minimize_scalar_bounded_alt(func,
bounds,
xatol=1e-5,
maxiter=500,
**extra):
# Adapted from
# https://github.com/scipy/scipy/blob/v1.5.2/scipy/optimize/optimize.py
maxfun = maxiter
x1, x2 = bounds
assert x1 <= x2
flag = 0
sqrt_eps = sqrt(2.2e-16)
golden_mean = 0.5 * (3.0 - sqrt(5.0))
a, b = x1, x2
fulc = a + golden_mean * (b - a)
nfc, xf = fulc, fulc
rat = e = 0.0
x = xf
fx = func(x)
num = 1
fu = inf
ffulc = fnfc = fx
xm = 0.5 * (a + b)
tol1 = sqrt_eps * abs(xf) + xatol / 3.0
tol2 = 2.0 * tol1
while abs(xf - xm) > (tol2 - 0.5 * (b - a)):
golden = 1
# Check for parabolic fit
if abs(e) > tol1:
golden = 0
r = (xf - nfc) * (fx - ffulc)
q = (xf - fulc) * (fx - fnfc)
p = (xf - fulc) * q - (xf - nfc) * r
q = 2.0 * (q - r)
if q > 0.0:
p = -p
q = abs(q)
r = e
e = rat
# Check for acceptability of parabola
if ((abs(p) < abs(0.5 * q * r)) and (p > q * (a - xf))
and (p < q * (b - xf))):
rat = (p + 0.0) / q
x = xf + rat
if ((x - a) < tol2) or ((b - x) < tol2):
si = copysign(1, xm - xf) + ((xm - xf) == 0)
rat = tol1 * si
else: # do a golden-section step
golden = 1
if golden: # do a golden-section step
if xf >= xm:
e = a - xf
else:
e = b - xf
rat = golden_mean * e
si = copysign(1, rat) + (rat == 0)
x = xf + si * max(abs(rat), tol1)
fu = func(x)
num += 1
if fu <= fx:
if x >= xf:
a = xf
else:
b = xf
fulc, ffulc = nfc, fnfc
nfc, fnfc = xf, fx
xf, fx = x, fu
else:
if x < xf:
a = x
else:
b = x
if (fu <= fnfc) or (nfc == xf):
fulc, ffulc = nfc, fnfc
nfc, fnfc = x, fu
elif (fu <= ffulc) or (fulc == xf) or (fulc == nfc):
fulc, ffulc = x, fu
xm = 0.5 * (a + b)
tol1 = sqrt_eps * abs(xf) + xatol / 3.0
tol2 = 2.0 * tol1
if num >= maxfun:
flag = 1
break
if isnan(xf) or isnan(fx) or isnan(fu):
flag = 2
fval = fx
result = OptimizeResult(fun=fval,
status=flag,
success=(flag == 0),
message={
0: 'Solution found.',
1: 'Maximum number of function calls '
'reached.',
2: _status_message['nan']
}.get(flag, ''),
x=xf,
nfev=num)
return result
def _minimize_lbfgsb_timeup(
fun, x0, args=(), jac=None, bounds=None,
disp=None, maxcor=10, ftol=2.2204460492503131e-09,
gtol=1e-5, eps=1e-8, maxfun=15000, maxiter=15000,
iprint=-1, callback=None, maxls=20,
t0=None, timeup=float("inf"),
**unknown_options): # JFF: added time-up check
"""
Minimize a scalar function of one or more variables using the L-BFGS-B
algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxcor : int
The maximum number of variable metric corrections used to
define the limited memory matrix. (The limited memory BFGS
method does not store the full hessian but uses this many terms
in an approximation to it.)
factr : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr * eps``, where ``eps``
is the machine precision, which is automatically generated by
the code. Typical values for `factr` are: 1e12 for low
accuracy; 1e7 for moderate accuracy; 10.0 for extremely high
accuracy.
ftol : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol``.
gtol : float
The iteration will stop when ``max{|proj g_i | i = 1, ..., n}
<= gtol`` where ``pg_i`` is the i-th component of the
projected gradient.
eps : float
Step size used for numerical approximation of the jacobian.
disp : int
Set to True to print convergence messages.
maxfun : int
Maximum number of function evaluations.
maxiter : int
Maximum number of iterations.
maxls : int, optional
Maximum number of line search steps (per iteration). Default is 20.
"""
_check_unknown_options(unknown_options)
m = maxcor
epsilon = eps
pgtol = gtol
factr = ftol / np.finfo(float).eps
x0 = asarray(x0).ravel()
n, = x0.shape
if bounds is None:
bounds = [(None, None)] * n
if len(bounds) != n:
raise ValueError('length of x0 != length of bounds')
# unbounded variables must use None, not +-inf, for optimizer to work properly
bounds = [(None if l == -np.inf else l, None if u == np.inf else u) for l, u in bounds]
if disp is not None:
if disp == 0:
iprint = -1
else:
iprint = disp
n_function_evals, fun = wrap_function(fun, ())
if jac is None:
def func_and_grad(x):
f = fun(x, *args)
g = _approx_fprime_helper(x, fun, epsilon, args=args, f0=f)
return f, g
else:
def func_and_grad(x):
f = fun(x, *args)
g = jac(x, *args)
return f, g
nbd = zeros(n, int32)
low_bnd = zeros(n, float64)
upper_bnd = zeros(n, float64)
bounds_map = {(None, None): 0,
(1, None): 1,
(1, 1): 2,
(None, 1): 3}
for i in range(0, n):
l, u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
if not maxls > 0:
raise ValueError('maxls must be positive.')
x = array(x0, float64)
f = array(0.0, float64)
g = zeros((n,), float64)
wa = zeros(2*m*n + 5*n + 11*m*m + 8*m, float64)
iwa = zeros(3*n, int32)
task = zeros(1, 'S60')
csave = zeros(1, 'S60')
lsave = zeros(4, int32)
isave = zeros(44, int32)
dsave = zeros(29, float64)
task[:] = 'START'
n_iterations = 0
if t0 is None:
t0 = time.time()
time_profile.predicted_inner_loop_func2_duration = 0.0
while 1:
# x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
_lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
pgtol, wa, iwa, task, iprint, csave, lsave,
isave, dsave, maxls)
task_str = task.tostring()
# begin EB
curr_time = time.time()
predicted_inner_loop_func2_duration = (curr_time +
time_profile.maximize_inner_time_profile.mean +
time_profile.func2_time_profile.mean - t0)
if predicted_inner_loop_func2_duration > timeup: # JFF: added time-up check
task[:] = ('STOP: PREDICTED COMPUTATION TIME EXCEEDS LIMIT')
break
# end EB
if task_str.startswith(b'FG'):
# The minimization routine wants f and g at the current x.
# Note that interruptions due to maxfun are postponed
# until the completion of the current minimization iteration.
# Overwrite f and g:
f, g = func_and_grad(x)
elif task_str.startswith(b'NEW_X'):
# new iteration
if n_iterations > maxiter:
task[:] = 'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT'
elif n_function_evals[0] > maxfun:
task[:] = ('STOP: TOTAL NO. of f AND g EVALUATIONS '
'EXCEEDS LIMIT')
else:
n_iterations += 1
if callback is not None:
callback(x)
else:
break
time_profile.predicted_inner_loop_func2_duration = predicted_inner_loop_func2_duration
task_str = task.tostring().strip(b'\x00').strip()
if task_str.startswith(b'CONV'):
warnflag = 0
elif n_function_evals[0] > maxfun:
warnflag = 1
elif n_iterations > maxiter:
warnflag = 1
else:
warnflag = 2
# These two portions of the workspace are described in the mainlb
# subroutine in lbfgsb.f. See line 363.
s = wa[0: m*n].reshape(m, n)
y = wa[m*n: 2*m*n].reshape(m, n)
# See lbfgsb.f line 160 for this portion of the workspace.
# isave(31) = the total number of BFGS updates prior the current iteration;
n_bfgs_updates = isave[30]
n_corrs = min(n_bfgs_updates, maxcor)
hess_inv = LbfgsInvHessProduct(s[:n_corrs], y[:n_corrs])
return OptimizeResult(fun=f, jac=g, nfev=n_function_evals[0],
nit=n_iterations, status=warnflag, message=task_str,
x=x, success=(warnflag == 0), hess_inv=hess_inv)
def solve(self, maximize=False):
"""
Minimize the objective function f starting from x0.
:return: x: the optimized parameters
"""
start_time = time.clock()
if self.verbose:
if maximize:
solving = 'Maximizing'
else:
solving = 'Minimizing'
self._write("{} the problem using Newton Method\n".format(solving))
xk = np.asarray(self.x0).flatten()
self.ep = 0
self.it = 0
I = np.eye(len(self.x0))
Bk = I
mult = 1
if maximize:
mult = -1
if self.biogeme is None:
fprime = lambda x: mult * self.grad(x)
else:
fprime = lambda x: mult * self.biogeme.calculateLikelihoodAndDerivatives(
x, hessian=False)[1]
f = lambda x: mult * self.func(x)
self.xs = []
self.epochs = []
self.fs = []
while self.ep < self.nbr_epochs:
fk = f(xk)
gk = fprime(xk)
self.xs.append(xk)
self.fs.append(fk)
self.epochs.append(self.ep)
if self.verbose:
self._write("Epoch {}:\n".format(self.ep))
self._write(" xk = [{}]\n".format(", ".join(
format(x, ".3f") for x in xk)))
self._write(" f(xk) = {:.3f}\n".format(fk))
step = -np.dot(Bk, gk)
if self.it > 0:
old_old_fval = self.fs[-2]
else:
old_old_fval = self.fs[-1] + np.linalg.norm(gk) / 2
alpha = ls_wolfe12(f, fprime, xk, step, gk, self.fs[-1],
old_old_fval)
xkp1 = xk + alpha * step
sk = xkp1 - xk
xk = xkp1
gkp1 = fprime(xkp1)
yk = gkp1 - gk
gk = gkp1
gnorm = np.linalg.norm(gk)
snorm = np.linalg.norm(step)
if self.verbose:
self._write(" ||gk|| = {:.3E}\n".format(gnorm))
self._write(" ||step|| = {:.3E}\n".format(
np.linalg.norm(step)))
self._write(" alpha = {:.3E}\n".format(alpha))
if (gnorm <= self.thresh) or (snorm <= self.thresh):
self.status = 'Optimum reached!'
if self.verbose:
self._write("Algorithm Optimized!\n")
self._write(" x* = [{}]\n".format(", ".join(
format(x, ".3f") for x in xk)))
self._write(" f(x*) = {:.3f}\n".format(fk))
self.optimized = True
break
try: # this was handled in numeric, let it remains for more safety
rhok = 1.0 / (np.dot(yk, sk))
except ZeroDivisionError:
rhok = 1000.0
if np.isinf(rhok): # this is patch for numpy
rhok = 1000.0
A1 = I - sk[:, np.newaxis] * yk[np.newaxis, :] * rhok
A2 = I - yk[:, np.newaxis] * sk[np.newaxis, :] * rhok
Bk = np.dot(A1, np.dot(
Bk, A2)) + (rhok * sk[:, np.newaxis] * sk[np.newaxis, :])
if self.verbose:
self._write('\n')
if self.bounds is not None:
tmp = []
for i, x in enumerate(xk):
if self.bounds[i][1] is not None and x > self.bounds[i][1]:
tmp.append(self.bounds[i][1])
elif self.bounds[i][
0] is not None and x < self.bounds[i][0]:
tmp.append(self.bounds[i][0])
else:
tmp.append(x)
xk = tmp
self.it += 1
self.ep += 1
if not self.optimized:
self.status = 'Optimum not reached'
if self.verbose and not self.optimized:
self._write("Algorithm not fully optimized!\n")
self._write(" x_n = [{}]\n".format(", ".join(
format(x, ".3f") for x in xk)))
self._write(" f(x_n) = {:.3f}\n".format(fk))
self.f = mult * f(xk)
self.x = xk
gk = fprime(xk)
self.opti_time = time.clock() - start_time
dct = {
'x': self.x,
'success': self.optimized,
'status': self.status,
'fun': self.f,
'jac': gk,
'hess': Bk,
'nit': self.it,
'nep': self.ep,
'opti_time': self.opti_time
}
return OptimizeResult(dct)
def _minimize_trust_region(fun,
x0,
subproblem=None,
initial_trust_radius=1.,
max_trust_radius=1000.,
eta=0.15,
gtol=1e-4,
max_iter=None,
disp=False,
return_all=False,
callback=None):
"""
Minimization of scalar function of one or more variables using a
trust-region algorithm.
Options for the trust-region algorithm are:
initial_trust_radius : float
Initial trust radius.
max_trust_radius : float
Never propose steps that are longer than this value.
eta : float
Trust region related acceptance stringency for proposed steps.
gtol : float
Gradient norm must be less than `gtol`
before successful termination.
max_iter : int
Maximum number of iterations to perform.
disp : bool
If True, print convergence message.
This function is called by :func:`torchmin.minimize`.
It is not supposed to be called directly.
"""
if subproblem is None:
raise ValueError('A subproblem solving strategy is required for '
'trust-region methods')
if not (0 <= eta < 0.25):
raise Exception('invalid acceptance stringency')
if max_trust_radius <= 0:
raise Exception('the max trust radius must be positive')
if initial_trust_radius <= 0:
raise ValueError('the initial trust radius must be positive')
if initial_trust_radius >= max_trust_radius:
raise ValueError('the initial trust radius must be less than the '
'max trust radius')
# Input check/pre-process
disp = int(disp)
if max_iter is None:
max_iter = x0.numel() * 200
# Construct scalar objective function
hessp = subproblem.hess_prod
sf = ScalarFunction(fun, x0.shape, hessp=hessp, hess=not hessp)
closure = sf.closure
# init the search status
warnflag = 1 # maximum iterations flag
k = 0
# initialize the search
trust_radius = torch.as_tensor(initial_trust_radius,
dtype=x0.dtype,
device=x0.device)
x = x0.detach().flatten()
if return_all:
allvecs = [x]
# initial subproblem
m = subproblem(x, closure)
# search for the function min
# do not even start if the gradient is small enough
while k < max_iter:
# Solve the sub-problem.
# This gives us the proposed step relative to the current position
# and it tells us whether the proposed step
# has reached the trust region boundary or not.
try:
p, hits_boundary = m.solve(trust_radius)
except RuntimeError as exc:
# TODO: catch general linalg error like np.linalg.linalg.LinAlgError
if 'singular' in exc.args[0]:
warnflag = 3
break
else:
raise
# calculate the predicted value at the proposed point
predicted_value = m(p)
# define the local approximation at the proposed point
x_proposed = x + p
m_proposed = subproblem(x_proposed, closure)
# evaluate the ratio defined in equation (4.4)
actual_reduction = m.fun - m_proposed.fun
predicted_reduction = m.fun - predicted_value
if predicted_reduction <= 0:
warnflag = 2
break
rho = actual_reduction / predicted_reduction
# update the trust radius according to the actual/predicted ratio
if rho < 0.25:
trust_radius = trust_radius.mul(0.25)
elif rho > 0.75 and hits_boundary:
trust_radius = torch.clamp(2 * trust_radius, max=max_trust_radius)
# if the ratio is high enough then accept the proposed step
if rho > eta:
x = x_proposed
m = m_proposed
# append the best guess, call back, increment the iteration count
if return_all:
allvecs.append(x.clone())
if callback is not None:
callback(x.clone())
k += 1
# verbosity check
if disp > 1:
print('iter %d - fval: %0.4f' % (k, m.fun))
# check if the gradient is small enough to stop
if m.jac_mag < gtol:
warnflag = 0
break
# print some stuff if requested
if disp:
msg = status_messages[warnflag]
if warnflag != 0:
msg = 'Warning: ' + msg
print(msg)
print(" Current function value: %f" % m.fun)
print(" Iterations: %d" % k)
print(" Function evaluations: %d" % sf.nfev)
# print(" Gradient evaluations: %d" % sf.ngev)
# print(" Hessian evaluations: %d" % (sf.nhev + nhessp[0]))
result = OptimizeResult(x=x.view_as(x0),
fun=m.fun,
grad=m.jac.view_as(x0),
success=(warnflag == 0),
status=warnflag,
nfev=sf.nfev,
nit=k,
message=status_messages[warnflag])
if not subproblem.hess_prod:
result['hess'] = m.hess.view(*x0.shape, *x0.shape)
if return_all:
result['allvecs'] = allvecs
return result
def _minimize_lbfgsb_multi(fun, x0, args=(), jac=None, bounds=None,
disp=None, maxcor=10, ftol=2.2204460492503131e-09,
gtol=1e-5, eps=1e-8, maxfun=15000, maxiter=15000,
iprint=-1, callback=None, maxls=20, **unknown_options):
"""
Minimize a scalar function of one or more variables using the L-BFGS-B
algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxcor : int
The maximum number of variable metric corrections used to
define the limited memory matrix. (The limited memory BFGS
method does not store the full hessian but uses this many terms
in an approximation to it.)
ftol : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol``.
gtol : float
The iteration will stop when ``max{|proj g_i | i = 1, ..., n}
<= gtol`` where ``pg_i`` is the i-th component of the
projected gradient.
eps : float
Step size used for numerical approximation of the jacobian.
disp : int
Set to True to print convergence messages.
maxfun : int
Maximum number of function evaluations.
maxiter : int
Maximum number of iterations.
maxls : int, optional
Maximum number of line search steps (per iteration). Default is 20.
Notes
-----
The option `ftol` is exposed via the `scipy.optimize.minimize` interface,
but calling `scipy.optimize.fmin_l_bfgs_b` directly exposes `factr`. The
relationship between the two is ``ftol = factr * numpy.finfo(float).eps``.
I.e., `factr` multiplies the default machine floating-point precision to
arrive at `ftol`.
"""
_check_unknown_options(unknown_options)
m = maxcor
epsilon = eps
pgtol = gtol
factr = ftol / np.finfo(float).eps
k,n = x0.shape
# x0 = [asarray(x).ravel() for x in x0]
# n, = x0.shape
if bounds is None:
bounds = [(None, None)] * n
if len(bounds) != n:
raise ValueError('length of x0 != length of bounds')
# unbounded variables must use None, not +-inf, for optimizer to work properly
bounds = [(None if l == -np.inf else l, None if u == np.inf else u) for l, u in bounds]
if disp is not None:
if disp == 0:
iprint = -1
else:
iprint = disp
if jac is None:
def func_and_grad(x):
# f = fun(x, *args)
f, g = _approx_fprime_helper(x, fun, epsilon, args=args)
return f, g
else:
def func_and_grad(x):
f = fun(x, *args)
g = jac(x, *args)
return f, g
nbd = zeros(n, int32)
low_bnd = zeros(n, float64)
upper_bnd = zeros(n, float64)
bounds_map = {(None, None): 0,
(1, None): 1,
(1, 1): 2,
(None, 1): 3}
for i in range(0, n):
l, u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
if not maxls > 0:
raise ValueError('maxls must be positive.')
# x = [array(x0_, float64) for x0_ in x0]
X = x0.copy()
f = [array(0.0, float64) for _ in range(k)]
g = [zeros((n,), float64) for _ in range(k)]
wa = [zeros(2*m*n + 5*n + 11*m*m + 8*m, float64) for _ in range(k)]
iwa = [zeros(3*n, int32) for _ in range(k)]
task = [zeros(1, 'S60') for _ in range(k)]
csave = [zeros(1, 'S60') for _ in range(k)]
lsave = [zeros(4, int32) for _ in range(k)]
isave = [zeros(44, int32) for _ in range(k)]
dsave = [zeros(29, float64) for _ in range(k)]
n_function_evals = 0
for i in range(k):
task[i][:] = 'START'
n_iterations = [0 for _ in range(k)]
k_running = [i for i in range(k)]
while 1:
# x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
request_index = []
# run all instance until they request a new point
# X contains only points for the remaining instances
for i in k_running:
while 1:
_lbfgsb.setulb(m, X[i], low_bnd, upper_bnd, nbd, f[i], g[i], factr,
pgtol, wa[i], iwa[i], task[i], iprint, csave[i], lsave[i],
isave[i], dsave[i], maxls)
task_str = task[i].tostring()
if task_str.startswith(b'FG'):
# The minimization routine wants f and g at the current x.
# Note that interruptions due to maxfun are postponed
# until the completion of the current minimization iteration.
# Overwrite f and g:
request_index.append(i)
break
elif task_str.startswith(b'NEW_X'):
# new iteration
if n_iterations[i] > maxiter:
task[i][:] = 'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT'
# break
elif n_function_evals > maxfun:
task[i][:] = ('STOP: TOTAL NO. of f AND g EVALUATIONS '
'EXCEEDS LIMIT')
# break
else:
n_iterations[i] += 1
# if callback is not None:
# callback(x)
else:
break
k_running = request_index
if len(k_running) == 0:
break
F,G = func_and_grad(X[request_index])
n_function_evals += 1
for ff,gg,i in zip(F,G,k_running):
f[i] = ff
g[i] = gg
warnflag = []
task_str = [t.tostring().strip(b'\x00').strip() for t in task]
for i,t in enumerate(task_str):
if t.startswith(b'CONV'):
warnflag.append(0)
elif n_function_evals > maxfun:
warnflag.append(1)
elif n_iterations[i] > maxiter:
warnflag.append(2)
else:
warnflag.append(3)
# These two portions of the workspace are described in the mainlb
# subroutine in lbfgsb.f. See line 363.
# s = [wa[i][0: m*n].reshape(m, n) for i in range(k)]
# y = [wa[i][m*n: 2*m*n].reshape(m, n) for i in range(k)]
# See lbfgsb.f line 160 for this portion of the workspace.
# isave(31) = the total number of BFGS updates prior the current iteration;
# n_bfgs_updates = isave[30]
# n_corrs = min(n_bfgs_updates, maxcor)
# hess_inv = LbfgsInvHessProduct(s[:n_corrs], y[:n_corrs])
return OptimizeResult(fun=f, jac=g, nfev=n_function_evals,
nit=n_iterations, status=warnflag, message=task_str,
x=X, success=(sum(warnflag) == 0))
def _minimize_lbfgsb(
fun,
x0,
bounds=None,
args=(),
kwargs={},
jac=None,
callback=None,
tol={"abs": 1e-05, "rel": 1e-08},
norm=np.Inf,
maxiter=None,
disp=False,
return_all=False,
**unknown_options
):
"""
Minimization of scalar function of one or more variables using the
BHHH algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxiter : int
Maximum number of iterations to perform.
tol : dict
Absolute and relative tolerance values.
norm : float
Order of norm (Inf is max, -Inf is min).
"""
_check_unknown_options(unknown_options)
def f(x0):
return fun(x0, *args, **kwargs)
fprime = jac
# epsilon = eps Add functionality
retall = return_all
k = 0
ns = 0
nsmax = 5
N = len(x0)
x0 = np.asarray(x0).flatten()
if x0.ndim == 0:
x0.shape = (1,)
if bounds is None:
bounds = np.array([np.inf] * N * 2).reshape((2, N))
bounds[0, :] = -bounds[0, :]
if bounds.shape[1] != N:
raise ValueError("length of x0 != length of bounds")
low = bounds[0, :]
up = bounds[1, :]
x0 = np.clip(x0, low, up)
if maxiter is None:
maxiter = len(x0) * 200
if not callable(fprime):
def myfprime(x0):
return approx_derivative(f, x0, args=args, kwargs=kwargs)
else:
myfprime = fprime
# Setup for iteration
old_fval = f(x0)
gf0 = myfprime(x0)
gfk = gf0
norm_pg0 = vecnorm(x0 - np.clip(x0 - gf0, low, up), ord=norm)
xk = x0
norm_pgk = norm_pg0
sstore = np.zeros((maxiter, N))
ystore = sstore.copy()
if retall:
allvecs = [x0]
warnflag = 0
# Calculate indices ofactive and inative set using projected gradient
epsilon = min(np.min(up - low) / 2, norm_pgk)
activeset = np.logical_or(xk - low <= epsilon, up - xk <= epsilon)
inactiveset = np.logical_not(activeset)
for _ in range(maxiter): # for loop instead.
# Check tolerance of gradient norm
if norm_pgk <= tol["abs"] + tol["rel"] * norm_pg0:
break
pk = -gfk
pk = bfgsrecb(ns, sstore, ystore, pk, activeset)
gfk_active = gfk.copy()
gfk_active[inactiveset] = 0
pk = -gfk_active + pk
# Sets the initial step guess to dx ~ 1
old_old_fval = old_fval + np.linalg.norm(gfk) / 2
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = _line_search_wolfe12(
f,
myfprime,
xk,
pk,
gfk,
old_fval,
old_old_fval,
amin=1e-100,
amax=1e100,
)
except _LineSearchError:
# Line search failed to find a better solution.
warnflag = 2
break
xkp1 = np.clip(xk + alpha_k * pk, low, up)
if retall:
allvecs.append(xkp1)
yk = myfprime(xkp1) - gfk
sk = xkp1 - xk
xk = xkp1
gfk = myfprime(xkp1)
norm_pgk = vecnorm(xk - np.clip(xk - gfk, low, up), ord=norm)
# Calculate indices ofactive and inative set using projected gradient
epsilon = min(np.min(up - low) / 2, norm_pgk)
activeset = np.logical_or(xk - low <= epsilon, up - xk <= epsilon)
inactiveset = np.logical_not(activeset)
yk[activeset] = 0
sk[activeset] = 0
# reset storage
ytsk = yk.dot(sk)
if ytsk <= 0:
ns = 0
if ns == nsmax:
print("ns reached maximum size")
ns = 0
elif ytsk > 0:
ns += 1
alpha0 = ytsk ** 0.5
sstore[ns - 1, :] = sk / alpha0
ystore[ns - 1, :] = yk / alpha0
k += 1
if callback is not None:
callback(xk)
if np.isinf(old_fval):
# We correctly found +-Inf as optimal value, or something went
# wrong.
warnflag = 2
break
if np.isnan(xk).any():
warnflag = 3
break
fval = old_fval
if warnflag == 2:
msg = _status_message["pr_loss"]
elif k >= maxiter:
warnflag = 1
msg = _status_message["maxiter"]
elif np.isnan(fval) or np.isnan(xk).any():
warnflag = 3
msg = _status_message["nan"]
else:
msg = _status_message["success"]
if disp:
print("{}{}".format("Warning: " if warnflag != 0 else "", msg))
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
result = OptimizeResult(
fun=fval,
jac=gfk,
status=warnflag,
success=(warnflag == 0),
message=msg,
x=xk,
nit=k,
)
if retall:
result["allvecs"] = allvecs
return result
def solve(self, maximize=False):
"""
Minimize the objective function f starting from x0.
:return: x: the optimized parameters
"""
start_time = time.clock()
if self.verbose:
if maximize:
solving = 'Maximizing'
else:
solving = 'Minimizing'
self._write("{} the problem using BFGS Method\n".format(solving))
xk = np.asarray(self.x0).flatten()
self.ep = 0
self.it = 0
I = np.eye(len(self.x0))
Bk = I
mult = 1
if maximize:
mult = -1
self.xs = []
self.epochs = []
self.fs = []
self.fs_full = []
self.batches = []
if self.batch == -1:
self.batch = self.full_size
if self.seed != -1:
np.random.seed(self.seed)
while self.ep < self.nbr_epochs:
if self.biogeme is None:
idx = np.random.choice(self.full_size, self.batch, replace=False)
if self.batch == -1:
fprime = lambda x: mult*self.grad(x)
f = lambda x: mult*self.func(x)
else:
fprime = lambda x: mult*self.grad(x, idx)
f = lambda x: mult*self.func(x, idx)
full_f = lambda x: self.func(x)
else:
sample = self.biogeme.database.data.sample(n=self.batch, replace=False)
if self.batch == -1:
fprime = lambda x: mult*self.biogeme.calculateLikelihoodAndDerivatives(x, hessian=False)[1]
f = lambda x: mult*self.func(x)
else:
def fprime(x):
self.biogeme.theC.setData(sample)
tmp = back_to_bounds(x, self.bounds)
return mult*self.biogeme.calculateLikelihoodAndDerivatives(tmp, hessian=False)[1]
def f(x):
self.biogeme.theC.setData(sample)
tmp = back_to_bounds(x, self.bounds)
return mult*self.func(tmp)
def full_f(x):
self.biogeme.theC.setData(self.biogeme.database.data)
return self.func(x)
fk = f(xk)
fk_full = full_f(xk)
gk = fprime(xk)
self.xs.append(xk)
self.fs.append(fk)
self.fs_full.append(fk_full)
self.epochs.append(self.ep)
self.batches.append(self.batch)
if self.verbose:
self._write("Epoch {}:\n".format(self.ep))
self._write(" xk = [{}]\n".format(", ".join(format(x, ".3f") for x in xk)))
self._write(" f(xk) = {:.3f}\n".format(fk_full))
step = -np.dot(Bk, gk)
if self.it > 0:
old_old_fval = self.fs[-2]
else:
old_old_fval = self.fs[-1] + np.linalg.norm(gk) / 2
alpha = ls_wolfe12(f, fprime, xk, step, gk, self.fs[-1], old_old_fval)
xkp1 = xk + alpha * step
sk = xkp1 - xk
xk = xkp1
xk = back_to_bounds(xk, self.bounds)
gkp1 = fprime(xk)
yk = gkp1 - gk
gk = gkp1
gnorm = np.linalg.norm(gk)
snorm = np.linalg.norm(step)
if self.verbose:
self._write(" ||gk|| = {:.3E}\n".format(gnorm))
self._write(" ||step|| = {:.3E}\n".format(np.linalg.norm(step)))
self._write(" alpha = {:.3E}\n".format(alpha))
if (gnorm <= self.thresh) or (snorm <= self.thresh):
self.status = 'Optimum reached!'
if self.verbose:
self._write("Algorithm Optimized!\n")
self._write(" x* = [{}]\n".format(", ".join(format(x, ".3f") for x in xk)))
self._write(" f(x*) = {:.3f}\n".format(fk))
self.optimized = True
break
try: # this was handled in numeric, let it remaines for more safety
rhok = 1.0 / (np.dot(yk, sk))
except ZeroDivisionError:
rhok = 1000.0
if np.isinf(rhok): # this is patch for numpy
rhok = 1000.0
A1 = I - sk[:, np.newaxis] * yk[np.newaxis, :] * rhok
A2 = I - yk[:, np.newaxis] * sk[np.newaxis, :] * rhok
Bk = np.dot(A1, np.dot(Bk, A2)) + (rhok * sk[:, np.newaxis] *
sk[np.newaxis, :])
self.batch = self.abs.upd(self.it, fk_full, self.batch)
if self.verbose:
self._write('\n')
if self.batch == -1:
self.ep += 1
else:
self.ep += self.batch/self.full_size
self.it += 1
if not self.optimized:
self.status = 'Optimum not reached'
if self.verbose and not self.optimized:
self._write("Algorithm not fully optimized!\n")
self._write(" x_n = [{}]\n".format(", ".join(format(x, ".3f") for x in xk)))
self._write(" f(x_n) = {:.3f}\n".format(fk))
self.f = mult*f(xk)
self.x = xk
gk = fprime(xk)
self.opti_time = time.clock() - start_time
dct = {'x': self.x,
'success': self.optimized,
'status': self.status,
'fun': self.f,
'jac': gk,
'hess': Bk,
'nit': self.it,
'nep': self.ep,
'opti_time': self.opti_time}
return OptimizeResult(dct)
def _minimize_bhhh(fun,
x0,
bounds=None,
args=(),
jac=None,
callback=None,
tol={
"abs": 1e-05,
"rel": 1e-08
},
norm=np.Inf,
maxiter=None,
disp=False,
return_all=False,
**unknown_options):
"""
Minimization of scalar function of one or more variables using the
BHHH algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxiter : int
Maximum number of iterations to perform.
tol : dict
Absolute and relative tolerance values.
norm : float
Order of norm (Inf is max, -Inf is min).
"""
_check_unknown_options(unknown_options)
f = fun
fprime = jac
retall = return_all
k = 0
N = len(x0)
x0 = np.asarray(x0).flatten()
if x0.ndim == 0:
x0.shape = (1, )
if bounds is None:
bounds = np.array([np.inf] * N * 2).reshape((2, N))
bounds[0, :] = -bounds[0, :]
if bounds.shape[1] != N:
raise ValueError("length of x0 != length of bounds")
low = bounds[0, :]
up = bounds[1, :]
x0 = np.clip(x0, low, up)
if maxiter is None:
maxiter = len(x0) * 200
# Need the aggregate functions to take only x0 as an argument
func_calls, agg_fun = wrap_function_agg(f, args)
if not callable(fprime):
grad_calls, myfprime = wrap_function_num_dev(f, args)
else:
grad_calls, myfprime = wrap_function(fprime, args)
def agg_fprime(x0):
return myfprime(x0).sum(axis=0)
# Setup for iteration
old_fval = agg_fun(x0)
gf0 = agg_fprime(x0)
norm_pg0 = vecnorm(x0 - np.clip(x0 - gf0, low, up), ord=norm)
xk = x0
norm_pgk = norm_pg0
if retall:
allvecs = [x0]
warnflag = 0
for _ in range(maxiter): # for loop instead.
# Individual
gfk_obs = myfprime(xk)
# Aggregate fprime. Might replace by simply summing up gfk_obs
gfk = gfk_obs.sum(axis=0)
norm_pgk = vecnorm(xk - np.clip(xk - gfk, low, up), ord=norm)
# Check tolerance of gradient norm
if norm_pgk <= tol["abs"] + tol["rel"] * norm_pg0:
break
# Sets the initial step guess to dx ~ 1
old_old_fval = old_fval + np.linalg.norm(gfk) / 2
# Calculate BHHH hessian and step
Hk = np.dot(gfk_obs.T, gfk_obs)
Bk = np.linalg.inv(Hk)
pk = np.empty(N)
pk = -np.dot(Bk, gfk)
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = _line_search_wolfe12(
agg_fun,
agg_fprime,
xk,
pk,
gfk,
old_fval,
old_old_fval,
amin=1e-100,
amax=1e100,
)
except _LineSearchError:
# Line search failed to find a better solution.
warnflag = 2
break
xkp1 = np.clip(xk + alpha_k * pk, low, up)
if retall:
allvecs.append(xkp1)
xk = xkp1
if callback is not None:
callback(xk)
k += 1
if np.isinf(old_fval):
# We correctly found +-Inf as optimal value, or something went
# wrong.
warnflag = 2
break
fval = old_fval
if warnflag == 2:
msg = _status_message["pr_loss"]
elif k >= maxiter:
warnflag = 1
msg = _status_message["maxiter"]
elif np.isnan(fval) or np.isnan(xk).any():
warnflag = 3
msg = _status_message["nan"]
else:
msg = _status_message["success"]
if disp:
print("{}{}".format("Warning: " if warnflag != 0 else "", msg))
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
result = OptimizeResult(
fun=fval,
jac=gfk,
hess_inv=Bk,
nfev=func_calls[0],
njev=grad_calls[0],
status=warnflag,
success=(warnflag == 0),
message=msg,
x=xk,
nit=k,
)
if retall:
result["allvecs"] = allvecs
return result