def bfgs_more_gutted(C, u, z, rho, x, n):
max_iter = 15000
# fun = MemoizeJac(l2_log)
# jac = fun.derivative
m = 10
# epsilon = 1e-8
# pgtol = 1e-5
# factr = 1e-7#ftol / np.finfo(float).eps
x = x.ravel()
# def func_and_grad(x):
# f = fun(x, C, z, u, rho, n)
# g = jac(x, C, z, u, rho, n)
# return f, g
nbd = zeros(n, int32)
low_bnd = zeros(n, float64)
upper_bnd = zeros(n, float64)
# x = array(x0, float64)
# f = array(0.0, float64)
f = 0.0
g = zeros((n,), float64)
# wa = zeros(2*m*n + 5*n + 11*m*m + 8*m, float64)
wa = zeros(20*n + 5*n + 1180, float64)
iwa = zeros(3*n, int32)
task = zeros(1, 'S60')
csave = zeros(1, 'S60')
lsave = zeros(4, int32)
isave = zeros(44, int32)
dsave = zeros(29, float64)
task[:] = 'START'
n_iterations = 0
while 1:
setulb(m, x, low_bnd, upper_bnd, nbd, f, g, 1e-7,
1e-5, wa, iwa, task, -1, csave, lsave,isave, dsave, 20)
#task_str = task.tostring()
# if task_str.startswith(b'FG'):
task_str = task[0][:2]
if task_str == 'FG':
# reduce call time put the code here
y = x.reshape(n,1)
v = sub(add(y,u),z)
eCx = exp(dot(C,y))
rhov = rho * v
# fobj = umr_sum(log1p(eCx), None, None, None, False)
# f = fobj + 0.5 * umr_sum(mul(rhov,v), None, None, None, False)
f = umr_sum(log1p(eCx), None, None, None, False) + 0.5 * umr_sum(mul(rhov,v), None, None, None, False)
g = dot(C.T, div(eCx, (add(1, eCx)))) + rhov
elif task_str == 'NE':
# elif task_str.startswith(b'NEW_X'):
n_iterations += 1
if n_iterations >= max_iter:
return x
else:
return x
def _smooth_search(self):
"""Local search implementation using setulb core function
"""
m = 5
iteration = 0
f = 0
n = self._x.size
ndb = np.zeros(self._x.size)
ndb.fill(2)
iprint = -1
x = np.array(self._xbuffer, np.float64)
f = np.array(0.0, np.float64)
l = np.array(self._lower, np.float64)
u = np.array(self._upper, np.float64)
g = np.zeros(n, np.float64)
wa = np.zeros(2 * m * n + 5 * n + 11 * m * m + 8 * m, np.float64)
iwa = np.zeros(3 * n, np.int32)
task = np.zeros(1, 'S60')
csave = np.zeros(1, 'S60')
lsave = np.zeros(4, np.int32)
isave = np.zeros(44, np.int32)
dsave = np.zeros(29, np.float64)
task[:] = 'START'
if self.itsoftmax < 100:
self.itsoftmax = 100
elif self.itsoftmax > 1000:
self.itsoftmax = 1000
while True:
if iteration >= self.itsoftmax:
self._xbuffer = np.array(x)
self._fvalue = f
return
_lbfgsb.setulb(m, x, l, u, ndb,
f, g, self.factr, self.pgtol, wa, iwa, task,
iprint, csave, lsave, isave, dsave, 100)
iteration += 1
task_str = task.tostring()
if task_str.startswith(b'FG'):
self._xbuffer = np.array(x)
f = self._fobjective(self._xbuffer)
if self.know_real:
if f <= self.real_threshold:
self._xbuffer = np.array(x)
self._fvalue = f
return
g = np.array(self._compute_gradient(x), np.float64)
elif task_str.startswith(b'NEW_X'):
pass
else:
self._fvalue = f
self._xbuffer = np.array(x)
return
def _smooth_search(self):
"""Local search implementation using setulb core function
"""
m = 5
iteration = 0
f = 0
n = self._x.size
ndb = np.zeros(self._x.size)
ndb.fill(2)
iprint = -1
x = np.array(self._xbuffer, np.float64)
f = np.array(0.0, np.float64)
l = np.array(self._lower, np.float64)
u = np.array(self._upper, np.float64)
g = np.zeros(n, np.float64)
wa = np.zeros(2 * m * n + 5 * n + 11 * m * m + 8 * m, np.float64)
iwa = np.zeros(3 * n, np.int32)
task = np.zeros(1, 'S60')
csave = np.zeros(1, 'S60')
lsave = np.zeros(4, np.int32)
isave = np.zeros(44, np.int32)
dsave = np.zeros(29, np.float64)
task[:] = 'START'
if self.itsoftmax < 100:
self.itsoftmax = 100
elif self.itsoftmax > 1000:
self.itsoftmax = 1000
while True:
if iteration >= self.itsoftmax:
self._xbuffer = np.array(x)
self._fvalue = f
return
_lbfgsb.setulb(m, x, l, u, ndb, f, g, self.factr, self.pgtol, wa,
iwa, task, iprint, csave, lsave, isave, dsave, 100)
iteration += 1
task_str = task.tostring()
if task_str.startswith(b'FG'):
self._xbuffer = np.array(x)
f = self._fobjective(self._xbuffer)
if self.know_real:
if f <= self.real_threshold:
self._xbuffer = np.array(x)
self._fvalue = f
return
g = np.array(self._compute_gradient(x), np.float64)
elif task_str.startswith(b'NEW_X'):
pass
else:
self._fvalue = f
self._xbuffer = np.array(x)
return
def test_setulb_floatround():
"""test if setulb() violates bounds
checks for violation due to floating point rounding error
"""
n = 5
m = 10
factr = 1e7
pgtol = 1e-5
maxls = 20
iprint = -1
nbd = np.full((n, ), 2)
low_bnd = np.zeros(n, np.float64)
upper_bnd = np.ones(n, np.float64)
x0 = np.array([
0.8750000000000278, 0.7500000000000153, 0.9499999999999722,
0.8214285714285992, 0.6363636363636085
])
x = np.copy(x0)
f = np.array(0.0, np.float64)
g = np.zeros(n, np.float64)
fortran_int = _lbfgsb.types.intvar.dtype
wa = np.zeros(2 * m * n + 5 * n + 11 * m * m + 8 * m, np.float64)
iwa = np.zeros(3 * n, fortran_int)
task = np.zeros(1, 'S60')
csave = np.zeros(1, 'S60')
lsave = np.zeros(4, fortran_int)
isave = np.zeros(44, fortran_int)
dsave = np.zeros(29, np.float64)
task[:] = b'START'
for n_iter in range(7): # 7 steps required to reproduce error
f, g = objfun(x)
_lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr, pgtol, wa,
iwa, task, iprint, csave, lsave, isave, dsave, maxls)
assert (x <= upper_bnd).all() and (x >= low_bnd).all(), (
"_lbfgsb.setulb() stepped to a point outside of the bounds")
def bfgs_more_gutted(C, u, z, rho, x, n):
#max_iter = 15000
max_iter = 3000
nbd = zeros(n, int32)
low_bnd = zeros(n, float64)
upper_bnd = zeros(n, float64)
x = array(x, float64)
f = 0.0
g = zeros((n, ), float64)
wa = zeros(20 * n + 5 * n + 1180, float64)
iwa = zeros(3 * n, int32)
task = zeros(1, 'S60')
csave = zeros(1, 'S60')
lsave = zeros(4, int32)
isave = zeros(44, int32)
dsave = zeros(29, float64)
task[:] = 'START'
n_iterations = 0
while 1:
setulb(10, x, low_bnd, upper_bnd, nbd, f, g, 1e7, 1e-5, wa, iwa, task,
-1, csave, lsave, isave, dsave, 20)
task_str = task.tostring()
if task_str.startswith(b'FG'):
v = sub(add(x, u), z)
eCx = exp(dot(C, x))
rhov = rho * v
f = umr_sum(log1p(eCx), None, None, None, False) + 0.5 * umr_sum(
mul(rhov, v), None, None, None, False)
g = dot(C.T, div(eCx, (add(1, eCx)))) + rhov
elif task_str.startswith(b'NE'):
n_iterations += 1
if n_iterations >= max_iter:
return x
else:
return x
def fmin_l_bfgs_b(func, x0, fprime=None, args=(),
approx_grad=0,
bounds=None, m=10, factr=1e7, pgtol=1e-5,
epsilon=1e-8,
iprint=-1, maxfun=15000,
callback = None, maxstep=0.2, scale_steps=True): # added for AOF
"""
Minimize a function func using the L-BFGS-B algorithm.
Arguments:
func -- function to minimize. Called as func(x, *args)
x0 -- initial guess to minimum
fprime -- gradient of func. If None, then func returns the function
value and the gradient ( f, g = func(x, *args) ), unless
approx_grad is True then func returns only f.
Called as fprime(x, *args)
args -- arguments to pass to function
approx_grad -- if true, approximate the gradient numerically and func returns
only function value.
bounds -- a list of (min, max) pairs for each element in x, defining
the bounds on that parameter. Use None for one of min or max
when there is no bound in that direction
m -- the maximum number of variable metric corrections
used to define the limited memory matrix. (the limited memory BFGS
method does not store the full hessian but uses this many terms in an
approximation to it).
factr -- The iteration stops when
(f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr*epsmch
where epsmch is the machine precision, which is automatically
generated by the code. Typical values for factr: 1e12 for
low accuracy; 1e7 for moderate accuracy; 10.0 for extremely
high accuracy.
pgtol -- The iteration will stop when
max{|proj g_i | i = 1, ..., n} <= pgtol
where pg_i is the ith component of the projected gradient.
epsilon -- step size used when approx_grad is true, for numerically
calculating the gradient
iprint -- controls the frequency of output. <0 means no output.
maxfun -- maximum number of function evaluations.
Returns:
x, f, d = fmin_lbfgs_b(func, x0, ...)
x -- position of the minimum
f -- value of func at the minimum
d -- dictionary of information from routine
d['warnflag'] is
0 if converged,
1 if too many function evaluations,
2 if stopped for another reason, given in d['task']
d['grad'] is the gradient at the minimum (should be 0 ish)
d['funcalls'] is the number of function calls made.
License of L-BFGS-B (Fortran code)
==================================
The version included here (in fortran code) is 2.1 (released in 1997). It was
written by Ciyou Zhu, Richard Byrd, and Jorge Nocedal <*****@*****.**>. It
carries the following condition for use:
This software is freely available, but we expect that all publications
describing work using this software , or all commercial products using it,
quote at least one of the references given below.
References
* R. H. Byrd, P. Lu and J. Nocedal. A Limited Memory Algorithm for Bound
Constrained Optimization, (1995), SIAM Journal on Scientific and
Statistical Computing , 16, 5, pp. 1190-1208.
* C. Zhu, R. H. Byrd and J. Nocedal. L-BFGS-B: Algorithm 778: L-BFGS-B,
FORTRAN routines for large scale bound constrained optimization (1997),
ACM Transactions on Mathematical Software, Vol 23, Num. 4, pp. 550 - 560.
See also:
scikits.openopt, which offers a unified syntax to call this and other solvers
fmin, fmin_powell, fmin_cg,
fmin_bfgs, fmin_ncg -- multivariate local optimizers
leastsq -- nonlinear least squares minimizer
fmin_l_bfgs_b, fmin_tnc,
fmin_cobyla -- constrained multivariate optimizers
anneal, brute -- global optimizers
fminbound, brent, golden, bracket -- local scalar minimizers
fsolve -- n-dimenstional root-finding
brentq, brenth, ridder, bisect, newton -- one-dimensional root-finding
fixed_point -- scalar fixed-point finder
"""
n = len(x0)
if bounds is None:
bounds = [(None,None)] * n
if len(bounds) != n:
raise ValueError('length of x0 != length of bounds')
if approx_grad:
def func_and_grad(x):
f = func(x, *args)
g = approx_fprime(x, func, epsilon, *args)
return f, g
elif fprime is None:
def func_and_grad(x):
f, g = func(x, *args)
return f, g
else:
def func_and_grad(x):
f = func(x, *args)
g = fprime(x, *args)
return f, g
nbd = zeros((n,), int32)
low_bnd = zeros((n,), float64)
upper_bnd = zeros((n,), float64)
bounds_map = {(None, None): 0,
(1, None) : 1,
(1, 1) : 2,
(None, 1) : 3}
for i in range(0, n):
l,u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
x = array(x0, float64)
f = array(0.0, float64)
g = zeros((n,), float64)
wa = zeros((2*m*n+4*n + 12*m**2 + 12*m,), float64)
iwa = zeros((3*n,), int32)
task = zeros(1, 'S60')
csave = zeros(1,'S60')
lsave = zeros((4,), int32)
isave = zeros((44,), int32)
dsave = zeros((29,), float64)
task[:] = 'START'
global n_function_evals
n_function_evals = 0
while 1:
prevx = x.copy()
# see http://www.math.unm.edu/~vageli/courses/Ma579/lbfgsb.f
_lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
pgtol, wa, iwa, task, iprint, csave, lsave,
isave, dsave)
# scale step size, added for AOF
# A question that one might ask is: if the steps are always scaled,
# even during the line search, wont this cause problems if points on
# the line search are co-linear. The answer is: that subsequent steps
# in the line search, as generated by setulb(), never seem to be
# co-linear.
if scale_steps:
step = x - prevx
size = sqrt(dot(step, step))
#print "step size",size
if size > maxstep:
#print "*********** scaling step size"
step = step / size * maxstep
x = prevx + step
task_str = task.tostring()
#print "task_str",task_str
if task_str.startswith('FG'):
# minimization routine wants f and g at the current x
n_function_evals += 1
# Overwrite f and g:
#print "step x", str(x)
f, g = func_and_grad(x)
# print "Line searching, current f =", f
elif task_str.startswith('NEW_X'):
# added for aOF
print "g_max", g.max(), "pgtol", pgtol
#print "n_function_evals",n_function_evals
if callable(callback):
callback(x)
# new iteration
if n_function_evals > maxfun:
task[:] = 'STOP: TOTAL NO. of f AND g EVALUATIONS EXCEEDS LIMIT'
else:
break
task_str = task.tostring().strip('\x00').strip()
if task_str.startswith('CONV'):
warnflag = 0
elif n_function_evals > maxfun:
warnflag = 1
else:
warnflag = 2
d = {'grad' : g,
'task' : task_str,
'funcalls' : n_function_evals,
'warnflag' : warnflag
}
return x, f, d
def lbfgsb(func, x0,
bounds=None, m=10, factr=1e7, pgtol=1e-5, maxfun=100):
"""
Minimize a function func using the L-BFGS-B algorithm.
Arguments:
func -- function to minimize. Called as func(x, *args)
x0 -- initial guess to minimum
bounds -- a list of (min, max) pairs for each element in x, defining
the bounds on that parameter. Use None for one of min or max
when there is no bound in that direction
m -- the maximum number of variable metric corrections
used to define the limited memory matrix. (the limited memory BFGS
method does not store the full hessian but uses this many terms in an
approximation to it).
factr -- The iteration stops when
(f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr*epsmch
where epsmch is the machine precision, which is automatically
generated by the code. Typical values for factr: 1e12 for
low accuracy; 1e7 for moderate accuracy; 10.0 for extremely
high accuracy.
pgtol -- The iteration will stop when
max{|proj g_i | i = 1, ..., n} <= pgtol
where pg_i is the ith component of the projected gradient.
maxfun -- maximum number of function evaluations.
License of L-BFGS-B (Fortran code)
==================================
The version included here (in fortran code) is 2.1 (released in 1997). It was
written by Ciyou Zhu, Richard Byrd, and Jorge Nocedal <*****@*****.**>. It
carries the following condition for use:
This software is freely available, but we expect that all publications
describing work using this software , or all commercial products using it,
quote at least one of the references given below.
References
* R. H. Byrd, P. Lu and J. Nocedal. A Limited Memory Algorithm for Bound
Constrained Optimization, (1995), SIAM Journal on Scientific and
Statistical Computing , 16, 5, pp. 1190-1208.
* C. Zhu, R. H. Byrd and J. Nocedal. L-BFGS-B: Algorithm 778: L-BFGS-B,
FORTRAN routines for large scale bound constrained optimization (1997),
ACM Transactions on Mathematical Software, Vol 23, Num. 4, pp. 550 - 560.
"""
n = len(x0)
if bounds is None: bounds = [(None,None)] * n
if len(bounds) != n: raise ValueError('length of x0 != length of bounds')
nbd = np.zeros((n,), np.int32)
low_bnd = np.zeros((n,), np.float64)
upper_bnd = np.zeros((n,), np.float64)
bounds_map = {(None, None): 0, (1, None): 1, (1, 1): 2, (None, 1): 3}
for i in range(0, n):
l,u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
wa = np.zeros((2*m*n + 4*n + 12*m**2 + 12*m,), np.float64)
iwa = np.zeros((3*n,), np.int32)
task = np.zeros(1, 'S60')
csave = np.zeros(1, 'S60')
lsave = np.zeros((4, ), np.int32)
isave = np.zeros((44,), np.int32)
dsave = np.zeros((29,), np.float64)
# allocate space for our path.
ns = np.empty(maxfun+1, np.int32)
xs = np.empty((maxfun+1, n), np.float64)
fs = np.empty(maxfun+1, np.float64)
gs = np.empty((maxfun+1, n), np.float64)
# initialize the first step.
x = np.array(x0, np.float64)
f, g = func(x)
ns[0], xs[0], fs[0], gs[0] = 0, x, f, g
i = 1
numevals = 0
task[:] = 'START'
while 1:
_lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
pgtol, wa, iwa, task, -1, csave, lsave,
isave, dsave)
task_str = task.tostring()
if task_str.startswith('FG'):
# minimization routine wants f and g at the current x
numevals += 1
f, g = func(x)
elif task_str.startswith('NEW_X'):
# new iteration
ns[i], xs[i], fs[i], gs[i] = numevals, x, f, g
i += 1
if numevals > maxfun:
task[:] = 'STOP: TOTAL NO. of f AND g EVALUATIONS EXCEEDS LIMIT'
else:
break
ns.resize(i)
xs.resize((i, n))
fs.resize(i)
gs.resize((i, n))
return x, f, dict(numevals=ns, x=xs, f=fs, g=gs)
def _minimize_lbfgsb_timeup(
fun, x0, args=(), jac=None, bounds=None,
disp=None, maxcor=10, ftol=2.2204460492503131e-09,
gtol=1e-5, eps=1e-8, maxfun=15000, maxiter=15000,
iprint=-1, callback=None, maxls=20,
t0=None, timeup=float("inf"),
**unknown_options): # JFF: added time-up check
"""
Minimize a scalar function of one or more variables using the L-BFGS-B
algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxcor : int
The maximum number of variable metric corrections used to
define the limited memory matrix. (The limited memory BFGS
method does not store the full hessian but uses this many terms
in an approximation to it.)
factr : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr * eps``, where ``eps``
is the machine precision, which is automatically generated by
the code. Typical values for `factr` are: 1e12 for low
accuracy; 1e7 for moderate accuracy; 10.0 for extremely high
accuracy.
ftol : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol``.
gtol : float
The iteration will stop when ``max{|proj g_i | i = 1, ..., n}
<= gtol`` where ``pg_i`` is the i-th component of the
projected gradient.
eps : float
Step size used for numerical approximation of the jacobian.
disp : int
Set to True to print convergence messages.
maxfun : int
Maximum number of function evaluations.
maxiter : int
Maximum number of iterations.
maxls : int, optional
Maximum number of line search steps (per iteration). Default is 20.
"""
_check_unknown_options(unknown_options)
m = maxcor
epsilon = eps
pgtol = gtol
factr = ftol / np.finfo(float).eps
x0 = asarray(x0).ravel()
n, = x0.shape
if bounds is None:
bounds = [(None, None)] * n
if len(bounds) != n:
raise ValueError('length of x0 != length of bounds')
# unbounded variables must use None, not +-inf, for optimizer to work properly
bounds = [(None if l == -np.inf else l, None if u == np.inf else u) for l, u in bounds]
if disp is not None:
if disp == 0:
iprint = -1
else:
iprint = disp
n_function_evals, fun = wrap_function(fun, ())
if jac is None:
def func_and_grad(x):
f = fun(x, *args)
g = _approx_fprime_helper(x, fun, epsilon, args=args, f0=f)
return f, g
else:
def func_and_grad(x):
f = fun(x, *args)
g = jac(x, *args)
return f, g
nbd = zeros(n, int32)
low_bnd = zeros(n, float64)
upper_bnd = zeros(n, float64)
bounds_map = {(None, None): 0,
(1, None): 1,
(1, 1): 2,
(None, 1): 3}
for i in range(0, n):
l, u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
if not maxls > 0:
raise ValueError('maxls must be positive.')
x = array(x0, float64)
f = array(0.0, float64)
g = zeros((n,), float64)
wa = zeros(2*m*n + 5*n + 11*m*m + 8*m, float64)
iwa = zeros(3*n, int32)
task = zeros(1, 'S60')
csave = zeros(1, 'S60')
lsave = zeros(4, int32)
isave = zeros(44, int32)
dsave = zeros(29, float64)
task[:] = 'START'
n_iterations = 0
if t0 is None:
t0 = time.time()
time_profile.predicted_inner_loop_func2_duration = 0.0
while 1:
# x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
_lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
pgtol, wa, iwa, task, iprint, csave, lsave,
isave, dsave, maxls)
task_str = task.tostring()
# begin EB
curr_time = time.time()
predicted_inner_loop_func2_duration = (curr_time +
time_profile.maximize_inner_time_profile.mean +
time_profile.func2_time_profile.mean - t0)
if predicted_inner_loop_func2_duration > timeup: # JFF: added time-up check
task[:] = ('STOP: PREDICTED COMPUTATION TIME EXCEEDS LIMIT')
break
# end EB
if task_str.startswith(b'FG'):
# The minimization routine wants f and g at the current x.
# Note that interruptions due to maxfun are postponed
# until the completion of the current minimization iteration.
# Overwrite f and g:
f, g = func_and_grad(x)
elif task_str.startswith(b'NEW_X'):
# new iteration
if n_iterations > maxiter:
task[:] = 'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT'
elif n_function_evals[0] > maxfun:
task[:] = ('STOP: TOTAL NO. of f AND g EVALUATIONS '
'EXCEEDS LIMIT')
else:
n_iterations += 1
if callback is not None:
callback(x)
else:
break
time_profile.predicted_inner_loop_func2_duration = predicted_inner_loop_func2_duration
task_str = task.tostring().strip(b'\x00').strip()
if task_str.startswith(b'CONV'):
warnflag = 0
elif n_function_evals[0] > maxfun:
warnflag = 1
elif n_iterations > maxiter:
warnflag = 1
else:
warnflag = 2
# These two portions of the workspace are described in the mainlb
# subroutine in lbfgsb.f. See line 363.
s = wa[0: m*n].reshape(m, n)
y = wa[m*n: 2*m*n].reshape(m, n)
# See lbfgsb.f line 160 for this portion of the workspace.
# isave(31) = the total number of BFGS updates prior the current iteration;
n_bfgs_updates = isave[30]
n_corrs = min(n_bfgs_updates, maxcor)
hess_inv = LbfgsInvHessProduct(s[:n_corrs], y[:n_corrs])
return OptimizeResult(fun=f, jac=g, nfev=n_function_evals[0],
nit=n_iterations, status=warnflag, message=task_str,
x=x, success=(warnflag == 0), hess_inv=hess_inv)
def lbfgsb(func, x0, bounds=None, m=10, factr=1e7, pgtol=1e-5, maxfun=100):
"""
Minimize a function func using the L-BFGS-B algorithm.
Arguments:
func -- function to minimize. Called as func(x, *args)
x0 -- initial guess to minimum
bounds -- a list of (min, max) pairs for each element in x, defining
the bounds on that parameter. Use None for one of min or max
when there is no bound in that direction
m -- the maximum number of variable metric corrections
used to define the limited memory matrix. (the limited memory BFGS
method does not store the full hessian but uses this many terms in an
approximation to it).
factr -- The iteration stops when
(f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr*epsmch
where epsmch is the machine precision, which is automatically
generated by the code. Typical values for factr: 1e12 for
low accuracy; 1e7 for moderate accuracy; 10.0 for extremely
high accuracy.
pgtol -- The iteration will stop when
max{|proj g_i | i = 1, ..., n} <= pgtol
where pg_i is the ith component of the projected gradient.
maxfun -- maximum number of function evaluations.
License of L-BFGS-B (Fortran code)
==================================
The version included here (in fortran code) is 2.1 (released in 1997). It was
written by Ciyou Zhu, Richard Byrd, and Jorge Nocedal <*****@*****.**>. It
carries the following condition for use:
This software is freely available, but we expect that all publications
describing work using this software , or all commercial products using it,
quote at least one of the references given below.
References
* R. H. Byrd, P. Lu and J. Nocedal. A Limited Memory Algorithm for Bound
Constrained Optimization, (1995), SIAM Journal on Scientific and
Statistical Computing , 16, 5, pp. 1190-1208.
* C. Zhu, R. H. Byrd and J. Nocedal. L-BFGS-B: Algorithm 778: L-BFGS-B,
FORTRAN routines for large scale bound constrained optimization (1997),
ACM Transactions on Mathematical Software, Vol 23, Num. 4, pp. 550 - 560.
"""
n = len(x0)
if bounds is None: bounds = [(None, None)] * n
if len(bounds) != n: raise ValueError('length of x0 != length of bounds')
nbd = np.zeros((n, ), np.int32)
low_bnd = np.zeros((n, ), np.float64)
upper_bnd = np.zeros((n, ), np.float64)
bounds_map = {(None, None): 0, (1, None): 1, (1, 1): 2, (None, 1): 3}
for i in range(0, n):
l, u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
wa = np.zeros((2 * m * n + 4 * n + 12 * m**2 + 12 * m, ), np.float64)
iwa = np.zeros((3 * n, ), np.int32)
task = np.zeros(1, 'S60')
csave = np.zeros(1, 'S60')
lsave = np.zeros((4, ), np.int32)
isave = np.zeros((44, ), np.int32)
dsave = np.zeros((29, ), np.float64)
# allocate space for our path.
ns = np.empty(maxfun + 1, np.int32)
xs = np.empty((maxfun + 1, n), np.float64)
fs = np.empty(maxfun + 1, np.float64)
gs = np.empty((maxfun + 1, n), np.float64)
# initialize the first step.
x = np.array(x0, np.float64)
f, g = func(x)
ns[0], xs[0], fs[0], gs[0] = 0, x, f, g
i = 1
numevals = 0
task[:] = 'START'
while 1:
_lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr, pgtol, wa,
iwa, task, -1, csave, lsave, isave, dsave)
task_str = task.tostring()
if task_str.startswith('FG'):
# minimization routine wants f and g at the current x
numevals += 1
f, g = func(x)
elif task_str.startswith('NEW_X'):
# new iteration
ns[i], xs[i], fs[i], gs[i] = numevals, x, f, g
i += 1
if numevals > maxfun:
task[:] = 'STOP: TOTAL NO. of f AND g EVALUATIONS EXCEEDS LIMIT'
else:
break
ns.resize(i)
xs.resize((i, n))
fs.resize(i)
gs.resize((i, n))
return x, f, dict(numevals=ns, x=xs, f=fs, g=gs)
def batched_fmin_lbfgs_b(func, x0, num_batches, fprime=None, args=(),
bounds=None, m=10, factr=1e7, pgtol=1e-5,
epsilon=1e-8,
iprint=-1, maxiter=15000,
maxls=20):
"""A batch-aware L-BFGS-B implementation to minimize a loss function `f` given
an initial set of parameters `x0`.
Parameters
----------
func : function (x: array) -> array[M] (M = n_batches)
The function to minimize. The function should return an array of
size = `num_batches`
x0 : array
Starting parameters
fprime : function (x: array) -> array[M*n_params] (optional)
The gradient. Should return an array of derivatives for each
parameter over batches.
When omitted, uses Finite-differencing to estimate the gradient.
args : Tuple
Additional arguments to func and fprime
bounds : List[Tuple[float, float]]
Box-constrains on the parameters
m : int
L-BFGS parameter: number of previous arrays to store when
estimating inverse Hessian.
factr : float
Stopping criterion when function evaluation not progressing.
Stop when `|f(xk+1) - f(xk)| < factor*eps_mach`
where `eps_mach` is the machine precision
pgtol : float
Stopping criterion when gradient is sufficiently "flat".
Stop when |grad| < pgtol.
epsilon : float
Finite differencing step size when approximating `fprime`
iprint : int
-1 for no diagnostic info
n=1-100 for diagnostic info every n steps.
>100 for detailed diagnostic info
maxiter : int
Maximum number of L-BFGS iterations
maxls : int
Maximum number of line-search iterations.
"""
if has_scipy():
from scipy.optimize import _lbfgsb
else:
raise RuntimeError("Scipy is needed to run batched_fmin_lbfgs_b")
nvtx_range_push("LBFGS")
n = len(x0) // num_batches
if fprime is None:
def fprime_f(x):
return _fd_fprime(x, func, epsilon)
fprime = fprime_f
if bounds is None:
bounds = [(None, None)] * n
nbd = np.zeros(n, np.int32)
low_bnd = np.zeros(n, np.float64)
upper_bnd = np.zeros(n, np.float64)
bounds_map = {(None, None): 0,
(1, None): 1,
(1, 1): 2,
(None, 1): 3}
for i in range(0, n):
lb, ub = bounds[i]
if lb is not None:
low_bnd[i] = lb
lb = 1
if ub is not None:
upper_bnd[i] = ub
ub = 1
nbd[i] = bounds_map[lb, ub]
# working arrays needed by L-BFGS-B implementation in SciPy.
# One for each series
x = [np.copy(np.array(x0[ib*n:(ib+1)*n],
np.float64)) for ib in range(num_batches)]
f = [np.copy(np.array(0.0,
np.float64)) for ib in range(num_batches)]
g = [np.copy(np.zeros((n,), np.float64)) for ib in range(num_batches)]
wa = [np.copy(np.zeros(2*m*n + 5*n + 11*m*m + 8*m,
np.float64)) for ib in range(num_batches)]
iwa = [np.copy(np.zeros(3*n, np.int32)) for ib in range(num_batches)]
task = [np.copy(np.zeros(1, 'S60')) for ib in range(num_batches)]
csave = [np.copy(np.zeros(1, 'S60')) for ib in range(num_batches)]
lsave = [np.copy(np.zeros(4, np.int32)) for ib in range(num_batches)]
isave = [np.copy(np.zeros(44, np.int32)) for ib in range(num_batches)]
dsave = [np.copy(np.zeros(29, np.float64)) for ib in range(num_batches)]
for ib in range(num_batches):
task[ib][:] = 'START'
n_iterations = np.zeros(num_batches, dtype=np.int32)
converged = num_batches * [False]
warn_flag = np.zeros(num_batches)
while not all(converged):
nvtx_range_push("LBFGS-ITERATION")
for ib in range(num_batches):
if converged[ib]:
continue
_lbfgsb.setulb(m, x[ib],
low_bnd, upper_bnd,
nbd,
f[ib], g[ib],
factr, pgtol,
wa[ib], iwa[ib],
task[ib],
iprint,
csave[ib],
lsave[ib],
isave[ib],
dsave[ib],
maxls)
xk = np.concatenate(x)
fk = func(xk)
gk = fprime(xk)
for ib in range(num_batches):
if converged[ib]:
continue
task_str = task[ib].tostring()
task_str_strip = task[ib].tostring().strip(b'\x00').strip()
if task_str.startswith(b'FG'):
# needs function evalation
f[ib] = fk[ib]
g[ib] = gk[ib*n:(ib+1)*n]
elif task_str.startswith(b'NEW_X'):
n_iterations[ib] += 1
if n_iterations[ib] >= maxiter:
task[ib][:] = 'STOP: TOTAL NO. of ITERATIONS REACHED LIMIT'
elif task_str_strip.startswith(b'CONV'):
converged[ib] = True
warn_flag[ib] = 0
else:
converged[ib] = True
warn_flag[ib] = 2
continue
nvtx_range_pop()
xk = np.concatenate(x)
if iprint > 0:
print("CONVERGED in ({}-{}) iterations (|\\/f|={})".format(
np.min(n_iterations),
np.max(n_iterations),
np.linalg.norm(fprime(xk), np.inf)))
if (warn_flag > 0).any():
for ib in range(num_batches):
if warn_flag[ib] > 0:
print("WARNING: id={} convergence issue: {}".format(
ib, task[ib].tostring()))
nvtx_range_pop()
return xk, n_iterations, warn_flag
def _minimize_lbfgsb(fun,
x0,
args=(),
jac=None,
bounds=None,
disp=None,
maxcor=10,
ftol=2.2204460492503131e-09,
gtol=1e-5,
eps=1e-8,
maxfun=15000,
maxiter=15000,
maxtime=0,
iprint=-1,
callback=None,
**unknown_options):
"""
Minimize a scalar function of one or more variables using the L-BFGS-B
algorithm.
Options for the L-BFGS-B algorithm are:
disp : bool
Set to True to print convergence messages.
maxcor : int
The maximum number of variable metric corrections used to
define the limited memory matrix. (The limited memory BFGS
method does not store the full hessian but uses this many terms
in an approximation to it.)
factr : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr * eps``, where ``eps``
is the machine precision, which is automatically generated by
the code. Typical values for `factr` are: 1e12 for low
accuracy; 1e7 for moderate accuracy; 10.0 for extremely high
accuracy.
gtol : float
The iteration will stop when ``max{|proj g_i | i = 1, ..., n}
<= gtol`` where ``pg_i`` is the i-th component of the
projected gradient.
eps : float
Step size used for numerical approximation of the jacobian.
disp : int
Set to True to print convergence messages.
maxfun : int
Maximum number of function evaluations.
maxiter : int
Maximum number of iterations.
This function is called by the `minimize` function with
`method=L-BFGS-B`. It is not supposed to be called directly.
"""
_check_unknown_options(unknown_options)
m = maxcor
epsilon = eps
pgtol = gtol
factr = ftol / np.finfo(float).eps
x0 = asarray(x0).ravel()
n, = x0.shape
if bounds is None:
bounds = [(None, None)] * n
if len(bounds) != n:
raise ValueError('length of x0 != length of bounds')
if disp is not None:
if disp == 0:
iprint = -1
else:
iprint = disp
if jac is None:
def func_and_grad(x):
f = fun(x, *args)
g = approx_fprime(x, fun, epsilon, *args)
return f, g
else:
def func_and_grad(x):
f = fun(x, *args)
g = jac(x, *args)
return f, g
nbd = zeros(n, int32)
low_bnd = zeros(n, float64)
upper_bnd = zeros(n, float64)
bounds_map = {(None, None): 0, (1, None): 1, (1, 1): 2, (None, 1): 3}
for i in range(0, n):
l, u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
x = array(x0, float64)
f = array(0.0, float64)
g = zeros((n, ), float64)
wa = zeros(2 * m * n + 5 * n + 11 * m * m + 8 * m, float64)
iwa = zeros(3 * n, int32)
task = zeros(1, 'S60')
csave = zeros(1, 'S60')
lsave = zeros(4, int32)
isave = zeros(44, int32)
dsave = zeros(29, float64)
task[:] = 'START'
n_function_evals = 0
n_iterations = 0
start_time = time.clock()
while 1:
# x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
_lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr, pgtol, wa,
iwa, task, iprint, csave, lsave, isave, dsave)
task_str = task.tostring()
if task_str.startswith(asbytes('FG')):
if n_function_evals > maxfun:
task[:] = 'STOP: TOTAL NO. of f AND g EVALUATIONS EXCEEDS LIMIT'
else:
# minimization routine wants f and g at the current x
n_function_evals += 1
# Overwrite f and g:
f, g = func_and_grad(x)
elif task_str.startswith(asbytes('NEW_X')):
# new iteration
if n_iterations > maxiter:
task[:] = 'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT'
elif (maxtime > 0) and ((time.clock() - start_time) > maxtime):
task[:] = 'STOP: TOTAL TIME EXCEEDS LIMIT'
else:
n_iterations += 1
if callback is not None:
callback(x)
else:
break
task_str = task.tostring().strip(asbytes('\x00')).strip()
if task_str.startswith(asbytes('CONV')):
warnflag = 0
elif n_function_evals > maxfun:
warnflag = 1
elif n_iterations > maxiter:
warnflag = 1
else:
warnflag = 2
return Result(fun=f,
jac=g,
nfev=n_function_evals,
nit=n_iterations,
status=warnflag,
message=task_str,
x=x,
success=(warnflag == 0))
def _minimize_lbfgsb_multi(fun, x0, args=(), jac=None, bounds=None,
disp=None, maxcor=10, ftol=2.2204460492503131e-09,
gtol=1e-5, eps=1e-8, maxfun=15000, maxiter=15000,
iprint=-1, callback=None, maxls=20, **unknown_options):
"""
Minimize a scalar function of one or more variables using the L-BFGS-B
algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxcor : int
The maximum number of variable metric corrections used to
define the limited memory matrix. (The limited memory BFGS
method does not store the full hessian but uses this many terms
in an approximation to it.)
ftol : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol``.
gtol : float
The iteration will stop when ``max{|proj g_i | i = 1, ..., n}
<= gtol`` where ``pg_i`` is the i-th component of the
projected gradient.
eps : float
Step size used for numerical approximation of the jacobian.
disp : int
Set to True to print convergence messages.
maxfun : int
Maximum number of function evaluations.
maxiter : int
Maximum number of iterations.
maxls : int, optional
Maximum number of line search steps (per iteration). Default is 20.
Notes
-----
The option `ftol` is exposed via the `scipy.optimize.minimize` interface,
but calling `scipy.optimize.fmin_l_bfgs_b` directly exposes `factr`. The
relationship between the two is ``ftol = factr * numpy.finfo(float).eps``.
I.e., `factr` multiplies the default machine floating-point precision to
arrive at `ftol`.
"""
_check_unknown_options(unknown_options)
m = maxcor
epsilon = eps
pgtol = gtol
factr = ftol / np.finfo(float).eps
k,n = x0.shape
# x0 = [asarray(x).ravel() for x in x0]
# n, = x0.shape
if bounds is None:
bounds = [(None, None)] * n
if len(bounds) != n:
raise ValueError('length of x0 != length of bounds')
# unbounded variables must use None, not +-inf, for optimizer to work properly
bounds = [(None if l == -np.inf else l, None if u == np.inf else u) for l, u in bounds]
if disp is not None:
if disp == 0:
iprint = -1
else:
iprint = disp
if jac is None:
def func_and_grad(x):
# f = fun(x, *args)
f, g = _approx_fprime_helper(x, fun, epsilon, args=args)
return f, g
else:
def func_and_grad(x):
f = fun(x, *args)
g = jac(x, *args)
return f, g
nbd = zeros(n, int32)
low_bnd = zeros(n, float64)
upper_bnd = zeros(n, float64)
bounds_map = {(None, None): 0,
(1, None): 1,
(1, 1): 2,
(None, 1): 3}
for i in range(0, n):
l, u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
if not maxls > 0:
raise ValueError('maxls must be positive.')
# x = [array(x0_, float64) for x0_ in x0]
X = x0.copy()
f = [array(0.0, float64) for _ in range(k)]
g = [zeros((n,), float64) for _ in range(k)]
wa = [zeros(2*m*n + 5*n + 11*m*m + 8*m, float64) for _ in range(k)]
iwa = [zeros(3*n, int32) for _ in range(k)]
task = [zeros(1, 'S60') for _ in range(k)]
csave = [zeros(1, 'S60') for _ in range(k)]
lsave = [zeros(4, int32) for _ in range(k)]
isave = [zeros(44, int32) for _ in range(k)]
dsave = [zeros(29, float64) for _ in range(k)]
n_function_evals = 0
for i in range(k):
task[i][:] = 'START'
n_iterations = [0 for _ in range(k)]
k_running = [i for i in range(k)]
while 1:
# x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
request_index = []
# run all instance until they request a new point
# X contains only points for the remaining instances
for i in k_running:
while 1:
_lbfgsb.setulb(m, X[i], low_bnd, upper_bnd, nbd, f[i], g[i], factr,
pgtol, wa[i], iwa[i], task[i], iprint, csave[i], lsave[i],
isave[i], dsave[i], maxls)
task_str = task[i].tostring()
if task_str.startswith(b'FG'):
# The minimization routine wants f and g at the current x.
# Note that interruptions due to maxfun are postponed
# until the completion of the current minimization iteration.
# Overwrite f and g:
request_index.append(i)
break
elif task_str.startswith(b'NEW_X'):
# new iteration
if n_iterations[i] > maxiter:
task[i][:] = 'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT'
# break
elif n_function_evals > maxfun:
task[i][:] = ('STOP: TOTAL NO. of f AND g EVALUATIONS '
'EXCEEDS LIMIT')
# break
else:
n_iterations[i] += 1
# if callback is not None:
# callback(x)
else:
break
k_running = request_index
if len(k_running) == 0:
break
F,G = func_and_grad(X[request_index])
n_function_evals += 1
for ff,gg,i in zip(F,G,k_running):
f[i] = ff
g[i] = gg
warnflag = []
task_str = [t.tostring().strip(b'\x00').strip() for t in task]
for i,t in enumerate(task_str):
if t.startswith(b'CONV'):
warnflag.append(0)
elif n_function_evals > maxfun:
warnflag.append(1)
elif n_iterations[i] > maxiter:
warnflag.append(2)
else:
warnflag.append(3)
# These two portions of the workspace are described in the mainlb
# subroutine in lbfgsb.f. See line 363.
# s = [wa[i][0: m*n].reshape(m, n) for i in range(k)]
# y = [wa[i][m*n: 2*m*n].reshape(m, n) for i in range(k)]
# See lbfgsb.f line 160 for this portion of the workspace.
# isave(31) = the total number of BFGS updates prior the current iteration;
# n_bfgs_updates = isave[30]
# n_corrs = min(n_bfgs_updates, maxcor)
# hess_inv = LbfgsInvHessProduct(s[:n_corrs], y[:n_corrs])
return OptimizeResult(fun=f, jac=g, nfev=n_function_evals,
nit=n_iterations, status=warnflag, message=task_str,
x=X, success=(sum(warnflag) == 0))
def _minimize_lbfgsb(fun, x0, args=(), jac=None, bounds=None,
disp=None, maxcor=10, ftol=2.2204460492503131e-09,
gtol=1e-5, eps=1e-8, maxfun=15000, maxiter=15000, maxtime=0,
iprint=-1, callback=None, **unknown_options):
"""
Minimize a scalar function of one or more variables using the L-BFGS-B
algorithm.
Options for the L-BFGS-B algorithm are:
disp : bool
Set to True to print convergence messages.
maxcor : int
The maximum number of variable metric corrections used to
define the limited memory matrix. (The limited memory BFGS
method does not store the full hessian but uses this many terms
in an approximation to it.)
factr : float
The iteration stops when ``(f^k -
f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr * eps``, where ``eps``
is the machine precision, which is automatically generated by
the code. Typical values for `factr` are: 1e12 for low
accuracy; 1e7 for moderate accuracy; 10.0 for extremely high
accuracy.
gtol : float
The iteration will stop when ``max{|proj g_i | i = 1, ..., n}
<= gtol`` where ``pg_i`` is the i-th component of the
projected gradient.
eps : float
Step size used for numerical approximation of the jacobian.
disp : int
Set to True to print convergence messages.
maxfun : int
Maximum number of function evaluations.
maxiter : int
Maximum number of iterations.
This function is called by the `minimize` function with
`method=L-BFGS-B`. It is not supposed to be called directly.
"""
_check_unknown_options(unknown_options)
m = maxcor
epsilon = eps
pgtol = gtol
factr = ftol / np.finfo(float).eps
x0 = asarray(x0).ravel()
n, = x0.shape
if bounds is None:
bounds = [(None, None)] * n
if len(bounds) != n:
raise ValueError('length of x0 != length of bounds')
if disp is not None:
if disp == 0:
iprint = -1
else:
iprint = disp
if jac is None:
def func_and_grad(x):
f = fun(x, *args)
g = approx_fprime(x, fun, epsilon, *args)
return f, g
else:
def func_and_grad(x):
f = fun(x, *args)
g = jac(x, *args)
return f, g
nbd = zeros(n, int32)
low_bnd = zeros(n, float64)
upper_bnd = zeros(n, float64)
bounds_map = {
(None, None): 0,
(1, None): 1,
(1, 1): 2,
(None, 1): 3
}
for i in range(0, n):
l, u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
x = array(x0, float64)
f = array(0.0, float64)
g = zeros((n,), float64)
wa = zeros(2 * m * n + 5 * n + 11 * m * m + 8 * m, float64)
iwa = zeros(3 * n, int32)
task = zeros(1, 'S60')
csave = zeros(1, 'S60')
lsave = zeros(4, int32)
isave = zeros(44, int32)
dsave = zeros(29, float64)
task[:] = 'START'
n_function_evals = 0
n_iterations = 0
start_time = time.clock()
while 1:
# x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
_lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
pgtol, wa, iwa, task, iprint, csave, lsave,
isave, dsave)
task_str = task.tostring()
if task_str.startswith(asbytes('FG')):
if n_function_evals > maxfun:
task[
:] = 'STOP: TOTAL NO. of f AND g EVALUATIONS EXCEEDS LIMIT'
else:
# minimization routine wants f and g at the current x
n_function_evals += 1
# Overwrite f and g:
f, g = func_and_grad(x)
elif task_str.startswith(asbytes('NEW_X')):
# new iteration
if n_iterations > maxiter:
task[:] = 'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT'
elif (maxtime > 0) and ((time.clock() - start_time) > maxtime):
task[:] = 'STOP: TOTAL TIME EXCEEDS LIMIT'
else:
n_iterations += 1
if callback is not None:
callback(x)
else:
break
task_str = task.tostring().strip(asbytes('\x00')).strip()
if task_str.startswith(asbytes('CONV')):
warnflag = 0
elif n_function_evals > maxfun:
warnflag = 1
elif n_iterations > maxiter:
warnflag = 1
else:
warnflag = 2
return Result(fun=f, jac=g, nfev=n_function_evals, nit=n_iterations,
status=warnflag, message=task_str, x=x,
success=(warnflag == 0))
def fmin_l_bfgs_b(func,
x0,
fprime=None,
args=(),
approx_grad=0,
bounds=None,
m=10,
factr=1e7,
pgtol=1e-5,
epsilon=1e-8,
iprint=-1,
maxfun=15000,
callback=None,
maxstep=0.2,
scale_steps=True): # added for AOF
"""
Minimize a function func using the L-BFGS-B algorithm.
Arguments:
func -- function to minimize. Called as func(x, *args)
x0 -- initial guess to minimum
fprime -- gradient of func. If None, then func returns the function
value and the gradient ( f, g = func(x, *args) ), unless
approx_grad is True then func returns only f.
Called as fprime(x, *args)
args -- arguments to pass to function
approx_grad -- if true, approximate the gradient numerically and func returns
only function value.
bounds -- a list of (min, max) pairs for each element in x, defining
the bounds on that parameter. Use None for one of min or max
when there is no bound in that direction
m -- the maximum number of variable metric corrections
used to define the limited memory matrix. (the limited memory BFGS
method does not store the full hessian but uses this many terms in an
approximation to it).
factr -- The iteration stops when
(f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= factr*epsmch
where epsmch is the machine precision, which is automatically
generated by the code. Typical values for factr: 1e12 for
low accuracy; 1e7 for moderate accuracy; 10.0 for extremely
high accuracy.
pgtol -- The iteration will stop when
max{|proj g_i | i = 1, ..., n} <= pgtol
where pg_i is the ith component of the projected gradient.
epsilon -- step size used when approx_grad is true, for numerically
calculating the gradient
iprint -- controls the frequency of output. <0 means no output.
maxfun -- maximum number of function evaluations.
Returns:
x, f, d = fmin_lbfgs_b(func, x0, ...)
x -- position of the minimum
f -- value of func at the minimum
d -- dictionary of information from routine
d['warnflag'] is
0 if converged,
1 if too many function evaluations,
2 if stopped for another reason, given in d['task']
d['grad'] is the gradient at the minimum (should be 0 ish)
d['funcalls'] is the number of function calls made.
License of L-BFGS-B (Fortran code)
==================================
The version included here (in fortran code) is 2.1 (released in 1997). It was
written by Ciyou Zhu, Richard Byrd, and Jorge Nocedal <*****@*****.**>. It
carries the following condition for use:
This software is freely available, but we expect that all publications
describing work using this software , or all commercial products using it,
quote at least one of the references given below.
References
* R. H. Byrd, P. Lu and J. Nocedal. A Limited Memory Algorithm for Bound
Constrained Optimization, (1995), SIAM Journal on Scientific and
Statistical Computing , 16, 5, pp. 1190-1208.
* C. Zhu, R. H. Byrd and J. Nocedal. L-BFGS-B: Algorithm 778: L-BFGS-B,
FORTRAN routines for large scale bound constrained optimization (1997),
ACM Transactions on Mathematical Software, Vol 23, Num. 4, pp. 550 - 560.
See also:
scikits.openopt, which offers a unified syntax to call this and other solvers
fmin, fmin_powell, fmin_cg,
fmin_bfgs, fmin_ncg -- multivariate local optimizers
leastsq -- nonlinear least squares minimizer
fmin_l_bfgs_b, fmin_tnc,
fmin_cobyla -- constrained multivariate optimizers
anneal, brute -- global optimizers
fminbound, brent, golden, bracket -- local scalar minimizers
fsolve -- n-dimenstional root-finding
brentq, brenth, ridder, bisect, newton -- one-dimensional root-finding
fixed_point -- scalar fixed-point finder
"""
n = len(x0)
if bounds is None:
bounds = [(None, None)] * n
if len(bounds) != n:
raise ValueError('length of x0 != length of bounds')
if approx_grad:
def func_and_grad(x):
f = func(x, *args)
g = approx_fprime(x, func, epsilon, *args)
return f, g
elif fprime is None:
def func_and_grad(x):
f, g = func(x, *args)
return f, g
else:
def func_and_grad(x):
f = func(x, *args)
g = fprime(x, *args)
return f, g
nbd = zeros((n, ), int32)
low_bnd = zeros((n, ), float64)
upper_bnd = zeros((n, ), float64)
bounds_map = {(None, None): 0, (1, None): 1, (1, 1): 2, (None, 1): 3}
for i in range(0, n):
l, u = bounds[i]
if l is not None:
low_bnd[i] = l
l = 1
if u is not None:
upper_bnd[i] = u
u = 1
nbd[i] = bounds_map[l, u]
x = array(x0, float64)
f = array(0.0, float64)
g = zeros((n, ), float64)
wa = zeros((2 * m * n + 4 * n + 12 * m**2 + 12 * m, ), float64)
iwa = zeros((3 * n, ), int32)
task = zeros(1, 'S60')
csave = zeros(1, 'S60')
lsave = zeros((4, ), int32)
isave = zeros((44, ), int32)
dsave = zeros((29, ), float64)
task[:] = 'START'
global n_function_evals
n_function_evals = 0
while 1:
prevx = x.copy()
# see http://www.math.unm.edu/~vageli/courses/Ma579/lbfgsb.f
_lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr, pgtol, wa,
iwa, task, iprint, csave, lsave, isave, dsave)
# scale step size, added for AOF
# A question that one might ask is: if the steps are always scaled,
# even during the line search, wont this cause problems if points on
# the line search are co-linear. The answer is: that subsequent steps
# in the line search, as generated by setulb(), never seem to be
# co-linear.
if scale_steps:
step = x - prevx
size = sqrt(dot(step, step))
#print "step size",size
if size > maxstep:
#print "*********** scaling step size"
step = step / size * maxstep
x = prevx + step
task_str = task.tostring()
#print "task_str",task_str
if task_str.startswith('FG'):
# minimization routine wants f and g at the current x
n_function_evals += 1
# Overwrite f and g:
#print "step x", str(x)
f, g = func_and_grad(x)
# print "Line searching, current f =", f
elif task_str.startswith('NEW_X'):
# added for aOF
print "g_max", g.max(), "pgtol", pgtol
#print "n_function_evals",n_function_evals
if callable(callback):
callback(x)
# new iteration
if n_function_evals > maxfun:
task[:] = 'STOP: TOTAL NO. of f AND g EVALUATIONS EXCEEDS LIMIT'
else:
break
task_str = task.tostring().strip('\x00').strip()
if task_str.startswith('CONV'):
warnflag = 0
elif n_function_evals > maxfun:
warnflag = 1
else:
warnflag = 2
d = {
'grad': g,
'task': task_str,
'funcalls': n_function_evals,
'warnflag': warnflag
}
return x, f, d