def Uniformcheck(ds):
ds = sorted(ds, key=lambda x: x.f[0], reverse=False)
d = []
for i, dpi in enumerate(ds):
if i != 0:
d.append(vecnorm(dpi.f - ds[i - 1].f))
# d = []
# for i, dpi in enumerate(ds):
# di = 10
# for j, dpj in enumerate(ds):
# if i != j:
# dt = vecnorm(dpi.f - dpj.f)
# if dt < di:
# di = dt
# d.append(di)
sum = 0
for di in d:
sum += di
td = sum / len(ds)
sum2 = 0
for di in d:
sum2 += (di / td)**2
return np.sqrt(sum2 / (len(ds) - 1))
def loopThing(inputParams):
for key in inputParams:
exec(key + " = inputParams['" + key + "']")
pk = -numpy.dot(Hk,gfk)
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
linesearch.line_search(f,myfprime,xk,pk,gfk,
old_fval,old_old_fval)
if alpha_k is None: # line search failed try different one.
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
line_search(f,myfprime,xk,pk,gfk,
old_fval,old_old_fval)
if alpha_k is None:
# This line search also failed to find a better solution.
warnflag = 2
outputParams = getOutputParams(inputParams, locals().items())
return outputParams
xkp1 = xk + alpha_k * pk
if retall:
allvecs.append(xkp1)
sk = xkp1 - xk
xk = xkp1
if gfkp1 is None:
gfkp1 = myfprime(xkp1)
yk = gfkp1 - gfk
gfk = gfkp1
if callback is not None:
callback(xk)
k += 1
gnorm = vecnorm(gfk,ord=norm)
if (gnorm <= gtol):
outputParams = getOutputParams(inputParams, locals().items())
return outputParams
try: # this was handled in numeric, let it remaines for more safety
rhok = 1.0 / (numpy.dot(yk,sk))
except ZeroDivisionError:
rhok = 1000.0
print "Divide-by-zero encountered: rhok assumed large"
if numpy.isinf(rhok): # this is patch for numpy
rhok = 1000.0
print "Divide-by-zero encountered: rhok assumed large"
A1 = I - sk[:,numpy.newaxis] * yk[numpy.newaxis,:] * rhok
A2 = I - yk[:,numpy.newaxis] * sk[numpy.newaxis,:] * rhok
Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
* sk[numpy.newaxis,:]
outputParams = getOutputParams(inputParams, locals().items())
return outputParams
def loopThing_BFGS(inputParams):
for key in inputParams:
exec(key + " = inputParams['" + key + "']")
pk = -numpy.dot(Hk, gfk)
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
linesearch.line_search(f,myfprime,xk,pk,gfk,
old_fval,old_old_fval)
if alpha_k is None: # line search failed try different one.
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
line_search(f,myfprime,xk,pk,gfk,
old_fval,old_old_fval)
if alpha_k is None:
# This line search also failed to find a better solution.
warnflag = 2
outputParams = getOutputParams(inputParams, list(locals().items()))
return outputParams
xkp1 = xk + alpha_k * pk
if retall:
allvecs.append(xkp1)
sk = xkp1 - xk
xk = xkp1
if gfkp1 is None:
gfkp1 = myfprime(xkp1)
yk = gfkp1 - gfk
gfk = gfkp1
if callback is not None:
callback(xk)
k += 1
gnorm = vecnorm(gfk, ord=norm)
if (gnorm <= gtol):
outputParams = getOutputParams(inputParams, list(locals().items()))
return outputParams
try: # this was handled in numeric, let it remaines for more safety
rhok = 1.0 / (numpy.dot(yk, sk))
except ZeroDivisionError:
rhok = 1000.0
print("Divide-by-zero encountered: rhok assumed large")
if numpy.isinf(rhok): # this is patch for numpy
rhok = 1000.0
print("Divide-by-zero encountered: rhok assumed large")
A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok
A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok
Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
* sk[numpy.newaxis,:]
outputParams = getOutputParams(inputParams, list(locals().items()))
return outputParams
def fminLooped(f,
x0,
fprime=None,
args=(),
gtol=1e-5,
norm=Inf,
epsilon=numpy.sqrt(numpy.finfo(float).eps),
maxiter=None,
full_output=0,
disp=1,
retall=0,
callback=None):
testVar = 0
x0 = asarray(x0).squeeze()
if x0.ndim == 0:
x0.shape = (1, )
if maxiter is None:
maxiter = len(x0) * 200
func_calls, f = wrap_function(f, args)
if fprime is None:
grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
else:
grad_calls, myfprime = wrap_function(fprime, args)
gfk = myfprime(x0)
k = 0
N = len(x0)
I = numpy.eye(N, dtype=int)
Hk = I
old_fval = f(x0)
old_old_fval = old_fval + 5000
xk = x0
if retall:
allvecs = [x0]
sk = [2 * gtol]
warnflag = 0
gnorm = vecnorm(gfk, ord=norm)
newInputParams = locals()
pickleBles, NonPickles = ParamsManager.filterUnpickles(newInputParams)
# import dill
# pickle.dump(pickleBles, open('inputParams.dat', 'w'))
# pickle.dump(loopThing, open('loopFunc.dat', 'w'))
for loopI in range(300):
newInputParams = loopThing(newInputParams)
def Extension(ds, ds_ini):
list = []
for dpi in ds_ini:
num = 10
for dp in ds:
d = vecnorm(dp.f - dpi)
if d < num:
num = d
list.append(num)
dd = 0
for l in list:
dd += l**2
EX = np.sqrt(dd) / len(ds_ini)
def Eveness(ds):
ds = sorted(ds, key=lambda x: x.f[0], reverse=False)
d = []
for i, dpi in enumerate(ds):
if i != 0:
d.append(vecnorm(dpi.f - ds[i - 1].f))
sum = 0
for di in d:
sum += di
mu = sum/len(d)
sum2 = 0
for i, di in enumerate(d):
if i != 0:
sum2 += (di - mu) ** 2
si = np.sqrt(sum2 / (len(d) - 1))
# return si / mu
ol = mu - si
i = 0
while i < len(d)-1:
if d[i] < ol:
d[i+1] += d[i]
del d[i]
else:
i += 1
mu = sum / len(d)
sum2 = 0
for i, di in enumerate(d):
if i != 0:
sum2 += (di - mu) ** 2
sum2 += (d[i-1] - mu) ** 2
si = np.sqrt(sum2 / (2*len(d) - 1))
return si / mu
def fminLooped(f, x0, fprime=None, args=(), gtol=1e-5, norm=Inf,
epsilon= numpy.sqrt(numpy.finfo(float).eps), maxiter=None, full_output=0, disp=1,
retall=0, callback=None):
testVar = 0
x0 = asarray(x0).squeeze()
if x0.ndim == 0:
x0.shape = (1,)
if maxiter is None:
maxiter = len(x0)*200
func_calls, f = wrap_function(f, args)
if fprime is None:
grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
else:
grad_calls, myfprime = wrap_function(fprime, args)
gfk = myfprime(x0)
k = 0
N = len(x0)
I = numpy.eye(N,dtype=int)
Hk = I
old_fval = f(x0)
old_old_fval = old_fval + 5000
xk = x0
if retall:
allvecs = [x0]
sk = [2*gtol]
warnflag = 0
gnorm = vecnorm(gfk,ord=norm)
newInputParams = locals()
pickleBles, NonPickles = ParamsManager.filterUnpickles(newInputParams)
import dill
pickle.dump(pickleBles, open('inputParams.dat', 'w'))
pickle.dump(loopThing, open('loopFunc.dat', 'w'))
for loopI in range(10):
newInputParams = loopThing(newInputParams)
def minimize_bfgs(func: Callable,
init_param_vec: Sequence,
grad: Callable = None,
grad_tol: float = 1e-5,
return_all: bool = True,
last_record: 'OptimizeRecord' = None,
notes: dict = None):
if notes is None:
notes = {}
notes["grad_tol"] = grad_tol
notes["method"] = "BFGS"
f = func
fprime = grad
epsilon = np.finfo(float).eps**0.5
gtol = grad_tol
norm = np.Inf
x0 = init_param_vec
if x0.ndim == 0:
x0.shape = (1, )
maxiter = len(x0) * 200
func_calls, f = wrap_function(f, ())
if fprime is None:
grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
notes["grad_approx"] = True
else:
grad_calls, myfprime = wrap_function(fprime, ())
notes["grad_approx"] = False
k = 0
N = len(x0)
I = np.eye(N, dtype=int)
if last_record:
old_fval = last_record.final_func
gfk = last_record.final_grad
old_old_fval = last_record.last_vars["old_old_fval"]
Hk = last_record.last_vars["Hk"]
else:
# Sets the initial step guess to dx ~ 1
old_fval = f(x0)
gfk = myfprime(x0)
old_old_fval = old_fval + np.linalg.norm(gfk) / 2
Hk = I
all_param_vec = [x0]
all_func = [old_fval]
all_grad = [gfk]
xk = x0
warnflag = 0
gnorm = vecnorm(gfk, ord=norm)
while (gnorm > gtol) and (k < maxiter):
pk = -np.dot(Hk, gfk)
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
_line_search_wolfe12(f, myfprime, xk, pk, gfk,
old_fval, old_old_fval, amin=1e-100, amax=1e100)
except _LineSearchError:
# Line search failed to find a better solution.
warnflag = 2
break
xkp1 = xk + alpha_k * pk
sk = xkp1 - xk
xk = xkp1
if gfkp1 is None:
gfkp1 = myfprime(xkp1)
yk = gfkp1 - gfk
gfk = gfkp1
k += 1
if return_all:
all_param_vec.append(xk)
all_func.append(old_fval)
all_grad.append(gfk)
gnorm = vecnorm(gfk, ord=norm)
if (gnorm <= gtol):
break
if not np.isfinite(old_fval):
# We correctly found +-Inf as optimal value, or something went
# wrong.
warnflag = 2
break
try: # this was handled in numeric, let it remaines for more safety
rhok = 1.0 / (np.dot(yk, sk))
except ZeroDivisionError:
rhok = 1000.0
print("Divide-by-zero encountered: rhok assumed large")
if np.isinf(rhok): # this is patch for numpy
rhok = 1000.0
print("Divide-by-zero encountered: rhok assumed large")
A1 = I - sk[:, np.newaxis] * yk[np.newaxis, :] * rhok
A2 = I - yk[:, np.newaxis] * sk[np.newaxis, :] * rhok
Hk = np.dot(A1, np.dot(
Hk, A2)) + (rhok * sk[:, np.newaxis] * sk[np.newaxis, :])
fval = old_fval
if np.isnan(fval):
# This can happen if the first call to f returned NaN;
# the loop is then never entered.
warnflag = 2
if warnflag == 2:
msg = _status_message['pr_loss']
elif k >= maxiter:
warnflag = 1
msg = _status_message['maxiter']
else:
msg = _status_message['success']
history = {"func": all_func, "grad": all_grad, "param_vec": all_param_vec}
final_status = {
"msg": msg,
"warnflag": warnflag,
"num_func_call": func_calls[0],
"num_grad_call": grad_calls[0],
"num_iter": k
}
last_vars = {"Hk": Hk, "old_old_fval": old_old_fval}
record = OptimizeRecord(history, final_status, last_vars, notes)
return record
def Customfmin_bfgs(f, x0, fprime=None, args=(), gtol=1e-5, norm=Inf,
epsilon= numpy.sqrt(numpy.finfo(float).eps), maxiter=None, full_output=0, disp=1,
retall=0, callback=None):
testVar = 0
x0 = asarray(x0).squeeze()
if x0.ndim == 0:
x0.shape = (1,)
if maxiter is None:
maxiter = len(x0)*200
func_calls, f = wrap_function(f, args)
if fprime is None:
grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
else:
grad_calls, myfprime = wrap_function(fprime, args)
gfk = myfprime(x0)
k = 0
N = len(x0)
I = numpy.eye(N,dtype=int)
Hk = I
old_fval = f(x0)
old_old_fval = old_fval + 5000
xk = x0
if retall:
allvecs = [x0]
sk = [2*gtol]
warnflag = 0
gnorm = vecnorm(gfk,ord=norm)
while (gnorm > gtol) and (k < maxiter):
pk = -numpy.dot(Hk,gfk)
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
linesearch.line_search(f,myfprime,xk,pk,gfk,
old_fval,old_old_fval)
if alpha_k is None: # line search failed try different one.
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
line_search(f,myfprime,xk,pk,gfk,
old_fval,old_old_fval)
if alpha_k is None:
# This line search also failed to find a better solution.
warnflag = 2
break
xkp1 = xk + alpha_k * pk
if retall:
allvecs.append(xkp1)
sk = xkp1 - xk
xk = xkp1
if gfkp1 is None:
gfkp1 = myfprime(xkp1)
yk = gfkp1 - gfk
gfk = gfkp1
if callback is not None:
callback(xk)
k += 1
gnorm = vecnorm(gfk,ord=norm)
if (gnorm <= gtol):
break
try: # this was handled in numeric, let it remaines for more safety
rhok = 1.0 / (numpy.dot(yk,sk))
except ZeroDivisionError:
rhok = 1000.0
print "Divide-by-zero encountered: rhok assumed large"
if numpy.isinf(rhok): # this is patch for numpy
rhok = 1000.0
print "Divide-by-zero encountered: rhok assumed large"
A1 = I - sk[:,numpy.newaxis] * yk[numpy.newaxis,:] * rhok
A2 = I - yk[:,numpy.newaxis] * sk[numpy.newaxis,:] * rhok
Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
* sk[numpy.newaxis,:]
if disp or full_output:
fval = old_fval
if warnflag == 2:
if disp:
print "Warning: Desired error not necessarily achieved" \
"due to precision loss"
print " Current function value: %f" % fval
print " Iterations: %d" % k
print " Function evaluations: %d" % func_calls[0]
print " Gradient evaluations: %d" % grad_calls[0]
elif k >= maxiter:
warnflag = 1
if disp:
print "Warning: Maximum number of iterations has been exceeded"
print " Current function value: %f" % fval
print " Iterations: %d" % k
print " Function evaluations: %d" % func_calls[0]
print " Gradient evaluations: %d" % grad_calls[0]
else:
if disp:
print "Optimization terminated successfully."
print " Current function value: %f" % fval
print " Iterations: %d" % k
print " Function evaluations: %d" % func_calls[0]
print " Gradient evaluations: %d" % grad_calls[0]
if full_output:
retlist = xk, fval, gfk, Hk, func_calls[0], grad_calls[0], warnflag
if retall:
retlist += (allvecs,)
else:
retlist = xk
if retall:
retlist = (xk, allvecs)
return retlist
def _minimize(fun, x0, args=(), jac=None, callback=None,
gtol=1e-5, fxtol=1e-09, xtol=1e-09, norm=Inf,
eps=_epsilon, maxiter=None, disp=False,
return_all=False, **unknown_options):
_check_unknown_options(unknown_options)
f = fun
fprime = jac
epsilon = eps
retall = return_all
x0 = asarray(x0).flatten()
if x0.ndim == 0:
x0.shape = (1,)
if maxiter is None:
maxiter = len(x0) * 200
func_calls, f = wrap_function(f, args)
grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
gfk = myfprime(x0)
k = 0
N = len(x0)
I = numpy.eye(N, dtype=int)
Hk = I
old_fval = f(x0)
old_old_fval = None
xk = x0
if retall:
allvecs = [x0]
sk = [2 * gtol]
warnflag = 0
gnorm = vecnorm(gfk, ord=norm)
xnorm = np.Inf
fx = np.Inf
print_lst = []
while (gnorm > gtol) and (xnorm > xtol) and (fx > fxtol) and (k < maxiter):
pk = -numpy.dot(Hk, gfk)
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
_line_search_wolfe12(f, myfprime, xk, pk, gfk,
old_fval, old_old_fval)
except _LineSearchError:
# search failed to find a better solution.
print_lst.append("Przeszukiwanie liniowe zawiodlo lub nie moze osiagnac lepszego rozwiazania")
warnflag = 2
break
xkp1 = xk + alpha_k * pk
fx = np.absolute(old_old_fval - old_fval)
xnorm = vecnorm(xkp1 - xk)
if retall:
allvecs.append(xkp1)
sk = xkp1 - xk
xk = xkp1
if gfkp1 is None:
gfkp1 = myfprime(xkp1)
yk = gfkp1 - gfk
gfk = gfkp1
if callback is not None:
callback(xk)
k += 1
if disp:
print_ = ('Iter: ' + str(k) + '\n')
print_ += ('x: ' + str(xk) + '\n')
print_ += ('f(x): ' + str(f(xk)) + '\n') #zmiana na fx
print_ +=('gtol: ' + str(gnorm) + '\n')
print_ +=('xtol: ' + str(xnorm) + '\n')
print_ +=('fxtol: ' + str(fx) + '\n')
print_lst.append(print_)
gnorm = vecnorm(gfk, ord=norm)
if (gnorm <= gtol):
break
if not numpy.isfinite(old_fval):
# We correctly found +-Inf as optimal value, or something went
# wrong.
print_lst.append("Zlaneziono +-Inf za optymalna wartosc... lub cos poszlo zle.")
warnflag = 2
break
try: # this was handled in numeric, let it remaines for more safety
rhok = 1.0 / (numpy.dot(yk, sk))
except ZeroDivisionError:
rhok = 1000.0
if disp:
print_lst.append("Dzielenie przez zero!!")
if isinf(rhok): # this is patch for numpy
rhok = 1000.0
if disp:
print_lst.appedn("Dzielenie przez zero!!")
A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok
A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok
Hk = numpy.dot(A1, numpy.dot(Hk, A2)) + (rhok * sk[:, numpy.newaxis] *
sk[numpy.newaxis, :])
fval = old_fval
if np.isnan(fval):
# This can happen if the first call to f returned NaN;
# the loop is then never entered.
print_lst.append("Osiagnieto Nan w pierwszym wywolaniem algorytmu.")
warnflag = 2
if warnflag == 2:
msg = _status_message['pr_loss']
if disp:
print_ = ("Ostrzezenie: " + msg)
print_ += (" Wartosc funkcji celu: %f" % fval)
print_ += (" Iteracje: %d" % k)
print_ += (" Wywolania funkcji: %d" % func_calls[0])
print_ += (" Wywolania gradientu: %d" % grad_calls[0])
elif k >= maxiter:
warnflag = 1
msg = _status_message['maxiter']
if disp:
print_ = ("Ostrzerzenie: " + msg)
print_ += (" Wartosc funkcji celu: %f" % fval)
print_ += (" Iteracje: %d" % k)
print_ += (" Wywolania funkcji: %d" % func_calls[0])
print_ += (" Wywolania gradientu: %d" % grad_calls[0])
print_lst.append(print_)
else:
msg = _status_message['success']
if disp:
print_ = (msg + '\n')
print_ += (" Wartosc funkcji celu: %f" % fval)
print_ += (" Iteracje: %d" % k)
print_ += (" Wywolania funkcji: %d" % func_calls[0])
print_ += (" Wywolania gradientu: %d" % grad_calls[0])
print_lst.append(print_)
[print(line) for line in print_lst]
result = OptimizeResult(fun=fval,lst=print_lst, jac=gfk, hess_inv=Hk, nfev=func_calls[0],
njev=grad_calls[0], status=warnflag,
success=(warnflag == 0), message=msg, x=xk,
nit=k)
if retall:
result['allvecs'] = allvecs
return result
def _minimize_cg(fun, x0, args=(), jac=None, callback=None,
gtol=1e-5, norm=Inf, eps=_epsilon, maxiter=None,
disp=False, return_all=False,
xtol= 1e-6,
**unknown_options):
"""
Minimization of scalar function of one or more variables using the
conjugate gradient algorithm.
Options for the conjugate gradient algorithm are:
disp : bool
Set to True to print convergence messages.
maxiter : int
Maximum number of iterations to perform.
gtol : float
Gradient norm must be less than `gtol` before successful
termination.
norm : float
Order of norm (Inf is max, -Inf is min).
eps : float or ndarray
If `jac` is approximated, use this value for the step size.
This function is called by the `minimize` function with `method=CG`. It
is not supposed to be called directly.
"""
_check_unknown_options(unknown_options)
f = fun
fprime = jac
epsilon = eps
retall = return_all
x0 = asarray(x0).flatten()
if maxiter is None:
maxiter = len(x0) * 200
func_calls, f = wrap_function(f, args)
if fprime is None:
grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
else:
grad_calls, myfprime = wrap_function(fprime, args)
gfk = myfprime(x0)
k = 0
xk = x0
old_fval = f(xk)
old_old_fval = None
if retall:
allvecs = [xk]
warnflag = 0
pk = -gfk
gnorm = vecnorm(gfk, ord=norm)
while (gnorm > gtol) and (k < maxiter):
deltak = numpy.dot(gfk, gfk)
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
_line_search_wolfe12(f, myfprime, xk, pk, gfk, old_fval,
old_old_fval, c2=0.4, xtol=xtol)
except _LineSearchError:
# Line search failed to find a better solution.
warnflag = 2
break
xk = xk + alpha_k * pk
if retall:
allvecs.append(xk)
if gfkp1 is None:
gfkp1 = myfprime(xk)
yk = gfkp1 - gfk
beta_k = max(0, numpy.dot(yk, gfkp1) / deltak)
pk = -gfkp1 + beta_k * pk
gfk = gfkp1
gnorm = vecnorm(gfk, ord=norm)
if callback is not None:
callback(xk)
k += 1
fval = old_fval
if warnflag == 2:
msg = _status_message['pr_loss']
if disp:
print("Warning: " + msg)
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
print(" Function evaluations: %d" % func_calls[0])
print(" Gradient evaluations: %d" % grad_calls[0])
elif k >= maxiter:
warnflag = 1
msg = _status_message['maxiter']
if disp:
print("Warning: " + msg)
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
print(" Function evaluations: %d" % func_calls[0])
print(" Gradient evaluations: %d" % grad_calls[0])
else:
msg = _status_message['success']
if disp:
print(msg)
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
print(" Function evaluations: %d" % func_calls[0])
print(" Gradient evaluations: %d" % grad_calls[0])
result = OptimizeResult(fun=fval, jac=gfk, nfev=func_calls[0],
njev=grad_calls[0], status=warnflag,
success=(warnflag == 0), message=msg, x=xk)
if retall:
result['allvecs'] = allvecs
return result
def _minimize_lbfgsb(
fun,
x0,
bounds=None,
args=(),
kwargs={},
jac=None,
callback=None,
tol={"abs": 1e-05, "rel": 1e-08},
norm=np.Inf,
maxiter=None,
disp=False,
return_all=False,
**unknown_options
):
"""
Minimization of scalar function of one or more variables using the
BHHH algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxiter : int
Maximum number of iterations to perform.
tol : dict
Absolute and relative tolerance values.
norm : float
Order of norm (Inf is max, -Inf is min).
"""
_check_unknown_options(unknown_options)
def f(x0):
return fun(x0, *args, **kwargs)
fprime = jac
# epsilon = eps Add functionality
retall = return_all
k = 0
ns = 0
nsmax = 5
N = len(x0)
x0 = np.asarray(x0).flatten()
if x0.ndim == 0:
x0.shape = (1,)
if bounds is None:
bounds = np.array([np.inf] * N * 2).reshape((2, N))
bounds[0, :] = -bounds[0, :]
if bounds.shape[1] != N:
raise ValueError("length of x0 != length of bounds")
low = bounds[0, :]
up = bounds[1, :]
x0 = np.clip(x0, low, up)
if maxiter is None:
maxiter = len(x0) * 200
if not callable(fprime):
def myfprime(x0):
return approx_derivative(f, x0, args=args, kwargs=kwargs)
else:
myfprime = fprime
# Setup for iteration
old_fval = f(x0)
gf0 = myfprime(x0)
gfk = gf0
norm_pg0 = vecnorm(x0 - np.clip(x0 - gf0, low, up), ord=norm)
xk = x0
norm_pgk = norm_pg0
sstore = np.zeros((maxiter, N))
ystore = sstore.copy()
if retall:
allvecs = [x0]
warnflag = 0
# Calculate indices ofactive and inative set using projected gradient
epsilon = min(np.min(up - low) / 2, norm_pgk)
activeset = np.logical_or(xk - low <= epsilon, up - xk <= epsilon)
inactiveset = np.logical_not(activeset)
for _ in range(maxiter): # for loop instead.
# Check tolerance of gradient norm
if norm_pgk <= tol["abs"] + tol["rel"] * norm_pg0:
break
pk = -gfk
pk = bfgsrecb(ns, sstore, ystore, pk, activeset)
gfk_active = gfk.copy()
gfk_active[inactiveset] = 0
pk = -gfk_active + pk
# Sets the initial step guess to dx ~ 1
old_old_fval = old_fval + np.linalg.norm(gfk) / 2
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = _line_search_wolfe12(
f,
myfprime,
xk,
pk,
gfk,
old_fval,
old_old_fval,
amin=1e-100,
amax=1e100,
)
except _LineSearchError:
# Line search failed to find a better solution.
warnflag = 2
break
xkp1 = np.clip(xk + alpha_k * pk, low, up)
if retall:
allvecs.append(xkp1)
yk = myfprime(xkp1) - gfk
sk = xkp1 - xk
xk = xkp1
gfk = myfprime(xkp1)
norm_pgk = vecnorm(xk - np.clip(xk - gfk, low, up), ord=norm)
# Calculate indices ofactive and inative set using projected gradient
epsilon = min(np.min(up - low) / 2, norm_pgk)
activeset = np.logical_or(xk - low <= epsilon, up - xk <= epsilon)
inactiveset = np.logical_not(activeset)
yk[activeset] = 0
sk[activeset] = 0
# reset storage
ytsk = yk.dot(sk)
if ytsk <= 0:
ns = 0
if ns == nsmax:
print("ns reached maximum size")
ns = 0
elif ytsk > 0:
ns += 1
alpha0 = ytsk ** 0.5
sstore[ns - 1, :] = sk / alpha0
ystore[ns - 1, :] = yk / alpha0
k += 1
if callback is not None:
callback(xk)
if np.isinf(old_fval):
# We correctly found +-Inf as optimal value, or something went
# wrong.
warnflag = 2
break
if np.isnan(xk).any():
warnflag = 3
break
fval = old_fval
if warnflag == 2:
msg = _status_message["pr_loss"]
elif k >= maxiter:
warnflag = 1
msg = _status_message["maxiter"]
elif np.isnan(fval) or np.isnan(xk).any():
warnflag = 3
msg = _status_message["nan"]
else:
msg = _status_message["success"]
if disp:
print("{}{}".format("Warning: " if warnflag != 0 else "", msg))
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
result = OptimizeResult(
fun=fval,
jac=gfk,
status=warnflag,
success=(warnflag == 0),
message=msg,
x=xk,
nit=k,
)
if retall:
result["allvecs"] = allvecs
return result
def Customfmin_bfgs(f,
x0,
fprime=None,
args=(),
gtol=1e-5,
norm=Inf,
epsilon=numpy.sqrt(numpy.finfo(float).eps),
maxiter=None,
full_output=0,
disp=1,
retall=0,
callback=None):
testVar = 0
x0 = asarray(x0).squeeze()
if x0.ndim == 0:
x0.shape = (1, )
if maxiter is None:
maxiter = len(x0) * 200
func_calls, f = wrap_function(f, args)
if fprime is None:
grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
else:
grad_calls, myfprime = wrap_function(fprime, args)
gfk = myfprime(x0)
k = 0
N = len(x0)
I = numpy.eye(N, dtype=int)
Hk = I
old_fval = f(x0)
old_old_fval = old_fval + 5000
xk = x0
if retall:
allvecs = [x0]
sk = [2 * gtol]
warnflag = 0
gnorm = vecnorm(gfk, ord=norm)
while (gnorm > gtol) and (k < maxiter):
pk = -numpy.dot(Hk, gfk)
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
linesearch.line_search(f,myfprime,xk,pk,gfk,
old_fval,old_old_fval)
if alpha_k is None: # line search failed try different one.
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
line_search(f,myfprime,xk,pk,gfk,
old_fval,old_old_fval)
if alpha_k is None:
# This line search also failed to find a better solution.
warnflag = 2
break
xkp1 = xk + alpha_k * pk
if retall:
allvecs.append(xkp1)
sk = xkp1 - xk
xk = xkp1
if gfkp1 is None:
gfkp1 = myfprime(xkp1)
yk = gfkp1 - gfk
gfk = gfkp1
if callback is not None:
callback(xk)
k += 1
gnorm = vecnorm(gfk, ord=norm)
if (gnorm <= gtol):
break
try: # this was handled in numeric, let it remaines for more safety
rhok = 1.0 / (numpy.dot(yk, sk))
except ZeroDivisionError:
rhok = 1000.0
print("Divide-by-zero encountered: rhok assumed large")
if numpy.isinf(rhok): # this is patch for numpy
rhok = 1000.0
print("Divide-by-zero encountered: rhok assumed large")
A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok
A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok
Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
* sk[numpy.newaxis,:]
if disp or full_output:
fval = old_fval
if warnflag == 2:
if disp:
print("Warning: Desired error not necessarily achieved" \
"due to precision loss")
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
print(" Function evaluations: %d" % func_calls[0])
print(" Gradient evaluations: %d" % grad_calls[0])
elif k >= maxiter:
warnflag = 1
if disp:
print("Warning: Maximum number of iterations has been exceeded")
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
print(" Function evaluations: %d" % func_calls[0])
print(" Gradient evaluations: %d" % grad_calls[0])
else:
if disp:
print("Optimization terminated successfully.")
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
print(" Function evaluations: %d" % func_calls[0])
print(" Gradient evaluations: %d" % grad_calls[0])
if full_output:
retlist = xk, fval, gfk, Hk, func_calls[0], grad_calls[0], warnflag
if retall:
retlist += (allvecs, )
else:
retlist = xk
if retall:
retlist = (xk, allvecs)
return retlist
def leon_ncg_python(make_f, w_0, make_fprime=None, gtol=1e-5, norm=numpy.Inf,
maxiter=None, full_output=0, disp=1, retall=0, callback=None,
direction='hestenes-stiefel',
minibatch_size=None,
minibatch_offset=None,
restart_every=0,
normalize=False,
constrain_lambda=True,
):
"""Minimize a function using a nonlinear conjugate gradient algorithm.
Parameters
----------
make_f : callable make_f(k0, k1)
When called with (k0, k1) as arguments, return a function f such that
f(w) is the objective to be minimize at parameter w, on minibatch x_k0
to x_k1. If k1 is None then the minibatch should contain all the
remaining data.
w_0 : ndarray
Initial guess.
make_fprime : callable make_f'(k0, k1)
Same as `make_f`, but to compute the derivative of f on a minibatch.
gtol : float
Stop when norm of gradient is less than gtol.
norm : float
Order of vector norm to use. -Inf is min, Inf is max.
size (can be scalar or vector).
callback : callable
An optional user-supplied function, called after each
iteration. Called as callback(w_t, lambda_t), where w_t is the
current parameter vector and lambda_t the coefficient for the
new direction.
direction : string
Formula used to computed the new direction, among:
- polak-ribiere
- hestenes-stiefel
minibatch_size : int
Size of each minibatch. Use None for batch learning.
minibatch_offset : int
Shift of the minibatch. Use None to use the minibatch size (i.e. no
overlap at all).
restart_every : int
Force restart every this number of iterations. If <= 0, then never
force a restart.
normalize : bool
If True, then use the normalized gradient instead of the gradient
itself to find the next search direction, and always normalize the
search direction.
constrain_lambda : bool
If True, then the `lambda_t` factor used to compute conjugate directions
is constrained to be non-negative (it is thus set to zero if the formula
given by `direction` computes a negative value).
Returns
-------
xopt : ndarray
Parameters which minimize f, i.e. f(xopt) == fopt.
fopt : float
Minimum value found, f(xopt).
func_calls : int
The number of function_calls made.
grad_calls : int
The number of gradient calls made.
warnflag : int
1 : Maximum number of iterations exceeded.
2 : Gradient and/or function calls not changing.
allvecs : ndarray
If retall is True (see other parameters below), then this
vector containing the result at each iteration is returned.
Other Parameters
----------------
maxiter : int
Maximum number of iterations to perform.
full_output : bool
If True then return fopt, func_calls, grad_calls, and
warnflag in addition to xopt.
disp : bool
Print convergence message if True.
retall : bool
Return a list of results at each iteration if True.
Notes
-----
Optimize the function, f, whose gradient is given by fprime
using the nonlinear conjugate gradient algorithm of Polak and
Ribiere. See Wright & Nocedal, 'Numerical Optimization',
1999, pg. 120-122.
"""
if minibatch_offset is None:
if minibatch_size is None:
# Batch learning: no offset is needed.
minibatch_offset = 0
else:
# Use the same offset as the minibatch size.
minibatch_offset = minibatch_size
w_0 = numpy.asarray(w_0).flatten()
if maxiter is None:
maxiter = len(w_0)*200
k0 = 0
k1 = minibatch_size
assert make_fprime is not None
f = make_f(k0, k1)
fprime = make_fprime(k0, k1)
func_calls = [0]
grad_calls = [0]
tmp_func_calls, f = wrap_function(f, ())
tmp_grad_calls, myfprime = wrap_function(fprime, ())
g_t = myfprime(w_0)
t = 0
N = len(w_0)
w_t = w_0
if retall:
allvecs = [w_t]
warnflag = 0
if normalize:
d_t = -g_t / numpy.linalg.norm(g_t)
else:
d_t = -g_t
gnorm = vecnorm(g_t, ord=norm)
w_t_previous = None
while (gnorm > gtol) and (t < maxiter):
#print '||g_t|| = %s' % numpy.linalg.norm(g_t)
# Since the function changes at each iteration, we cannot re-use
# previous function values.
old_fval = f(w_t)
if w_t_previous is None:
old_old_fval = old_fval + 5000
else:
old_old_fval = f(w_t_previous)
# These values are modified by the line search, even if it fails.
old_fval_backup = old_fval
old_old_fval_backup = old_old_fval
alpha_t, fc, gc, old_fval, old_old_fval, h_t = \
line_search_wolfe1(f, myfprime, w_t, d_t, g_t, old_fval,
old_old_fval, c2=0.4)
if alpha_t is None: # line search failed -- use different one.
print '*********************************** LINE SEARCH FAILURE *********************************'
alpha_t, fc, gc, old_fval, old_old_fval, h_t = \
line_search_wolfe2(f, myfprime, w_t, d_t, g_t,
old_fval_backup, old_old_fval_backup)
print '*********************************** %s *********************************' % alpha_t
if alpha_t is None or alpha_t == 0:
# This line search also failed to find a better solution.
raise AssertionError()
warnflag = 2
break
print 'alpha_t = %s' % alpha_t
# Update weights.
w_tp1 = w_t + alpha_t * d_t
# Compute derivative after the weight update, if not done already.
if h_t is None:
h_t = myfprime(w_tp1)
else:
assert (h_t == myfprime(w_tp1)).all() # Sanity check.
# Switch to next minibatch.
func_calls[0] += tmp_func_calls[0]
grad_calls[0] += tmp_grad_calls[0]
k0 += minibatch_offset
if minibatch_size is None:
k1 = None
else:
k1 = k0 + minibatch_size
tmp_func_calls, f = wrap_function(make_f(k0, k1), ())
tmp_grad_calls, myfprime = wrap_function(make_fprime(k0, k1), ())
# Compute derivative on new minibatch.
g_tp1 = myfprime(w_tp1)
if normalize:
g_tp1_for_dt = g_tp1 / numpy.linalg.norm(g_tp1)
else:
g_tp1_for_dt = g_tp1
if retall:
allvecs.append(w_tp1)
h_t_minus_g_t = h_t - g_t
if direction == 'polak-ribiere':
# Polak-Ribiere.
delta_t = numpy.dot(g_t, g_t)
lambda_t = numpy.dot(h_t_minus_g_t, g_tp1_for_dt) / delta_t
elif direction == 'hestenes-stiefel':
# Hestenes-Stiefel.
lambda_t = numpy.dot(h_t_minus_g_t, g_tp1_for_dt) / numpy.dot(h_t_minus_g_t, d_t)
else:
raise NotImplementedError(direction)
if constrain_lambda and lambda_t < 0:
lambda_t = 0
if restart_every > 0 and (t + 1) % restart_every == 0:
lambda_t = 0
if lambda_t == 0:
print '*** RESTART ***'
else:
print 'lambda_t = %s' % lambda_t
d_t = -g_tp1_for_dt + lambda_t * d_t
if normalize:
d_t /= numpy.linalg.norm(d_t)
g_t = g_tp1
w_t_previous = w_t
w_t = w_tp1
gnorm = vecnorm(g_t, ord=norm)
if callback is not None:
callback(w_t, lambda_t)
t += 1
if disp or full_output:
fval = old_fval
if warnflag == 2:
if disp:
print "Warning: Desired error not necessarily achieved due to precision loss"
print " Current function value: %f" % fval
print " Iterations: %d" % t
print " Function evaluations: %d" % func_calls[0]
print " Gradient evaluations: %d" % grad_calls[0]
elif t >= maxiter:
warnflag = 1
if disp:
print "Warning: Maximum number of iterations has been exceeded"
print " Current function value: %f" % fval
print " Iterations: %d" % t
print " Function evaluations: %d" % func_calls[0]
print " Gradient evaluations: %d" % grad_calls[0]
else:
if disp:
print "Optimization terminated successfully."
print " Current function value: %f" % fval
print " Iterations: %d" % t
print " Function evaluations: %d" % func_calls[0]
print " Gradient evaluations: %d" % grad_calls[0]
if full_output:
retlist = w_t, fval, func_calls[0], grad_calls[0], warnflag
if retall:
retlist += (allvecs,)
else:
retlist = w_t
if retall:
retlist = (w_t, allvecs)
return retlist
def _minimize_bhhh(fun,
x0,
bounds=None,
args=(),
jac=None,
callback=None,
tol={
"abs": 1e-05,
"rel": 1e-08
},
norm=np.Inf,
maxiter=None,
disp=False,
return_all=False,
**unknown_options):
"""
Minimization of scalar function of one or more variables using the
BHHH algorithm.
Options
-------
disp : bool
Set to True to print convergence messages.
maxiter : int
Maximum number of iterations to perform.
tol : dict
Absolute and relative tolerance values.
norm : float
Order of norm (Inf is max, -Inf is min).
"""
_check_unknown_options(unknown_options)
f = fun
fprime = jac
retall = return_all
k = 0
N = len(x0)
x0 = np.asarray(x0).flatten()
if x0.ndim == 0:
x0.shape = (1, )
if bounds is None:
bounds = np.array([np.inf] * N * 2).reshape((2, N))
bounds[0, :] = -bounds[0, :]
if bounds.shape[1] != N:
raise ValueError("length of x0 != length of bounds")
low = bounds[0, :]
up = bounds[1, :]
x0 = np.clip(x0, low, up)
if maxiter is None:
maxiter = len(x0) * 200
# Need the aggregate functions to take only x0 as an argument
func_calls, agg_fun = wrap_function_agg(f, args)
if not callable(fprime):
grad_calls, myfprime = wrap_function_num_dev(f, args)
else:
grad_calls, myfprime = wrap_function(fprime, args)
def agg_fprime(x0):
return myfprime(x0).sum(axis=0)
# Setup for iteration
old_fval = agg_fun(x0)
gf0 = agg_fprime(x0)
norm_pg0 = vecnorm(x0 - np.clip(x0 - gf0, low, up), ord=norm)
xk = x0
norm_pgk = norm_pg0
if retall:
allvecs = [x0]
warnflag = 0
for _ in range(maxiter): # for loop instead.
# Individual
gfk_obs = myfprime(xk)
# Aggregate fprime. Might replace by simply summing up gfk_obs
gfk = gfk_obs.sum(axis=0)
norm_pgk = vecnorm(xk - np.clip(xk - gfk, low, up), ord=norm)
# Check tolerance of gradient norm
if norm_pgk <= tol["abs"] + tol["rel"] * norm_pg0:
break
# Sets the initial step guess to dx ~ 1
old_old_fval = old_fval + np.linalg.norm(gfk) / 2
# Calculate BHHH hessian and step
Hk = np.dot(gfk_obs.T, gfk_obs)
Bk = np.linalg.inv(Hk)
pk = np.empty(N)
pk = -np.dot(Bk, gfk)
try:
alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = _line_search_wolfe12(
agg_fun,
agg_fprime,
xk,
pk,
gfk,
old_fval,
old_old_fval,
amin=1e-100,
amax=1e100,
)
except _LineSearchError:
# Line search failed to find a better solution.
warnflag = 2
break
xkp1 = np.clip(xk + alpha_k * pk, low, up)
if retall:
allvecs.append(xkp1)
xk = xkp1
if callback is not None:
callback(xk)
k += 1
if np.isinf(old_fval):
# We correctly found +-Inf as optimal value, or something went
# wrong.
warnflag = 2
break
fval = old_fval
if warnflag == 2:
msg = _status_message["pr_loss"]
elif k >= maxiter:
warnflag = 1
msg = _status_message["maxiter"]
elif np.isnan(fval) or np.isnan(xk).any():
warnflag = 3
msg = _status_message["nan"]
else:
msg = _status_message["success"]
if disp:
print("{}{}".format("Warning: " if warnflag != 0 else "", msg))
print(" Current function value: %f" % fval)
print(" Iterations: %d" % k)
result = OptimizeResult(
fun=fval,
jac=gfk,
hess_inv=Bk,
nfev=func_calls[0],
njev=grad_calls[0],
status=warnflag,
success=(warnflag == 0),
message=msg,
x=xk,
nit=k,
)
if retall:
result["allvecs"] = allvecs
return result
