def predint_multi(x: np.ndarray, xd: np.ndarray, yd: np.ndarray,
func: Callable[[np.ndarray, np.ndarray],
np.ndarray], res: OptimizeResult, **kwargs):
"""
This function estimates the prediction bands for the fit
(see: https://www.mathworks.com/help/curvefit/confidence-and-prediction-bounds.html)
Parameters
----------
x: np.ndarray
The requested x points for the bands
xd: np.ndarray
The x datapoints
yd: np.ndarray
The y datapoints
func: Callable[[np.ndarray, np.ndarray]
The fitted function
res: OptimizeResult
The optimzied result from least_squares minimization
**kwargs
confidence: float
The confidence level (default 0.95)
simulateneous: bool
True if the bound type is simultaneous, false otherwise
mode: [functional, observation]
Default observation
"""
if len(yd) != len(xd):
raise ValueError('The length of the observations should be the same ' +
'as the length of the predictions.')
if len(yd) <= 1:
raise ValueError('Too few datapoints')
from scipy.optimize import optimize
if not isinstance(res, optimize.OptimizeResult):
raise ValueError(
'Argument \'res\' should be an instance of \'scipy.optimize.OptimizeResult\''
)
simultaneous = kwargs.get('simultaneous', True)
mode = kwargs.get('mode', 'observation')
confidence = kwargs.get('confidence', 0.95)
p = len(res.x)
# Needs to estimate the jacobian at the predictor point!!!
ypred = func(x, res.x)
cols = ypred.shape[1]
rows = len(x)
if callable(res.jac):
delta = res.jac(x)
else:
delta = np.zeros((cols * rows, p))
fdiffstep = np.spacing(np.abs(res.x))**(1 / 3)
# print('diff_step = {0}'.format(fdiffstep))
# print('popt = {0}'.format(res.x))
for i in range(p):
change = np.zeros(p)
if res.x[i] == 0:
nb = np.sqrt(LA.norm(res.x))
change[i] = fdiffstep[i] * (nb + (nb == 0))
else:
change[i] = fdiffstep[i] * res.x[i]
predplus = func(x, res.x + change)
for j in range(cols):
for k in range(rows):
n = int(j * rows + k)
with warnings.catch_warnings():
warnings.filterwarnings('error')
try:
delta[n, i] = (predplus[k, j] -
ypred[k, j]) / change[i]
except Warning as e:
print(e)
print('change:')
print(change)
print('res.x:')
print(res.x)
# print('delta = {0}'.format(delta))
# Find R to get the variance
_, R = LA.qr(res.jac)
# Get the rank of jac
rankJ = res.jac.shape[1]
Rinv = LA.pinv(R)
pinvJTJ = np.dot(Rinv, Rinv.T)
# The residual
resid = res.fun
n = len(resid)
# Get MSE. The degrees of freedom when J is full rank is v = n-p and n-rank(J) otherwise
mse = (LA.norm(resid))**2 / (n - rankJ)
# Calculate Sigma if usingJ
Sigma = mse * pinvJTJ
# Compute varpred
varpred = np.sum(np.dot(delta, Sigma) * delta, axis=1)
# print('varpred = {0}, len: '.format(varpred,len(varpred)))
alpha = 1.0 - confidence
if mode == 'observation':
# Assume a constant variance model if errorModelInfo and weights are
# not supplied.
errorVar = mse * np.ones(delta.shape[0])
# print('errorVar = {0}, len: '.format(errorVar,len(errorVar)))
varpred += errorVar
# The significance
if simultaneous:
from scipy.stats.distributions import f
sch = [rankJ + 1]
crit = f.ppf(1.0 - alpha, sch, n - rankJ)
else:
from scipy.stats.distributions import t
crit = t.ppf(1.0 - alpha / 2.0, n - rankJ)
delta = np.sqrt(varpred) * crit
lpb = np.empty((rows, cols), dtype=np.float)
upb = np.empty((rows, cols), dtype=np.float)
for j in range(cols):
for k in range(rows):
n = int(j * rows + k)
lpb[k, j] = ypred[k, j] - delta[n]
upb[k, j] = ypred[k, j] + delta[n]
# lpb = ypred - delta
# upb = ypred + delta
return ypred, lpb, upb
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If `polish`
was employed, and a lower minimum was obtained by the polishing,
then OptimizeResult also contains the ``jac`` attribute.
"""
nit, warning_flag = 0, False
status_message = _status_message['success']
# The population may have just been initialized (all entries are
# np.inf). If it has you have to calculate the initial energies.
# Although this is also done in the evolve generator it's possible
# that someone can set maxiter=0, at which point we still want the
# initial energies to be calculated (the following loop isn't run).
if np.all(np.isinf(self.population_energies)):
self._calculate_population_energies()
# do the optimisation.
for nit in xrange(1, self.maxiter + 1):
# evolve the population by a generation
try:
next(self)
except StopIteration:
warning_flag = True
status_message = _status_message['maxfev']
break
if self.disp:
print("differential_evolution step %d: f(x)= %g"
% (nit,
self.population_energies[0]))
# should the solver terminate?
convergence = self.convergence
if (self.callback and
self.callback(self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
break
if np.any(np.isinf(self.population_energies)):
intol = False
else:
intol = (np.std(self.population_energies) <=
self.atol +
self.tol * np.abs(np.mean(self.population_energies)))
if warning_flag or intol:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(
x=self.x,
fun=self.population_energies[0],
nfev=self._nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
self._nfev += result.nfev
DE_result.nfev = self._nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If polish
was employed, then OptimizeResult also contains the ``hess_inv`` and
``jac`` attributes.
"""
nfev, nit, warning_flag = 0, 0, False
status_message = _status_message['success']
# calculate energies to start with
for index, candidate in enumerate(self.population):
parameters = self._scale_parameters(candidate)
self.population_energies[index] = self.func(parameters, *self.args)
nfev += 1
if nfev > self.maxfun:
warning_flag = True
status_message = _status_message['maxfev']
break
minval = np.argmin(self.population_energies)
# put the lowest energy into the best solution position.
lowest_energy = self.population_energies[minval]
self.population_energies[minval] = self.population_energies[0]
self.population_energies[0] = lowest_energy
self.population[[0, minval], :] = self.population[[minval, 0], :]
if warning_flag:
return OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag != True))
# do the optimisation.
for nit in range(1, self.maxiter + 1):
if self.dither is not None:
self.scale = self.random_number_generator.rand() * (
self.dither[1] - self.dither[0]) + self.dither[0]
for candidate in range(np.size(self.population, 0)):
if nfev > self.maxfun:
warning_flag = True
status_message = _status_message['maxfev']
break
trial = self._mutate(candidate)
self._ensure_constraint(trial)
parameters = self._scale_parameters(trial)
energy = self.func(parameters, *self.args)
nfev += 1
if energy < self.population_energies[candidate]:
self.population[candidate] = trial
self.population_energies[candidate] = energy
if energy < self.population_energies[0]:
self.population_energies[0] = energy
self.population[0] = trial
# stop when the fractional s.d. of the population is less than tol
# of the mean energy
convergence = (
np.std(self.population_energies) /
np.abs(np.mean(self.population_energies) + _MACHEPS))
if self.disp:
print("differential_evolution step %d: f(x)= %g" %
(nit, self.population_energies[0]))
if (self.callback and
self.callback(self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
break
if convergence < self.tol or warning_flag:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag != True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
nfev += result.nfev
DE_result.nfev = nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If `polish`
was employed, and a lower minimum was obtained by the polishing,
then OptimizeResult also contains the ``jac`` attribute.
"""
nit, warning_flag = 0, False
status_message = _status_message['success']
# The population may have just been initialized (all entries are
# np.inf). If it has you have to calculate the initial energies.
# Although this is also done in the evolve generator it's possible
# that someone can set maxiter=0, at which point we still want the
# initial energies to be calculated (the following loop isn't run).
if np.all(np.isinf(self.population_energies)):
self.population_energies[:] = self._calculate_population_energies(
self.population)
self._promote_lowest_energy()
# do the optimisation.
for nit in xrange(1, self.maxiter + 1):
# evolve the population by a generation
try:
next(self)
except StopIteration:
warning_flag = True
if self._nfev > self.maxfun:
status_message = _status_message['maxfev']
elif self._nfev == self.maxfun:
status_message = ('Maximum number of function evaluations'
' has been reached.')
break
if self.disp:
print("differential_evolution step %d: f(x)= %g"
% (nit,
self.population_energies[0]))
# should the solver terminate?
convergence = self.convergence
if (self.callback and
self.callback(self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
break
if np.any(np.isinf(self.population_energies)):
intol = False
else:
intol = (np.std(self.population_energies) <=
self.atol +
self.tol * np.abs(np.mean(self.population_energies)))
if warning_flag or intol:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(
x=self.x,
fun=self.population_energies[0],
nfev=self._nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T)
self._nfev += result.nfev
DE_result.nfev = self._nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If `polish`
was employed, and a lower minimum was obtained by the polishing,
then OptimizeResult also contains the ``jac`` attribute.
"""
nit, warning_flag = 0, False
status_message = _status_message['success']
# The population may have just been initialized (all entries are
# np.inf). If it has you have to calculate the initial energies.
# Although this is also done in the evolve generator it's possible
# that someone can set maxiter=0, at which point we still want the
# initial energies to be calculated (the following loop isn't run).
if np.all(np.isinf(self.population_energies)):
self._calculate_population_energies()
for nmig in xrange(1,self.number_of_migrations+1):
if nmig != 1:
# Get the host node
host = int(self.island_marker[-1])
# Get all the neighbors list
neighbors = self.topology.neighbors(host)
neighbor_results = {}
neighbor_energy_results = {}
for each_neighbor in neighbors:
replacement = client.get(self.key + str(each_neighbor))
if replacement is None:
for _ in range(int(self.wait_time / self.poll_time)):
replacement = client.get(self.key + str(each_neighbor))
if replacement is None:
print("POLLING!!!")
time.sleep(self.poll_time)
else:
break
if replacement is not None:
neighbor_results[each_neighbor] = np.array([float(items) for items in replacement.split(",")])
neighbor_energy_results[each_neighbor] = self.func(neighbor_results[each_neighbor],*self.args)
total_computed_neighbors = len(neighbor_results)
energies = []
for each_neighbor in neighbor_results.keys():
energies.append((neighbor_results[each_neighbor],neighbor_energy_results[each_neighbor]))
for pop_index in range(1,total_computed_neighbors+1):
energies.append((self.population[pop_index],self.population_energies[pop_index]))
energies.sort(key=lambda x:x[-1])
energies = energies[:total_computed_neighbors]
for pop_index in range(1, total_computed_neighbors+1):
self.population[pop_index] = energies[pop_index-1][0]
self.population_energies[pop_index] = energies[pop_index-1][1]
# do the optimisation.
is_optimisation_complete = False
for nit in xrange(1, self.maxiter + 1):
# evolve the population by a generation
try:
next(self)
except StopIteration:
warning_flag = True
status_message = _status_message['maxfev']
#is_optimisation_complete = False
break
if self.disp:
print("differential_evolution step %d: f(x)= %g"
% (nit,
self.population_energies[0]))
# should the solver terminate?
convergence = self.convergence
if (self.callback and
self.callback(self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
is_optimisation_complete = False
break
intol = (np.std(self.population_energies) <=
self.atol +
self.tol * np.abs(np.mean(self.population_energies)))
if intol:
is_optimisation_complete = False
if warning_flag or intol:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
client.set(self.island_marker, ",".join([str(items) for items in self.x]))
print("MARKED IN MEMCACHE")
print(self.island_marker, ",".join([str(items) for items in self.x]))
if not is_optimisation_complete:
#break
print("Exited due to some break condition above!!", status_message)
DE_result = OptimizeResult(
x=self.x,
fun=self.population_energies[0],
nfev=self._nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
self._nfev += result.nfev
DE_result.nfev = self._nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If polish
was employed, then OptimizeResult also contains the ``hess_inv`` and
``jac`` attributes.
"""
nfev, nit, warning_flag = 0, 0, False
status_message = _status_message['success']
# calculate energies to start with
for index, candidate in enumerate(self.population):
parameters = self._scale_parameters(candidate)
self.population_energies[index] = self.func(parameters,
*self.args)
nfev += 1
if nfev > self.maxfun:
warning_flag = True
status_message = _status_message['maxfev']
break
minval = np.argmin(self.population_energies)
# put the lowest energy into the best solution position.
lowest_energy = self.population_energies[minval]
self.population_energies[minval] = self.population_energies[0]
self.population_energies[0] = lowest_energy
self.population[[0, minval], :] = self.population[[minval, 0], :]
if warning_flag:
return OptimizeResult(
x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag != True))
# do the optimisation.
for nit in range(1, self.maxiter + 1):
if self.dither is not None:
self.scale = self.random_number_generator.rand(
) * (self.dither[1] - self.dither[0]) + self.dither[0]
for candidate in range(np.size(self.population, 0)):
if nfev > self.maxfun:
warning_flag = True
status_message = _status_message['maxfev']
break
trial = self._mutate(candidate)
self._ensure_constraint(trial)
parameters = self._scale_parameters(trial)
energy = self.func(parameters, *self.args)
nfev += 1
if energy < self.population_energies[candidate]:
self.population[candidate] = trial
self.population_energies[candidate] = energy
if energy < self.population_energies[0]:
self.population_energies[0] = energy
self.population[0] = trial
# stop when the fractional s.d. of the population is less than tol
# of the mean energy
convergence = (np.std(self.population_energies) /
np.abs(np.mean(self.population_energies) +
_MACHEPS))
if self.disp:
print("differential_evolution step %d: f(x)= %g"
% (nit,
self.population_energies[0]))
if (self.callback and
self.callback(self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
break
if convergence < self.tol or warning_flag:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(
x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag != True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
nfev += result.nfev
DE_result.nfev = nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def fmin_bfgs_f(f_g,
x0,
B0=None,
M=2,
gtol=1e-5,
Delta=10.0,
maxiter=None,
callback=None,
norm_ord=np.Inf,
**_kwargs):
"""test BFGS with nonmonote line search"""
fk, gk = f_g(x0)
if B0 is None:
Bk = np.eye(len(x0))
else:
Bk = B0
Hk = np.linalg.inv(Bk)
maxiter = 200 * len(x0) if maxiter is None else maxiter
xk = x0
norm = lambda x: np.linalg.norm(x, ord=norm_ord)
theta = 0.9
C = 0.5
k = 0
old_old_fval = fk + np.linalg.norm(gk) / 2
old_fval = fk
f_s = Seq(M)
f_s.add(fk)
flag = 0
re_search = 0
for k in range(maxiter):
if norm(gk) <= gtol:
break
dki = -np.dot(Hk, gk)
try:
pk = dki
f = f_g.fun
myfprime = f_g.grad
gfk = gk
old_fval = fk
(
alpha_k,
fc,
gc,
old_fval,
old_old_fval,
gfkp1,
) = line_search_wolfe2(f, myfprime, xk, pk, gfk, f_s.get_max(),
old_fval, old_old_fval)
except Exception as e:
print(e)
re_search += 1
xk = xk + dki
fk, gk = f_g(xk)
old_fval, old_old_fval = fk, old_fval
f_s.add(fk)
if re_search > 2:
flag = 1
break
continue
if alpha_k is None:
print("alpha is None")
xk = xk + dki
fk, gk = f_g(xk)
old_fval, old_old_fval = fk, old_fval
f_s.add(fk)
re_search += 1
if re_search > 2:
flag = 1
break
continue
dki = alpha_k * pk
# fki, gki = f_g(xk + dki)
fki, gki = old_fval, gfkp1
Aredk = fk - fki
Predk = -(np.dot(gk, dki) + 0.5 * np.dot(np.dot(Bk, dki), dki))
rk = Aredk / Predk
xk = xk + dki
fk = fki
yk = gki - gk
tk = C + max(0, -np.dot(yk, dki) / norm(dki)**2) / norm(gk)
ystark = (1 - theta) * yk + theta * tk * norm(gk) * dki
gk = gki
bs = np.dot(Bk, dki)
Bk = (Bk + np.outer(yk, yk) / np.dot(yk, dki) -
np.outer(bs, bs) / np.dot(bs, dki))
# sk = dki
# rhok = 1.0 / (np.dot(yk, sk))
# A1 = 1 - np.outer(sk, yk) * rhok
# A2 = 1 - np.outer(yk, sk) * rhok
# Hk = np.dot(A2, np.dot(Hk, A1)) - (rhok * np.outer(sk, sk))
# Bk = Bk + np.outer(ystark, ystark)/np.dot(ystark, dki) - \
# np.outer(bs, bs)/np.dot(bs, dki) # MBFGS
# print(np.dot(Hk, Bk))
try:
Hk = np.linalg.inv(Bk)
except Exception:
pass
f_s.add(fk)
if callback is not None:
callback(xk)
else:
flag = 2
# print("fit final: ", k, p, f_g.ncall)
s = OptimizeResult()
s.messgae = message_dict[flag]
s.fun = float(fk)
s.nit = k
s.nfev = f_g.ncall
s.njev = f_g.ncall
s.status = flag
s.x = np.array(xk)
s.jac = np.array(gk)
s.hess = np.array(Bk)
s.success = flag == 0
return s
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If polish
was employed, then OptimizeResult also contains the ``hess_inv`` and
``jac`` attributes.
"""
#nit = self.niter
start_time = self.time
warning_flag = False
if time.time()-start_time > self.maxtime and self.maxtime is not None :
# result = {'population':self.population,
# 'population_energies':self.population_energies,
# 'niter' : nit,
# 'message': 'Maximum time has been exceeded.',
# 'success' : False }
result = OptimizeResult(
population = self.population,
population_energies = self.population_energies,
nit = self.niter,
message = 'Maximum time has been exceeded.',
success = False )
return result
status_message = _status_message['success']
#print(self.population_energies[0])
# do the optimisation.
for nit in range(self.niter, self.maxiter + 1):
population_count = np.size(self.population, 0)
if self.dither is not None:
self.scale = self.random_number_generator.rand(
) * (self.dither[1] - self.dither[0]) + self.dither[0]
Parameters=[]
Trials=[]
for candidate in range(population_count):
trial = self._mutate(candidate)
self._ensure_constraint(trial)
Trials.append(trial)
Parameters.append( self._scale_parameters(trial) )
pool=multiprocessing.Pool(self.ncore)
Energies = pool.map(self.func, Parameters)
pool.close()
pool.join()
iNan = []
for i in range(population_count):
if self.population_energies[i] != self.population_energies[i]:
iNan.append(i)
self.population_energies = np.delete(self.population_energies,iNan)
self.population = np.delete(self.population,iNan,0)
population_count = np.size(self.population, 0)
for candidate in range(population_count):
if Energies[candidate] < self.population_energies[candidate]:
self.population[candidate] = Trials[candidate]
self.population_energies[candidate] = Energies[candidate]
if Energies[candidate] < self.population_energies[0]:
self.population_energies[0] = Energies[candidate]
self.population[0] = Trials[candidate]
# stop when the fractional s.d. of the population is less than tol
# of the mean energy
convergence = (np.std(self.population_energies) /
np.abs(np.mean(self.population_energies) +
_MACHEPS))
if self.disp:
print("differential_evolution step %d: f(x)= %g"
% (nit,
self.population_energies[0]))
print("total population at step %d is %d"
%(nit, population_count ) )
if (self.callback and
self.callback(self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
break
if convergence < self.tol or warning_flag:
break
if time.time()-start_time > self.maxtime and self.maxtime is not None :
result = OptimizeResult(
population = self.population,
population_energies = self.population_energies,
nit = self.niter,
message = 'Maximum time has been exceeded.',
success = False )
return result
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(
x=self.x,
fun=self.population_energies[0],
nit=nit,
message=status_message,
success=(warning_flag != True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T)
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If `polish`
was employed, and a lower minimum was obtained by the polishing,
then OptimizeResult also contains the ``jac`` attribute.
"""
nfev, nit, warning_flag = 0, 0, False
status_message = _status_message['success']
# calculate energies to start with
parameters = np.zeros_like(self.population, order='F')
for index, candidate in enumerate(self.population):
parameters[index, :] = self._scale_parameters(candidate)
self.population_energies[:] = self.evaluate_func(parameters)
nfev += self.num_population_members
# put the lowest energy into the best solution position.
minval = np.argmin(self.population_energies)
self._swap_best(minval)
if warning_flag:
return OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
# do the optimisation.
trials = np.zeros_like(self.population, order='F')
for nit in range(1, self.maxiter + 1):
if self.dither is not None:
self.scale = self.random_number_generator.rand() * (
self.dither[1] - self.dither[0]) + self.dither[0]
# Unlike the standard DE, all the trials are created first and later
# evaluated simultaneously.
for index in range(self.num_population_members):
# create a trial solution
trials[index][:] = self._mutate(index)
# ensuring that it's in the range [0, 1)
self._ensure_constraint(trials[index])
# scale from [0, 1) to the actual parameter value
parameters[index][:] = self._scale_parameters(trials[index])
# determine the energy of the objective function
energies = self.evaluate_func(parameters)
nfev += self.num_population_members
# if the energy of the trial candidate is lower than the
# original population member then replace it
for index in range(self.num_population_members):
if energies[index] < self.population_energies[index]:
self.population[index] = trials[index]
self.population_energies[index] = energies[index]
# if the trial candidate also has a lower energy than the
# best solution then replace that as well
minval = np.argmin(self.population_energies)
self._swap_best(minval)
# stop when the fractional s.d. of the population is less than tol
# of the mean energy
convergence = (
np.std(self.population_energies) /
np.abs(np.mean(self.population_energies) + _MACHEPS))
if self.disp:
print("differential_evolution step %d: f(x)= %g" %
(nit, self.population_energies[0]))
if self.callbacks:
for callback in self.callbacks:
callback(step=nit,
parameter=self.x,
cost=self.population_energies[0])
if (self.earlystop and self.earlystop(
self.x, convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('earlystop function requested stop early '
'by returning True')
break
if convergence < self.tol or warning_flag:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
nfev += result.nfev
DE_result.nfev = nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def solve(self):
nfev, nit, warning_flag = 0, 0, False
status_message = _status_message['success']
# calculate energies to start with
for index, candidate in enumerate(self.population):
parameters = self._scale_parameters(candidate)
self.population_energies[index] = self.func(parameters,
*self.args)
nfev += 1
if nfev > self.maxfun:
warning_flag = True
status_message = _status_message['maxfev']
break
minval = np.argmin(self.population_energies)
# put the lowest energy into the best solution position.
lowest_energy = self.population_energies[minval]
self.population_energies[minval] = self.population_energies[0]
self.population_energies[0] = lowest_energy
self.population[[0, minval], :] = self.population[[minval, 0], :]
if warning_flag:
return OptimizeResult(
x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
# do the optimisation.
start_time = time.time()
nit = 0
while nit < self.maxiter + 1:
nit += 1
if start_time + self.max_execution_time < time.time():
warning_flag = True
status_message = 'Max execution time reached'
break
if self.dither is not None:
self.scale = self.random_number_generator.rand(
) * (self.dither[1] - self.dither[0]) + self.dither[0]
for candidate in range(np.size(self.population, 0)):
if nfev > self.maxfun:
warning_flag = True
status_message = _status_message['maxfev']
break
trial = self._mutate(candidate)
self._ensure_constraint(trial)
parameters = self._scale_parameters(trial)
energy = self.func(parameters, *self.args)
nfev += 1
if energy < self.population_energies[candidate]:
self.population[candidate] = trial
self.population_energies[candidate] = energy
if energy < self.population_energies[0]:
self.population_energies[0] = energy
self.population[0] = trial
# stop when the fractional s.d. of the population is less than tol
# of the mean energy
convergence = (np.std(self.population_energies) /
np.abs(np.mean(self.population_energies) +
_MACHEPS))
if self.disp:
print("differential_evolution step %d: f(x)= %g"
% (nit,
self.population_energies[0]))
if (self.callback and
self.callback(self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
break
if convergence < self.tol or warning_flag:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(
x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
nfev += result.nfev
DE_result.nfev = nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def optimize_minimize_mhmcmc_cluster(objective,
bounds,
args=(),
x0=None,
T=1,
N=3,
burnin=100000,
maxiter=1000000,
target_ar=0.4,
ar_tolerance=0.05,
cluster_eps=DEFAULT_CLUSTER_EPS,
rnd_seed=None,
collect_samples=None,
logger=None):
"""
Minimize objective function and return up to N local minima solutions.
:param objective: Objective function to minimize. Takes unpacked args as function call arguments and returns
a float.
:type objective: Callable(\*args) -> float
:param bounds: Bounds of the parameter space.
:type bounds: scipy.optimize.Bounds
:param args: Any additional fixed parameters needed to completely specify the objective function.
:type args: tuple or list
:param x0: Initial guess. If None, will be selected randomly and uniformly within the parameter bounds.
:type x0: numpy.array with same shape as elements of bounds
:param T: The "temperature" parameter for the accept or reject criterion. To sample the domain well,
should be in the order of the typical difference in local minima objective valuations.
:type T: float
:param N: Maximum number of minima to return
:type N: int
:param burnin: Number of random steps to discard before starting to accumulate statistics.
:type burnin: int
:param maxiter: Maximum number of steps to take (including burnin).
:type maxiter: int
:param target_ar: Target acceptance rate of point samples generated by stepping.
:type target_ar: float between 0 and 1
:param ar_tolerance: Tolerance on the acceptance rate before actively adapting the step size.
:type ar_tolerance: float
:param cluster_eps: Point proximity tolerance for DBSCAN clustering, in normalized bounds coordinates.
:type cluster_eps: float
:param rnd_seed: Random seed to force deterministic behaviour
:type rnd_seed: int
:param collect_samples: If not None and integral type, collect collect_samples at regular intervals
and return as part of solution.
:type collect_samples: int or NoneType
:param logger: Logger instance for outputting log messages.
:return: OptimizeResult containing solution(s) and solver data.
:rtype: scipy.optimize.OptimizeResult with additional attributes
"""
@call_counter
def obj_counted(*args):
return objective(*args)
# end func
assert maxiter >= 2 * burnin, "maxiter {} should be at least twice burnin steps {}".format(
maxiter, burnin)
main_iter = maxiter - burnin
if collect_samples is not None:
assert isinstance(collect_samples,
int), "collect_samples expected to be integral type"
assert collect_samples > 0, "collect_samples expected to be positive"
# end if
beta = 1.0 / T
if rnd_seed is None:
rnd_seed = int(time.time() * 1000) % (1 << 31)
# end if
np.random.seed(rnd_seed)
if logger:
logger.info('Using random seed {}'.format(rnd_seed))
# end
if x0 is None:
x0 = np.random.uniform(bounds.lb, bounds.ub)
# end if
assert np.all((x0 >= bounds.lb) & (x0 <= bounds.ub))
x = x0.copy()
funval = obj_counted(x, *args)
# Set up stepper with adaptive acceptance rate
stepper = BoundedRandNStepper(bounds)
stepper = AdaptiveStepsize(stepper,
accept_rate=target_ar,
ar_tolerance=ar_tolerance,
interval=50)
# -------------------------------
# DO BURN-IN
rejected_randomly = 0
accepted_burnin = 0
tracked_range = tqdm(range(burnin), total=burnin, desc='BURN-IN')
if logger:
stepper.logger = lambda msg: tracked_range.write(logger.name + ':' +
msg)
else:
stepper.logger = tracked_range.write
# end if
for _ in tracked_range:
x_new = stepper(x)
funval_new = obj_counted(x_new, *args)
log_alpha = -(funval_new - funval) * beta
if log_alpha > 0 or np.log(np.random.rand()) <= log_alpha:
x = x_new
funval = funval_new
stepper.notify_accept()
accepted_burnin += 1
elif log_alpha <= 0:
rejected_randomly += 1
# end if
# end for
ar = float(accepted_burnin) / burnin
if logger:
logger.info("Burn-in acceptance rate: {}".format(ar))
# end if
# -------------------------------
# DO MAIN LOOP
if collect_samples is not None:
nsamples = min(collect_samples, main_iter)
sample_cadence = main_iter / nsamples
samples = np.zeros((nsamples, len(x)))
samples_fval = np.zeros(nsamples)
# end if
accepted = 0
rejected_randomly = 0
minima_sorted = SortedList(
key=lambda rec: rec[1]) # Sort by objective function value
hist = HistogramIncremental(bounds, nbins=100)
# Cached a lot of potential minimum values, as these need to be clustered before return N results
N_cached = int(np.ceil(N * main_iter / 500))
next_sample = 0.0
sample_count = 0
tracked_range = tqdm(range(main_iter), total=main_iter, desc='MAIN')
if logger:
stepper.logger = lambda msg: tracked_range.write(logger.name + ':' +
msg)
else:
stepper.logger = tracked_range.write
# end if
for i in tracked_range:
if collect_samples and i >= next_sample:
assert sample_count < collect_samples
samples[sample_count] = x
samples_fval[sample_count] = funval
sample_count += 1
next_sample += sample_cadence
# end if
x_new = stepper(x)
funval_new = obj_counted(x_new, *args)
log_alpha = -(funval_new - funval) * beta
if log_alpha > 0 or np.log(np.random.rand()) <= log_alpha:
x = x_new
funval = funval_new
minima_sorted.add((x, funval))
if len(minima_sorted) > N_cached:
minima_sorted.pop()
# end if
stepper.notify_accept()
hist += x
accepted += 1
elif log_alpha <= 0:
rejected_randomly += 1
# end if
# end for
stepper.logger = None
ar = float(accepted) / main_iter
if logger:
logger.info("Acceptance rate: {}".format(ar))
logger.info("Best minima (before clustering):\n{}".format(
np.array([_mx[0] for _mx in minima_sorted[:10]])))
# end if
# -------------------------------
# Cluster minima and associate each cluster with a local minimum.
# Using a normalized coordinate space for cluster detection.
x_range = bounds.ub - bounds.lb
pts = np.array([x[0] for x in minima_sorted])
fvals = np.array([x[1] for x in minima_sorted])
pts_norm = (pts - bounds.lb) / x_range
_, labels = dbscan(pts_norm, eps=cluster_eps, min_samples=21, n_jobs=-1)
# Compute mean of each cluster and evaluate objective function at cluster mean locations.
minima_candidates = []
for grp in range(max(labels) + 1):
mask = (labels == grp)
mean_loc = np.mean(pts[mask, :], axis=0)
# Evaluate objective function precisely at the mean location of each cluster
fval = obj_counted(mean_loc, *args)
minima_candidates.append((mean_loc, grp, fval))
# end for
# Rank minima locations by objective function.
minima_candidates.sort(key=lambda c: c[2])
# Pick up to N solutions
solutions = minima_candidates[:N]
# Put results into OptimizeResult container.
# Add histograms to output result (in form of scipy.stats.rv_histogram)
solution = OptimizeResult()
solution.x = np.array([s[0] for s in solutions])
solution.clusters = [pts[(labels == s[1])] for s in solutions]
solution.cluster_funvals = [fvals[(labels == s[1])] for s in solutions]
solution.bins = hist.bins
solution.distribution = hist.histograms
solution.acceptance_rate = ar
solution.success = True
solution.status = 0
if len(solutions) > 0:
solution.message = 'SUCCESS: Found {} local minima'.format(
len(solutions))
else:
solution.message = 'WARNING: Found no clusters within tolerance {}'.format(
cluster_eps)
# end if
solution.fun = np.array([s[2] for s in solutions])
solution.jac = None
solution.nfev = obj_counted.counter
solution.njev = 0
solution.nit = main_iter
solution.maxcv = None
solution.samples = samples if collect_samples else None
solution.sample_funvals = samples_fval if collect_samples else None
solution.bounds = bounds
solution.version = 's0.3' # Solution version for future traceability
solution.rnd_seed = rnd_seed
return solution
def solve(self):
"""
Runs the DifferentialEvolutionSolver.
Returns
-------
res : OptimizeResult
The optimization result represented as a ``OptimizeResult`` object.
Important attributes are: ``x`` the solution array, ``success`` a
Boolean flag indicating if the optimizer exited successfully and
``message`` which describes the cause of the termination. See
`OptimizeResult` for a description of other attributes. If `polish`
was employed, and a lower minimum was obtained by the polishing,
then OptimizeResult also contains the ``jac`` attribute.
"""
nit, warning_flag = 0, False
# dictionary that holds standard status messages of optimizers
status_message = _status_message['success']
# The population may have just been initialized (all entries are
# np.inf). If it has you have to calculate the initial energies.
# Although this is also done in the evolve generator it's possible
# that someone can set maxiter=0, at which point we still want the
# initial energies to be calculated (the following loop isn't run).
#np.all checks that there are no 0's in the array
if self.maxiter == 0:
if np.all(np.isinf(self.population_energies)):
if self.disp:
print("Calculating initial energies when maxiter = 0")
self._calculate_population_energies()
# for i in range(self.num_population_members):
# print(self.population[i,:])
# do the optimisation.
for nit in xrange(1, self.maxiter + 1):
if self.disp:
print("iter: ", nit)
# evolve the population by a generation
try:
next(self)
except StopIteration:
warning_flag = True
status_message = _status_message['maxfev']
break
print("differential_evolution step %d: f(x)= %g" %
(nit, self.population_energies[0]))
#save populations at each iter and rank to analyze after
# np.save("before_rank"+str(self.rank)+"iter"+str(nit), self.population)
#migrate
self.migration()
# np.save("after_rank"+str(self.rank)+"iter"+str(nit), self.population)
# should the solver terminate?
# print("Checking if should converge")
# convergence = self.convergence
#
# if (self.callback and
# self.callback(self._scale_parameters(self.population[0]),
# convergence=self.tol / convergence) is True):
#
# warning_flag = True
# status_message = ('callback function requested stop early '
# 'by returning True')
# break
# print("checking if tolerance level reached")
## intol = (np.std(self.population_energies) <=
## self.atol +
## self.tol * np.abs(np.mean(self.population_energies)))
#
# intol = self.population_energies[0] <= self.mse_thresh
# if warning_flag or intol:
# print("stopping iterations")
# break
print("Starting next iter")
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=self._nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
print("done iters")
if self.polish:
print("performing final polishing")
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
self._nfev += result.nfev
DE_result.nfev = self._nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
def solve(self):
nfev, nit, warning_flag = 0, 0, False
status_message = _status_message['success']
# calculate energies to start with
for index, candidate in enumerate(self.population):
parameters = self._scale_parameters(candidate)
self.population_energies[index] = self.func(parameters, *self.args)
nfev += 1
if nfev > self.maxfun:
warning_flag = True
status_message = _status_message['maxfev']
break
minval = np.argmin(self.population_energies)
# put the lowest energy into the best solution position.
lowest_energy = self.population_energies[minval]
self.population_energies[minval] = self.population_energies[0]
self.population_energies[0] = lowest_energy
self.population[[0, minval], :] = self.population[[minval, 0], :]
if warning_flag:
return OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
# do the optimisation.
start_time = time.time()
nit = 0
while nit < self.maxiter + 1:
nit += 1
if start_time + self.max_execution_time < time.time():
warning_flag = True
status_message = 'Max execution time reached'
break
if self.dither is not None:
self.scale = self.random_number_generator.rand() * (
self.dither[1] - self.dither[0]) + self.dither[0]
for candidate in range(np.size(self.population, 0)):
if nfev > self.maxfun:
warning_flag = True
status_message = _status_message['maxfev']
break
trial = self._mutate(candidate)
self._ensure_constraint(trial)
parameters = self._scale_parameters(trial)
energy = self.func(parameters, *self.args)
nfev += 1
if energy < self.population_energies[candidate]:
self.population[candidate] = trial
self.population_energies[candidate] = energy
if energy < self.population_energies[0]:
self.population_energies[0] = energy
self.population[0] = trial
# stop when the fractional s.d. of the population is less than tol
# of the mean energy
convergence = (
np.std(self.population_energies) /
np.abs(np.mean(self.population_energies) + _MACHEPS))
if self.disp:
print("differential_evolution step %d: f(x)= %g" %
(nit, self.population_energies[0]))
if (self.callback and
self.callback(self._scale_parameters(self.population[0]),
convergence=self.tol / convergence) is True):
warning_flag = True
status_message = ('callback function requested stop early '
'by returning True')
break
if convergence < self.tol or warning_flag:
break
else:
status_message = _status_message['maxiter']
warning_flag = True
DE_result = OptimizeResult(x=self.x,
fun=self.population_energies[0],
nfev=nfev,
nit=nit,
message=status_message,
success=(warning_flag is not True))
if self.polish:
result = minimize(self.func,
np.copy(DE_result.x),
method='L-BFGS-B',
bounds=self.limits.T,
args=self.args)
nfev += result.nfev
DE_result.nfev = nfev
if result.fun < DE_result.fun:
DE_result.fun = result.fun
DE_result.x = result.x
DE_result.jac = result.jac
# to keep internal state consistent
self.population_energies[0] = result.fun
self.population[0] = self._unscale_parameters(result.x)
return DE_result
