def predint_multi(x: np.ndarray, xd: np.ndarray, yd: np.ndarray, func: Callable[[np.ndarray, np.ndarray], np.ndarray], res: OptimizeResult, **kwargs): """ This function estimates the prediction bands for the fit (see: https://www.mathworks.com/help/curvefit/confidence-and-prediction-bounds.html) Parameters ---------- x: np.ndarray The requested x points for the bands xd: np.ndarray The x datapoints yd: np.ndarray The y datapoints func: Callable[[np.ndarray, np.ndarray] The fitted function res: OptimizeResult The optimzied result from least_squares minimization **kwargs confidence: float The confidence level (default 0.95) simulateneous: bool True if the bound type is simultaneous, false otherwise mode: [functional, observation] Default observation """ if len(yd) != len(xd): raise ValueError('The length of the observations should be the same ' + 'as the length of the predictions.') if len(yd) <= 1: raise ValueError('Too few datapoints') from scipy.optimize import optimize if not isinstance(res, optimize.OptimizeResult): raise ValueError( 'Argument \'res\' should be an instance of \'scipy.optimize.OptimizeResult\'' ) simultaneous = kwargs.get('simultaneous', True) mode = kwargs.get('mode', 'observation') confidence = kwargs.get('confidence', 0.95) p = len(res.x) # Needs to estimate the jacobian at the predictor point!!! ypred = func(x, res.x) cols = ypred.shape[1] rows = len(x) if callable(res.jac): delta = res.jac(x) else: delta = np.zeros((cols * rows, p)) fdiffstep = np.spacing(np.abs(res.x))**(1 / 3) # print('diff_step = {0}'.format(fdiffstep)) # print('popt = {0}'.format(res.x)) for i in range(p): change = np.zeros(p) if res.x[i] == 0: nb = np.sqrt(LA.norm(res.x)) change[i] = fdiffstep[i] * (nb + (nb == 0)) else: change[i] = fdiffstep[i] * res.x[i] predplus = func(x, res.x + change) for j in range(cols): for k in range(rows): n = int(j * rows + k) with warnings.catch_warnings(): warnings.filterwarnings('error') try: delta[n, i] = (predplus[k, j] - ypred[k, j]) / change[i] except Warning as e: print(e) print('change:') print(change) print('res.x:') print(res.x) # print('delta = {0}'.format(delta)) # Find R to get the variance _, R = LA.qr(res.jac) # Get the rank of jac rankJ = res.jac.shape[1] Rinv = LA.pinv(R) pinvJTJ = np.dot(Rinv, Rinv.T) # The residual resid = res.fun n = len(resid) # Get MSE. The degrees of freedom when J is full rank is v = n-p and n-rank(J) otherwise mse = (LA.norm(resid))**2 / (n - rankJ) # Calculate Sigma if usingJ Sigma = mse * pinvJTJ # Compute varpred varpred = np.sum(np.dot(delta, Sigma) * delta, axis=1) # print('varpred = {0}, len: '.format(varpred,len(varpred))) alpha = 1.0 - confidence if mode == 'observation': # Assume a constant variance model if errorModelInfo and weights are # not supplied. errorVar = mse * np.ones(delta.shape[0]) # print('errorVar = {0}, len: '.format(errorVar,len(errorVar))) varpred += errorVar # The significance if simultaneous: from scipy.stats.distributions import f sch = [rankJ + 1] crit = f.ppf(1.0 - alpha, sch, n - rankJ) else: from scipy.stats.distributions import t crit = t.ppf(1.0 - alpha / 2.0, n - rankJ) delta = np.sqrt(varpred) * crit lpb = np.empty((rows, cols), dtype=np.float) upb = np.empty((rows, cols), dtype=np.float) for j in range(cols): for k in range(rows): n = int(j * rows + k) lpb[k, j] = ypred[k, j] - delta[n] upb[k, j] = ypred[k, j] + delta[n] # lpb = ypred - delta # upb = ypred + delta return ypred, lpb, upb
def solve(self): """ Runs the DifferentialEvolutionSolver. Returns ------- res : OptimizeResult The optimization result represented as a ``OptimizeResult`` object. Important attributes are: ``x`` the solution array, ``success`` a Boolean flag indicating if the optimizer exited successfully and ``message`` which describes the cause of the termination. See `OptimizeResult` for a description of other attributes. If `polish` was employed, and a lower minimum was obtained by the polishing, then OptimizeResult also contains the ``jac`` attribute. """ nit, warning_flag = 0, False status_message = _status_message['success'] # The population may have just been initialized (all entries are # np.inf). If it has you have to calculate the initial energies. # Although this is also done in the evolve generator it's possible # that someone can set maxiter=0, at which point we still want the # initial energies to be calculated (the following loop isn't run). if np.all(np.isinf(self.population_energies)): self._calculate_population_energies() # do the optimisation. for nit in xrange(1, self.maxiter + 1): # evolve the population by a generation try: next(self) except StopIteration: warning_flag = True status_message = _status_message['maxfev'] break if self.disp: print("differential_evolution step %d: f(x)= %g" % (nit, self.population_energies[0])) # should the solver terminate? convergence = self.convergence if (self.callback and self.callback(self._scale_parameters(self.population[0]), convergence=self.tol / convergence) is True): warning_flag = True status_message = ('callback function requested stop early ' 'by returning True') break if np.any(np.isinf(self.population_energies)): intol = False else: intol = (np.std(self.population_energies) <= self.atol + self.tol * np.abs(np.mean(self.population_energies))) if warning_flag or intol: break else: status_message = _status_message['maxiter'] warning_flag = True DE_result = OptimizeResult( x=self.x, fun=self.population_energies[0], nfev=self._nfev, nit=nit, message=status_message, success=(warning_flag is not True)) if self.polish: result = minimize(self.func, np.copy(DE_result.x), method='L-BFGS-B', bounds=self.limits.T, args=self.args) self._nfev += result.nfev DE_result.nfev = self._nfev if result.fun < DE_result.fun: DE_result.fun = result.fun DE_result.x = result.x DE_result.jac = result.jac # to keep internal state consistent self.population_energies[0] = result.fun self.population[0] = self._unscale_parameters(result.x) return DE_result
def solve(self): """ Runs the DifferentialEvolutionSolver. Returns ------- res : OptimizeResult The optimization result represented as a ``OptimizeResult`` object. Important attributes are: ``x`` the solution array, ``success`` a Boolean flag indicating if the optimizer exited successfully and ``message`` which describes the cause of the termination. See `OptimizeResult` for a description of other attributes. If polish was employed, then OptimizeResult also contains the ``hess_inv`` and ``jac`` attributes. """ nfev, nit, warning_flag = 0, 0, False status_message = _status_message['success'] # calculate energies to start with for index, candidate in enumerate(self.population): parameters = self._scale_parameters(candidate) self.population_energies[index] = self.func(parameters, *self.args) nfev += 1 if nfev > self.maxfun: warning_flag = True status_message = _status_message['maxfev'] break minval = np.argmin(self.population_energies) # put the lowest energy into the best solution position. lowest_energy = self.population_energies[minval] self.population_energies[minval] = self.population_energies[0] self.population_energies[0] = lowest_energy self.population[[0, minval], :] = self.population[[minval, 0], :] if warning_flag: return OptimizeResult(x=self.x, fun=self.population_energies[0], nfev=nfev, nit=nit, message=status_message, success=(warning_flag != True)) # do the optimisation. for nit in range(1, self.maxiter + 1): if self.dither is not None: self.scale = self.random_number_generator.rand() * ( self.dither[1] - self.dither[0]) + self.dither[0] for candidate in range(np.size(self.population, 0)): if nfev > self.maxfun: warning_flag = True status_message = _status_message['maxfev'] break trial = self._mutate(candidate) self._ensure_constraint(trial) parameters = self._scale_parameters(trial) energy = self.func(parameters, *self.args) nfev += 1 if energy < self.population_energies[candidate]: self.population[candidate] = trial self.population_energies[candidate] = energy if energy < self.population_energies[0]: self.population_energies[0] = energy self.population[0] = trial # stop when the fractional s.d. of the population is less than tol # of the mean energy convergence = ( np.std(self.population_energies) / np.abs(np.mean(self.population_energies) + _MACHEPS)) if self.disp: print("differential_evolution step %d: f(x)= %g" % (nit, self.population_energies[0])) if (self.callback and self.callback(self._scale_parameters(self.population[0]), convergence=self.tol / convergence) is True): warning_flag = True status_message = ('callback function requested stop early ' 'by returning True') break if convergence < self.tol or warning_flag: break else: status_message = _status_message['maxiter'] warning_flag = True DE_result = OptimizeResult(x=self.x, fun=self.population_energies[0], nfev=nfev, nit=nit, message=status_message, success=(warning_flag != True)) if self.polish: result = minimize(self.func, np.copy(DE_result.x), method='L-BFGS-B', bounds=self.limits.T, args=self.args) nfev += result.nfev DE_result.nfev = nfev if result.fun < DE_result.fun: DE_result.fun = result.fun DE_result.x = result.x DE_result.jac = result.jac # to keep internal state consistent self.population_energies[0] = result.fun self.population[0] = self._unscale_parameters(result.x) return DE_result
def solve(self): """ Runs the DifferentialEvolutionSolver. Returns ------- res : OptimizeResult The optimization result represented as a ``OptimizeResult`` object. Important attributes are: ``x`` the solution array, ``success`` a Boolean flag indicating if the optimizer exited successfully and ``message`` which describes the cause of the termination. See `OptimizeResult` for a description of other attributes. If `polish` was employed, and a lower minimum was obtained by the polishing, then OptimizeResult also contains the ``jac`` attribute. """ nit, warning_flag = 0, False status_message = _status_message['success'] # The population may have just been initialized (all entries are # np.inf). If it has you have to calculate the initial energies. # Although this is also done in the evolve generator it's possible # that someone can set maxiter=0, at which point we still want the # initial energies to be calculated (the following loop isn't run). if np.all(np.isinf(self.population_energies)): self.population_energies[:] = self._calculate_population_energies( self.population) self._promote_lowest_energy() # do the optimisation. for nit in xrange(1, self.maxiter + 1): # evolve the population by a generation try: next(self) except StopIteration: warning_flag = True if self._nfev > self.maxfun: status_message = _status_message['maxfev'] elif self._nfev == self.maxfun: status_message = ('Maximum number of function evaluations' ' has been reached.') break if self.disp: print("differential_evolution step %d: f(x)= %g" % (nit, self.population_energies[0])) # should the solver terminate? convergence = self.convergence if (self.callback and self.callback(self._scale_parameters(self.population[0]), convergence=self.tol / convergence) is True): warning_flag = True status_message = ('callback function requested stop early ' 'by returning True') break if np.any(np.isinf(self.population_energies)): intol = False else: intol = (np.std(self.population_energies) <= self.atol + self.tol * np.abs(np.mean(self.population_energies))) if warning_flag or intol: break else: status_message = _status_message['maxiter'] warning_flag = True DE_result = OptimizeResult( x=self.x, fun=self.population_energies[0], nfev=self._nfev, nit=nit, message=status_message, success=(warning_flag is not True)) if self.polish: result = minimize(self.func, np.copy(DE_result.x), method='L-BFGS-B', bounds=self.limits.T) self._nfev += result.nfev DE_result.nfev = self._nfev if result.fun < DE_result.fun: DE_result.fun = result.fun DE_result.x = result.x DE_result.jac = result.jac # to keep internal state consistent self.population_energies[0] = result.fun self.population[0] = self._unscale_parameters(result.x) return DE_result
def solve(self): """ Runs the DifferentialEvolutionSolver. Returns ------- res : OptimizeResult The optimization result represented as a ``OptimizeResult`` object. Important attributes are: ``x`` the solution array, ``success`` a Boolean flag indicating if the optimizer exited successfully and ``message`` which describes the cause of the termination. See `OptimizeResult` for a description of other attributes. If `polish` was employed, and a lower minimum was obtained by the polishing, then OptimizeResult also contains the ``jac`` attribute. """ nit, warning_flag = 0, False status_message = _status_message['success'] # The population may have just been initialized (all entries are # np.inf). If it has you have to calculate the initial energies. # Although this is also done in the evolve generator it's possible # that someone can set maxiter=0, at which point we still want the # initial energies to be calculated (the following loop isn't run). if np.all(np.isinf(self.population_energies)): self._calculate_population_energies() for nmig in xrange(1,self.number_of_migrations+1): if nmig != 1: # Get the host node host = int(self.island_marker[-1]) # Get all the neighbors list neighbors = self.topology.neighbors(host) neighbor_results = {} neighbor_energy_results = {} for each_neighbor in neighbors: replacement = client.get(self.key + str(each_neighbor)) if replacement is None: for _ in range(int(self.wait_time / self.poll_time)): replacement = client.get(self.key + str(each_neighbor)) if replacement is None: print("POLLING!!!") time.sleep(self.poll_time) else: break if replacement is not None: neighbor_results[each_neighbor] = np.array([float(items) for items in replacement.split(",")]) neighbor_energy_results[each_neighbor] = self.func(neighbor_results[each_neighbor],*self.args) total_computed_neighbors = len(neighbor_results) energies = [] for each_neighbor in neighbor_results.keys(): energies.append((neighbor_results[each_neighbor],neighbor_energy_results[each_neighbor])) for pop_index in range(1,total_computed_neighbors+1): energies.append((self.population[pop_index],self.population_energies[pop_index])) energies.sort(key=lambda x:x[-1]) energies = energies[:total_computed_neighbors] for pop_index in range(1, total_computed_neighbors+1): self.population[pop_index] = energies[pop_index-1][0] self.population_energies[pop_index] = energies[pop_index-1][1] # do the optimisation. is_optimisation_complete = False for nit in xrange(1, self.maxiter + 1): # evolve the population by a generation try: next(self) except StopIteration: warning_flag = True status_message = _status_message['maxfev'] #is_optimisation_complete = False break if self.disp: print("differential_evolution step %d: f(x)= %g" % (nit, self.population_energies[0])) # should the solver terminate? convergence = self.convergence if (self.callback and self.callback(self._scale_parameters(self.population[0]), convergence=self.tol / convergence) is True): warning_flag = True status_message = ('callback function requested stop early ' 'by returning True') is_optimisation_complete = False break intol = (np.std(self.population_energies) <= self.atol + self.tol * np.abs(np.mean(self.population_energies))) if intol: is_optimisation_complete = False if warning_flag or intol: break else: status_message = _status_message['maxiter'] warning_flag = True client.set(self.island_marker, ",".join([str(items) for items in self.x])) print("MARKED IN MEMCACHE") print(self.island_marker, ",".join([str(items) for items in self.x])) if not is_optimisation_complete: #break print("Exited due to some break condition above!!", status_message) DE_result = OptimizeResult( x=self.x, fun=self.population_energies[0], nfev=self._nfev, nit=nit, message=status_message, success=(warning_flag is not True)) if self.polish: result = minimize(self.func, np.copy(DE_result.x), method='L-BFGS-B', bounds=self.limits.T, args=self.args) self._nfev += result.nfev DE_result.nfev = self._nfev if result.fun < DE_result.fun: DE_result.fun = result.fun DE_result.x = result.x DE_result.jac = result.jac # to keep internal state consistent self.population_energies[0] = result.fun self.population[0] = self._unscale_parameters(result.x) return DE_result
def solve(self): """ Runs the DifferentialEvolutionSolver. Returns ------- res : OptimizeResult The optimization result represented as a ``OptimizeResult`` object. Important attributes are: ``x`` the solution array, ``success`` a Boolean flag indicating if the optimizer exited successfully and ``message`` which describes the cause of the termination. See `OptimizeResult` for a description of other attributes. If polish was employed, then OptimizeResult also contains the ``hess_inv`` and ``jac`` attributes. """ nfev, nit, warning_flag = 0, 0, False status_message = _status_message['success'] # calculate energies to start with for index, candidate in enumerate(self.population): parameters = self._scale_parameters(candidate) self.population_energies[index] = self.func(parameters, *self.args) nfev += 1 if nfev > self.maxfun: warning_flag = True status_message = _status_message['maxfev'] break minval = np.argmin(self.population_energies) # put the lowest energy into the best solution position. lowest_energy = self.population_energies[minval] self.population_energies[minval] = self.population_energies[0] self.population_energies[0] = lowest_energy self.population[[0, minval], :] = self.population[[minval, 0], :] if warning_flag: return OptimizeResult( x=self.x, fun=self.population_energies[0], nfev=nfev, nit=nit, message=status_message, success=(warning_flag != True)) # do the optimisation. for nit in range(1, self.maxiter + 1): if self.dither is not None: self.scale = self.random_number_generator.rand( ) * (self.dither[1] - self.dither[0]) + self.dither[0] for candidate in range(np.size(self.population, 0)): if nfev > self.maxfun: warning_flag = True status_message = _status_message['maxfev'] break trial = self._mutate(candidate) self._ensure_constraint(trial) parameters = self._scale_parameters(trial) energy = self.func(parameters, *self.args) nfev += 1 if energy < self.population_energies[candidate]: self.population[candidate] = trial self.population_energies[candidate] = energy if energy < self.population_energies[0]: self.population_energies[0] = energy self.population[0] = trial # stop when the fractional s.d. of the population is less than tol # of the mean energy convergence = (np.std(self.population_energies) / np.abs(np.mean(self.population_energies) + _MACHEPS)) if self.disp: print("differential_evolution step %d: f(x)= %g" % (nit, self.population_energies[0])) if (self.callback and self.callback(self._scale_parameters(self.population[0]), convergence=self.tol / convergence) is True): warning_flag = True status_message = ('callback function requested stop early ' 'by returning True') break if convergence < self.tol or warning_flag: break else: status_message = _status_message['maxiter'] warning_flag = True DE_result = OptimizeResult( x=self.x, fun=self.population_energies[0], nfev=nfev, nit=nit, message=status_message, success=(warning_flag != True)) if self.polish: result = minimize(self.func, np.copy(DE_result.x), method='L-BFGS-B', bounds=self.limits.T, args=self.args) nfev += result.nfev DE_result.nfev = nfev if result.fun < DE_result.fun: DE_result.fun = result.fun DE_result.x = result.x DE_result.jac = result.jac # to keep internal state consistent self.population_energies[0] = result.fun self.population[0] = self._unscale_parameters(result.x) return DE_result
def fmin_bfgs_f(f_g, x0, B0=None, M=2, gtol=1e-5, Delta=10.0, maxiter=None, callback=None, norm_ord=np.Inf, **_kwargs): """test BFGS with nonmonote line search""" fk, gk = f_g(x0) if B0 is None: Bk = np.eye(len(x0)) else: Bk = B0 Hk = np.linalg.inv(Bk) maxiter = 200 * len(x0) if maxiter is None else maxiter xk = x0 norm = lambda x: np.linalg.norm(x, ord=norm_ord) theta = 0.9 C = 0.5 k = 0 old_old_fval = fk + np.linalg.norm(gk) / 2 old_fval = fk f_s = Seq(M) f_s.add(fk) flag = 0 re_search = 0 for k in range(maxiter): if norm(gk) <= gtol: break dki = -np.dot(Hk, gk) try: pk = dki f = f_g.fun myfprime = f_g.grad gfk = gk old_fval = fk ( alpha_k, fc, gc, old_fval, old_old_fval, gfkp1, ) = line_search_wolfe2(f, myfprime, xk, pk, gfk, f_s.get_max(), old_fval, old_old_fval) except Exception as e: print(e) re_search += 1 xk = xk + dki fk, gk = f_g(xk) old_fval, old_old_fval = fk, old_fval f_s.add(fk) if re_search > 2: flag = 1 break continue if alpha_k is None: print("alpha is None") xk = xk + dki fk, gk = f_g(xk) old_fval, old_old_fval = fk, old_fval f_s.add(fk) re_search += 1 if re_search > 2: flag = 1 break continue dki = alpha_k * pk # fki, gki = f_g(xk + dki) fki, gki = old_fval, gfkp1 Aredk = fk - fki Predk = -(np.dot(gk, dki) + 0.5 * np.dot(np.dot(Bk, dki), dki)) rk = Aredk / Predk xk = xk + dki fk = fki yk = gki - gk tk = C + max(0, -np.dot(yk, dki) / norm(dki)**2) / norm(gk) ystark = (1 - theta) * yk + theta * tk * norm(gk) * dki gk = gki bs = np.dot(Bk, dki) Bk = (Bk + np.outer(yk, yk) / np.dot(yk, dki) - np.outer(bs, bs) / np.dot(bs, dki)) # sk = dki # rhok = 1.0 / (np.dot(yk, sk)) # A1 = 1 - np.outer(sk, yk) * rhok # A2 = 1 - np.outer(yk, sk) * rhok # Hk = np.dot(A2, np.dot(Hk, A1)) - (rhok * np.outer(sk, sk)) # Bk = Bk + np.outer(ystark, ystark)/np.dot(ystark, dki) - \ # np.outer(bs, bs)/np.dot(bs, dki) # MBFGS # print(np.dot(Hk, Bk)) try: Hk = np.linalg.inv(Bk) except Exception: pass f_s.add(fk) if callback is not None: callback(xk) else: flag = 2 # print("fit final: ", k, p, f_g.ncall) s = OptimizeResult() s.messgae = message_dict[flag] s.fun = float(fk) s.nit = k s.nfev = f_g.ncall s.njev = f_g.ncall s.status = flag s.x = np.array(xk) s.jac = np.array(gk) s.hess = np.array(Bk) s.success = flag == 0 return s
def solve(self): """ Runs the DifferentialEvolutionSolver. Returns ------- res : OptimizeResult The optimization result represented as a ``OptimizeResult`` object. Important attributes are: ``x`` the solution array, ``success`` a Boolean flag indicating if the optimizer exited successfully and ``message`` which describes the cause of the termination. See `OptimizeResult` for a description of other attributes. If polish was employed, then OptimizeResult also contains the ``hess_inv`` and ``jac`` attributes. """ #nit = self.niter start_time = self.time warning_flag = False if time.time()-start_time > self.maxtime and self.maxtime is not None : # result = {'population':self.population, # 'population_energies':self.population_energies, # 'niter' : nit, # 'message': 'Maximum time has been exceeded.', # 'success' : False } result = OptimizeResult( population = self.population, population_energies = self.population_energies, nit = self.niter, message = 'Maximum time has been exceeded.', success = False ) return result status_message = _status_message['success'] #print(self.population_energies[0]) # do the optimisation. for nit in range(self.niter, self.maxiter + 1): population_count = np.size(self.population, 0) if self.dither is not None: self.scale = self.random_number_generator.rand( ) * (self.dither[1] - self.dither[0]) + self.dither[0] Parameters=[] Trials=[] for candidate in range(population_count): trial = self._mutate(candidate) self._ensure_constraint(trial) Trials.append(trial) Parameters.append( self._scale_parameters(trial) ) pool=multiprocessing.Pool(self.ncore) Energies = pool.map(self.func, Parameters) pool.close() pool.join() iNan = [] for i in range(population_count): if self.population_energies[i] != self.population_energies[i]: iNan.append(i) self.population_energies = np.delete(self.population_energies,iNan) self.population = np.delete(self.population,iNan,0) population_count = np.size(self.population, 0) for candidate in range(population_count): if Energies[candidate] < self.population_energies[candidate]: self.population[candidate] = Trials[candidate] self.population_energies[candidate] = Energies[candidate] if Energies[candidate] < self.population_energies[0]: self.population_energies[0] = Energies[candidate] self.population[0] = Trials[candidate] # stop when the fractional s.d. of the population is less than tol # of the mean energy convergence = (np.std(self.population_energies) / np.abs(np.mean(self.population_energies) + _MACHEPS)) if self.disp: print("differential_evolution step %d: f(x)= %g" % (nit, self.population_energies[0])) print("total population at step %d is %d" %(nit, population_count ) ) if (self.callback and self.callback(self._scale_parameters(self.population[0]), convergence=self.tol / convergence) is True): warning_flag = True status_message = ('callback function requested stop early ' 'by returning True') break if convergence < self.tol or warning_flag: break if time.time()-start_time > self.maxtime and self.maxtime is not None : result = OptimizeResult( population = self.population, population_energies = self.population_energies, nit = self.niter, message = 'Maximum time has been exceeded.', success = False ) return result else: status_message = _status_message['maxiter'] warning_flag = True DE_result = OptimizeResult( x=self.x, fun=self.population_energies[0], nit=nit, message=status_message, success=(warning_flag != True)) if self.polish: result = minimize(self.func, np.copy(DE_result.x), method='L-BFGS-B', bounds=self.limits.T) if result.fun < DE_result.fun: DE_result.fun = result.fun DE_result.x = result.x DE_result.jac = result.jac # to keep internal state consistent self.population_energies[0] = result.fun self.population[0] = self._unscale_parameters(result.x) return DE_result
def solve(self): """ Runs the DifferentialEvolutionSolver. Returns ------- res : OptimizeResult The optimization result represented as a ``OptimizeResult`` object. Important attributes are: ``x`` the solution array, ``success`` a Boolean flag indicating if the optimizer exited successfully and ``message`` which describes the cause of the termination. See `OptimizeResult` for a description of other attributes. If `polish` was employed, and a lower minimum was obtained by the polishing, then OptimizeResult also contains the ``jac`` attribute. """ nfev, nit, warning_flag = 0, 0, False status_message = _status_message['success'] # calculate energies to start with parameters = np.zeros_like(self.population, order='F') for index, candidate in enumerate(self.population): parameters[index, :] = self._scale_parameters(candidate) self.population_energies[:] = self.evaluate_func(parameters) nfev += self.num_population_members # put the lowest energy into the best solution position. minval = np.argmin(self.population_energies) self._swap_best(minval) if warning_flag: return OptimizeResult(x=self.x, fun=self.population_energies[0], nfev=nfev, nit=nit, message=status_message, success=(warning_flag is not True)) # do the optimisation. trials = np.zeros_like(self.population, order='F') for nit in range(1, self.maxiter + 1): if self.dither is not None: self.scale = self.random_number_generator.rand() * ( self.dither[1] - self.dither[0]) + self.dither[0] # Unlike the standard DE, all the trials are created first and later # evaluated simultaneously. for index in range(self.num_population_members): # create a trial solution trials[index][:] = self._mutate(index) # ensuring that it's in the range [0, 1) self._ensure_constraint(trials[index]) # scale from [0, 1) to the actual parameter value parameters[index][:] = self._scale_parameters(trials[index]) # determine the energy of the objective function energies = self.evaluate_func(parameters) nfev += self.num_population_members # if the energy of the trial candidate is lower than the # original population member then replace it for index in range(self.num_population_members): if energies[index] < self.population_energies[index]: self.population[index] = trials[index] self.population_energies[index] = energies[index] # if the trial candidate also has a lower energy than the # best solution then replace that as well minval = np.argmin(self.population_energies) self._swap_best(minval) # stop when the fractional s.d. of the population is less than tol # of the mean energy convergence = ( np.std(self.population_energies) / np.abs(np.mean(self.population_energies) + _MACHEPS)) if self.disp: print("differential_evolution step %d: f(x)= %g" % (nit, self.population_energies[0])) if self.callbacks: for callback in self.callbacks: callback(step=nit, parameter=self.x, cost=self.population_energies[0]) if (self.earlystop and self.earlystop( self.x, convergence=self.tol / convergence) is True): warning_flag = True status_message = ('earlystop function requested stop early ' 'by returning True') break if convergence < self.tol or warning_flag: break else: status_message = _status_message['maxiter'] warning_flag = True DE_result = OptimizeResult(x=self.x, fun=self.population_energies[0], nfev=nfev, nit=nit, message=status_message, success=(warning_flag is not True)) if self.polish: result = minimize(self.func, np.copy(DE_result.x), method='L-BFGS-B', bounds=self.limits.T, args=self.args) nfev += result.nfev DE_result.nfev = nfev if result.fun < DE_result.fun: DE_result.fun = result.fun DE_result.x = result.x DE_result.jac = result.jac # to keep internal state consistent self.population_energies[0] = result.fun self.population[0] = self._unscale_parameters(result.x) return DE_result
def solve(self): nfev, nit, warning_flag = 0, 0, False status_message = _status_message['success'] # calculate energies to start with for index, candidate in enumerate(self.population): parameters = self._scale_parameters(candidate) self.population_energies[index] = self.func(parameters, *self.args) nfev += 1 if nfev > self.maxfun: warning_flag = True status_message = _status_message['maxfev'] break minval = np.argmin(self.population_energies) # put the lowest energy into the best solution position. lowest_energy = self.population_energies[minval] self.population_energies[minval] = self.population_energies[0] self.population_energies[0] = lowest_energy self.population[[0, minval], :] = self.population[[minval, 0], :] if warning_flag: return OptimizeResult( x=self.x, fun=self.population_energies[0], nfev=nfev, nit=nit, message=status_message, success=(warning_flag is not True)) # do the optimisation. start_time = time.time() nit = 0 while nit < self.maxiter + 1: nit += 1 if start_time + self.max_execution_time < time.time(): warning_flag = True status_message = 'Max execution time reached' break if self.dither is not None: self.scale = self.random_number_generator.rand( ) * (self.dither[1] - self.dither[0]) + self.dither[0] for candidate in range(np.size(self.population, 0)): if nfev > self.maxfun: warning_flag = True status_message = _status_message['maxfev'] break trial = self._mutate(candidate) self._ensure_constraint(trial) parameters = self._scale_parameters(trial) energy = self.func(parameters, *self.args) nfev += 1 if energy < self.population_energies[candidate]: self.population[candidate] = trial self.population_energies[candidate] = energy if energy < self.population_energies[0]: self.population_energies[0] = energy self.population[0] = trial # stop when the fractional s.d. of the population is less than tol # of the mean energy convergence = (np.std(self.population_energies) / np.abs(np.mean(self.population_energies) + _MACHEPS)) if self.disp: print("differential_evolution step %d: f(x)= %g" % (nit, self.population_energies[0])) if (self.callback and self.callback(self._scale_parameters(self.population[0]), convergence=self.tol / convergence) is True): warning_flag = True status_message = ('callback function requested stop early ' 'by returning True') break if convergence < self.tol or warning_flag: break else: status_message = _status_message['maxiter'] warning_flag = True DE_result = OptimizeResult( x=self.x, fun=self.population_energies[0], nfev=nfev, nit=nit, message=status_message, success=(warning_flag is not True)) if self.polish: result = minimize(self.func, np.copy(DE_result.x), method='L-BFGS-B', bounds=self.limits.T, args=self.args) nfev += result.nfev DE_result.nfev = nfev if result.fun < DE_result.fun: DE_result.fun = result.fun DE_result.x = result.x DE_result.jac = result.jac # to keep internal state consistent self.population_energies[0] = result.fun self.population[0] = self._unscale_parameters(result.x) return DE_result
def optimize_minimize_mhmcmc_cluster(objective, bounds, args=(), x0=None, T=1, N=3, burnin=100000, maxiter=1000000, target_ar=0.4, ar_tolerance=0.05, cluster_eps=DEFAULT_CLUSTER_EPS, rnd_seed=None, collect_samples=None, logger=None): """ Minimize objective function and return up to N local minima solutions. :param objective: Objective function to minimize. Takes unpacked args as function call arguments and returns a float. :type objective: Callable(\*args) -> float :param bounds: Bounds of the parameter space. :type bounds: scipy.optimize.Bounds :param args: Any additional fixed parameters needed to completely specify the objective function. :type args: tuple or list :param x0: Initial guess. If None, will be selected randomly and uniformly within the parameter bounds. :type x0: numpy.array with same shape as elements of bounds :param T: The "temperature" parameter for the accept or reject criterion. To sample the domain well, should be in the order of the typical difference in local minima objective valuations. :type T: float :param N: Maximum number of minima to return :type N: int :param burnin: Number of random steps to discard before starting to accumulate statistics. :type burnin: int :param maxiter: Maximum number of steps to take (including burnin). :type maxiter: int :param target_ar: Target acceptance rate of point samples generated by stepping. :type target_ar: float between 0 and 1 :param ar_tolerance: Tolerance on the acceptance rate before actively adapting the step size. :type ar_tolerance: float :param cluster_eps: Point proximity tolerance for DBSCAN clustering, in normalized bounds coordinates. :type cluster_eps: float :param rnd_seed: Random seed to force deterministic behaviour :type rnd_seed: int :param collect_samples: If not None and integral type, collect collect_samples at regular intervals and return as part of solution. :type collect_samples: int or NoneType :param logger: Logger instance for outputting log messages. :return: OptimizeResult containing solution(s) and solver data. :rtype: scipy.optimize.OptimizeResult with additional attributes """ @call_counter def obj_counted(*args): return objective(*args) # end func assert maxiter >= 2 * burnin, "maxiter {} should be at least twice burnin steps {}".format( maxiter, burnin) main_iter = maxiter - burnin if collect_samples is not None: assert isinstance(collect_samples, int), "collect_samples expected to be integral type" assert collect_samples > 0, "collect_samples expected to be positive" # end if beta = 1.0 / T if rnd_seed is None: rnd_seed = int(time.time() * 1000) % (1 << 31) # end if np.random.seed(rnd_seed) if logger: logger.info('Using random seed {}'.format(rnd_seed)) # end if x0 is None: x0 = np.random.uniform(bounds.lb, bounds.ub) # end if assert np.all((x0 >= bounds.lb) & (x0 <= bounds.ub)) x = x0.copy() funval = obj_counted(x, *args) # Set up stepper with adaptive acceptance rate stepper = BoundedRandNStepper(bounds) stepper = AdaptiveStepsize(stepper, accept_rate=target_ar, ar_tolerance=ar_tolerance, interval=50) # ------------------------------- # DO BURN-IN rejected_randomly = 0 accepted_burnin = 0 tracked_range = tqdm(range(burnin), total=burnin, desc='BURN-IN') if logger: stepper.logger = lambda msg: tracked_range.write(logger.name + ':' + msg) else: stepper.logger = tracked_range.write # end if for _ in tracked_range: x_new = stepper(x) funval_new = obj_counted(x_new, *args) log_alpha = -(funval_new - funval) * beta if log_alpha > 0 or np.log(np.random.rand()) <= log_alpha: x = x_new funval = funval_new stepper.notify_accept() accepted_burnin += 1 elif log_alpha <= 0: rejected_randomly += 1 # end if # end for ar = float(accepted_burnin) / burnin if logger: logger.info("Burn-in acceptance rate: {}".format(ar)) # end if # ------------------------------- # DO MAIN LOOP if collect_samples is not None: nsamples = min(collect_samples, main_iter) sample_cadence = main_iter / nsamples samples = np.zeros((nsamples, len(x))) samples_fval = np.zeros(nsamples) # end if accepted = 0 rejected_randomly = 0 minima_sorted = SortedList( key=lambda rec: rec[1]) # Sort by objective function value hist = HistogramIncremental(bounds, nbins=100) # Cached a lot of potential minimum values, as these need to be clustered before return N results N_cached = int(np.ceil(N * main_iter / 500)) next_sample = 0.0 sample_count = 0 tracked_range = tqdm(range(main_iter), total=main_iter, desc='MAIN') if logger: stepper.logger = lambda msg: tracked_range.write(logger.name + ':' + msg) else: stepper.logger = tracked_range.write # end if for i in tracked_range: if collect_samples and i >= next_sample: assert sample_count < collect_samples samples[sample_count] = x samples_fval[sample_count] = funval sample_count += 1 next_sample += sample_cadence # end if x_new = stepper(x) funval_new = obj_counted(x_new, *args) log_alpha = -(funval_new - funval) * beta if log_alpha > 0 or np.log(np.random.rand()) <= log_alpha: x = x_new funval = funval_new minima_sorted.add((x, funval)) if len(minima_sorted) > N_cached: minima_sorted.pop() # end if stepper.notify_accept() hist += x accepted += 1 elif log_alpha <= 0: rejected_randomly += 1 # end if # end for stepper.logger = None ar = float(accepted) / main_iter if logger: logger.info("Acceptance rate: {}".format(ar)) logger.info("Best minima (before clustering):\n{}".format( np.array([_mx[0] for _mx in minima_sorted[:10]]))) # end if # ------------------------------- # Cluster minima and associate each cluster with a local minimum. # Using a normalized coordinate space for cluster detection. x_range = bounds.ub - bounds.lb pts = np.array([x[0] for x in minima_sorted]) fvals = np.array([x[1] for x in minima_sorted]) pts_norm = (pts - bounds.lb) / x_range _, labels = dbscan(pts_norm, eps=cluster_eps, min_samples=21, n_jobs=-1) # Compute mean of each cluster and evaluate objective function at cluster mean locations. minima_candidates = [] for grp in range(max(labels) + 1): mask = (labels == grp) mean_loc = np.mean(pts[mask, :], axis=0) # Evaluate objective function precisely at the mean location of each cluster fval = obj_counted(mean_loc, *args) minima_candidates.append((mean_loc, grp, fval)) # end for # Rank minima locations by objective function. minima_candidates.sort(key=lambda c: c[2]) # Pick up to N solutions solutions = minima_candidates[:N] # Put results into OptimizeResult container. # Add histograms to output result (in form of scipy.stats.rv_histogram) solution = OptimizeResult() solution.x = np.array([s[0] for s in solutions]) solution.clusters = [pts[(labels == s[1])] for s in solutions] solution.cluster_funvals = [fvals[(labels == s[1])] for s in solutions] solution.bins = hist.bins solution.distribution = hist.histograms solution.acceptance_rate = ar solution.success = True solution.status = 0 if len(solutions) > 0: solution.message = 'SUCCESS: Found {} local minima'.format( len(solutions)) else: solution.message = 'WARNING: Found no clusters within tolerance {}'.format( cluster_eps) # end if solution.fun = np.array([s[2] for s in solutions]) solution.jac = None solution.nfev = obj_counted.counter solution.njev = 0 solution.nit = main_iter solution.maxcv = None solution.samples = samples if collect_samples else None solution.sample_funvals = samples_fval if collect_samples else None solution.bounds = bounds solution.version = 's0.3' # Solution version for future traceability solution.rnd_seed = rnd_seed return solution
def solve(self): """ Runs the DifferentialEvolutionSolver. Returns ------- res : OptimizeResult The optimization result represented as a ``OptimizeResult`` object. Important attributes are: ``x`` the solution array, ``success`` a Boolean flag indicating if the optimizer exited successfully and ``message`` which describes the cause of the termination. See `OptimizeResult` for a description of other attributes. If `polish` was employed, and a lower minimum was obtained by the polishing, then OptimizeResult also contains the ``jac`` attribute. """ nit, warning_flag = 0, False # dictionary that holds standard status messages of optimizers status_message = _status_message['success'] # The population may have just been initialized (all entries are # np.inf). If it has you have to calculate the initial energies. # Although this is also done in the evolve generator it's possible # that someone can set maxiter=0, at which point we still want the # initial energies to be calculated (the following loop isn't run). #np.all checks that there are no 0's in the array if self.maxiter == 0: if np.all(np.isinf(self.population_energies)): if self.disp: print("Calculating initial energies when maxiter = 0") self._calculate_population_energies() # for i in range(self.num_population_members): # print(self.population[i,:]) # do the optimisation. for nit in xrange(1, self.maxiter + 1): if self.disp: print("iter: ", nit) # evolve the population by a generation try: next(self) except StopIteration: warning_flag = True status_message = _status_message['maxfev'] break print("differential_evolution step %d: f(x)= %g" % (nit, self.population_energies[0])) #save populations at each iter and rank to analyze after # np.save("before_rank"+str(self.rank)+"iter"+str(nit), self.population) #migrate self.migration() # np.save("after_rank"+str(self.rank)+"iter"+str(nit), self.population) # should the solver terminate? # print("Checking if should converge") # convergence = self.convergence # # if (self.callback and # self.callback(self._scale_parameters(self.population[0]), # convergence=self.tol / convergence) is True): # # warning_flag = True # status_message = ('callback function requested stop early ' # 'by returning True') # break # print("checking if tolerance level reached") ## intol = (np.std(self.population_energies) <= ## self.atol + ## self.tol * np.abs(np.mean(self.population_energies))) # # intol = self.population_energies[0] <= self.mse_thresh # if warning_flag or intol: # print("stopping iterations") # break print("Starting next iter") else: status_message = _status_message['maxiter'] warning_flag = True DE_result = OptimizeResult(x=self.x, fun=self.population_energies[0], nfev=self._nfev, nit=nit, message=status_message, success=(warning_flag is not True)) print("done iters") if self.polish: print("performing final polishing") result = minimize(self.func, np.copy(DE_result.x), method='L-BFGS-B', bounds=self.limits.T, args=self.args) self._nfev += result.nfev DE_result.nfev = self._nfev if result.fun < DE_result.fun: DE_result.fun = result.fun DE_result.x = result.x DE_result.jac = result.jac # to keep internal state consistent self.population_energies[0] = result.fun self.population[0] = self._unscale_parameters(result.x) return DE_result
def solve(self): nfev, nit, warning_flag = 0, 0, False status_message = _status_message['success'] # calculate energies to start with for index, candidate in enumerate(self.population): parameters = self._scale_parameters(candidate) self.population_energies[index] = self.func(parameters, *self.args) nfev += 1 if nfev > self.maxfun: warning_flag = True status_message = _status_message['maxfev'] break minval = np.argmin(self.population_energies) # put the lowest energy into the best solution position. lowest_energy = self.population_energies[minval] self.population_energies[minval] = self.population_energies[0] self.population_energies[0] = lowest_energy self.population[[0, minval], :] = self.population[[minval, 0], :] if warning_flag: return OptimizeResult(x=self.x, fun=self.population_energies[0], nfev=nfev, nit=nit, message=status_message, success=(warning_flag is not True)) # do the optimisation. start_time = time.time() nit = 0 while nit < self.maxiter + 1: nit += 1 if start_time + self.max_execution_time < time.time(): warning_flag = True status_message = 'Max execution time reached' break if self.dither is not None: self.scale = self.random_number_generator.rand() * ( self.dither[1] - self.dither[0]) + self.dither[0] for candidate in range(np.size(self.population, 0)): if nfev > self.maxfun: warning_flag = True status_message = _status_message['maxfev'] break trial = self._mutate(candidate) self._ensure_constraint(trial) parameters = self._scale_parameters(trial) energy = self.func(parameters, *self.args) nfev += 1 if energy < self.population_energies[candidate]: self.population[candidate] = trial self.population_energies[candidate] = energy if energy < self.population_energies[0]: self.population_energies[0] = energy self.population[0] = trial # stop when the fractional s.d. of the population is less than tol # of the mean energy convergence = ( np.std(self.population_energies) / np.abs(np.mean(self.population_energies) + _MACHEPS)) if self.disp: print("differential_evolution step %d: f(x)= %g" % (nit, self.population_energies[0])) if (self.callback and self.callback(self._scale_parameters(self.population[0]), convergence=self.tol / convergence) is True): warning_flag = True status_message = ('callback function requested stop early ' 'by returning True') break if convergence < self.tol or warning_flag: break else: status_message = _status_message['maxiter'] warning_flag = True DE_result = OptimizeResult(x=self.x, fun=self.population_energies[0], nfev=nfev, nit=nit, message=status_message, success=(warning_flag is not True)) if self.polish: result = minimize(self.func, np.copy(DE_result.x), method='L-BFGS-B', bounds=self.limits.T, args=self.args) nfev += result.nfev DE_result.nfev = nfev if result.fun < DE_result.fun: DE_result.fun = result.fun DE_result.x = result.x DE_result.jac = result.jac # to keep internal state consistent self.population_energies[0] = result.fun self.population[0] = self._unscale_parameters(result.x) return DE_result