def Uniformcheck(ds): ds = sorted(ds, key=lambda x: x.f[0], reverse=False) d = [] for i, dpi in enumerate(ds): if i != 0: d.append(vecnorm(dpi.f - ds[i - 1].f)) # d = [] # for i, dpi in enumerate(ds): # di = 10 # for j, dpj in enumerate(ds): # if i != j: # dt = vecnorm(dpi.f - dpj.f) # if dt < di: # di = dt # d.append(di) sum = 0 for di in d: sum += di td = sum / len(ds) sum2 = 0 for di in d: sum2 += (di / td)**2 return np.sqrt(sum2 / (len(ds) - 1))
def loopThing(inputParams): for key in inputParams: exec(key + " = inputParams['" + key + "']") pk = -numpy.dot(Hk,gfk) alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ linesearch.line_search(f,myfprime,xk,pk,gfk, old_fval,old_old_fval) if alpha_k is None: # line search failed try different one. alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ line_search(f,myfprime,xk,pk,gfk, old_fval,old_old_fval) if alpha_k is None: # This line search also failed to find a better solution. warnflag = 2 outputParams = getOutputParams(inputParams, locals().items()) return outputParams xkp1 = xk + alpha_k * pk if retall: allvecs.append(xkp1) sk = xkp1 - xk xk = xkp1 if gfkp1 is None: gfkp1 = myfprime(xkp1) yk = gfkp1 - gfk gfk = gfkp1 if callback is not None: callback(xk) k += 1 gnorm = vecnorm(gfk,ord=norm) if (gnorm <= gtol): outputParams = getOutputParams(inputParams, locals().items()) return outputParams try: # this was handled in numeric, let it remaines for more safety rhok = 1.0 / (numpy.dot(yk,sk)) except ZeroDivisionError: rhok = 1000.0 print "Divide-by-zero encountered: rhok assumed large" if numpy.isinf(rhok): # this is patch for numpy rhok = 1000.0 print "Divide-by-zero encountered: rhok assumed large" A1 = I - sk[:,numpy.newaxis] * yk[numpy.newaxis,:] * rhok A2 = I - yk[:,numpy.newaxis] * sk[numpy.newaxis,:] * rhok Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \ * sk[numpy.newaxis,:] outputParams = getOutputParams(inputParams, locals().items()) return outputParams
def loopThing_BFGS(inputParams): for key in inputParams: exec(key + " = inputParams['" + key + "']") pk = -numpy.dot(Hk, gfk) alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ linesearch.line_search(f,myfprime,xk,pk,gfk, old_fval,old_old_fval) if alpha_k is None: # line search failed try different one. alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ line_search(f,myfprime,xk,pk,gfk, old_fval,old_old_fval) if alpha_k is None: # This line search also failed to find a better solution. warnflag = 2 outputParams = getOutputParams(inputParams, list(locals().items())) return outputParams xkp1 = xk + alpha_k * pk if retall: allvecs.append(xkp1) sk = xkp1 - xk xk = xkp1 if gfkp1 is None: gfkp1 = myfprime(xkp1) yk = gfkp1 - gfk gfk = gfkp1 if callback is not None: callback(xk) k += 1 gnorm = vecnorm(gfk, ord=norm) if (gnorm <= gtol): outputParams = getOutputParams(inputParams, list(locals().items())) return outputParams try: # this was handled in numeric, let it remaines for more safety rhok = 1.0 / (numpy.dot(yk, sk)) except ZeroDivisionError: rhok = 1000.0 print("Divide-by-zero encountered: rhok assumed large") if numpy.isinf(rhok): # this is patch for numpy rhok = 1000.0 print("Divide-by-zero encountered: rhok assumed large") A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \ * sk[numpy.newaxis,:] outputParams = getOutputParams(inputParams, list(locals().items())) return outputParams
def fminLooped(f, x0, fprime=None, args=(), gtol=1e-5, norm=Inf, epsilon=numpy.sqrt(numpy.finfo(float).eps), maxiter=None, full_output=0, disp=1, retall=0, callback=None): testVar = 0 x0 = asarray(x0).squeeze() if x0.ndim == 0: x0.shape = (1, ) if maxiter is None: maxiter = len(x0) * 200 func_calls, f = wrap_function(f, args) if fprime is None: grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon)) else: grad_calls, myfprime = wrap_function(fprime, args) gfk = myfprime(x0) k = 0 N = len(x0) I = numpy.eye(N, dtype=int) Hk = I old_fval = f(x0) old_old_fval = old_fval + 5000 xk = x0 if retall: allvecs = [x0] sk = [2 * gtol] warnflag = 0 gnorm = vecnorm(gfk, ord=norm) newInputParams = locals() pickleBles, NonPickles = ParamsManager.filterUnpickles(newInputParams) # import dill # pickle.dump(pickleBles, open('inputParams.dat', 'w')) # pickle.dump(loopThing, open('loopFunc.dat', 'w')) for loopI in range(300): newInputParams = loopThing(newInputParams)
def Extension(ds, ds_ini): list = [] for dpi in ds_ini: num = 10 for dp in ds: d = vecnorm(dp.f - dpi) if d < num: num = d list.append(num) dd = 0 for l in list: dd += l**2 EX = np.sqrt(dd) / len(ds_ini)
def Eveness(ds): ds = sorted(ds, key=lambda x: x.f[0], reverse=False) d = [] for i, dpi in enumerate(ds): if i != 0: d.append(vecnorm(dpi.f - ds[i - 1].f)) sum = 0 for di in d: sum += di mu = sum/len(d) sum2 = 0 for i, di in enumerate(d): if i != 0: sum2 += (di - mu) ** 2 si = np.sqrt(sum2 / (len(d) - 1)) # return si / mu ol = mu - si i = 0 while i < len(d)-1: if d[i] < ol: d[i+1] += d[i] del d[i] else: i += 1 mu = sum / len(d) sum2 = 0 for i, di in enumerate(d): if i != 0: sum2 += (di - mu) ** 2 sum2 += (d[i-1] - mu) ** 2 si = np.sqrt(sum2 / (2*len(d) - 1)) return si / mu
def fminLooped(f, x0, fprime=None, args=(), gtol=1e-5, norm=Inf, epsilon= numpy.sqrt(numpy.finfo(float).eps), maxiter=None, full_output=0, disp=1, retall=0, callback=None): testVar = 0 x0 = asarray(x0).squeeze() if x0.ndim == 0: x0.shape = (1,) if maxiter is None: maxiter = len(x0)*200 func_calls, f = wrap_function(f, args) if fprime is None: grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon)) else: grad_calls, myfprime = wrap_function(fprime, args) gfk = myfprime(x0) k = 0 N = len(x0) I = numpy.eye(N,dtype=int) Hk = I old_fval = f(x0) old_old_fval = old_fval + 5000 xk = x0 if retall: allvecs = [x0] sk = [2*gtol] warnflag = 0 gnorm = vecnorm(gfk,ord=norm) newInputParams = locals() pickleBles, NonPickles = ParamsManager.filterUnpickles(newInputParams) import dill pickle.dump(pickleBles, open('inputParams.dat', 'w')) pickle.dump(loopThing, open('loopFunc.dat', 'w')) for loopI in range(10): newInputParams = loopThing(newInputParams)
def minimize_bfgs(func: Callable, init_param_vec: Sequence, grad: Callable = None, grad_tol: float = 1e-5, return_all: bool = True, last_record: 'OptimizeRecord' = None, notes: dict = None): if notes is None: notes = {} notes["grad_tol"] = grad_tol notes["method"] = "BFGS" f = func fprime = grad epsilon = np.finfo(float).eps**0.5 gtol = grad_tol norm = np.Inf x0 = init_param_vec if x0.ndim == 0: x0.shape = (1, ) maxiter = len(x0) * 200 func_calls, f = wrap_function(f, ()) if fprime is None: grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon)) notes["grad_approx"] = True else: grad_calls, myfprime = wrap_function(fprime, ()) notes["grad_approx"] = False k = 0 N = len(x0) I = np.eye(N, dtype=int) if last_record: old_fval = last_record.final_func gfk = last_record.final_grad old_old_fval = last_record.last_vars["old_old_fval"] Hk = last_record.last_vars["Hk"] else: # Sets the initial step guess to dx ~ 1 old_fval = f(x0) gfk = myfprime(x0) old_old_fval = old_fval + np.linalg.norm(gfk) / 2 Hk = I all_param_vec = [x0] all_func = [old_fval] all_grad = [gfk] xk = x0 warnflag = 0 gnorm = vecnorm(gfk, ord=norm) while (gnorm > gtol) and (k < maxiter): pk = -np.dot(Hk, gfk) try: alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ _line_search_wolfe12(f, myfprime, xk, pk, gfk, old_fval, old_old_fval, amin=1e-100, amax=1e100) except _LineSearchError: # Line search failed to find a better solution. warnflag = 2 break xkp1 = xk + alpha_k * pk sk = xkp1 - xk xk = xkp1 if gfkp1 is None: gfkp1 = myfprime(xkp1) yk = gfkp1 - gfk gfk = gfkp1 k += 1 if return_all: all_param_vec.append(xk) all_func.append(old_fval) all_grad.append(gfk) gnorm = vecnorm(gfk, ord=norm) if (gnorm <= gtol): break if not np.isfinite(old_fval): # We correctly found +-Inf as optimal value, or something went # wrong. warnflag = 2 break try: # this was handled in numeric, let it remaines for more safety rhok = 1.0 / (np.dot(yk, sk)) except ZeroDivisionError: rhok = 1000.0 print("Divide-by-zero encountered: rhok assumed large") if np.isinf(rhok): # this is patch for numpy rhok = 1000.0 print("Divide-by-zero encountered: rhok assumed large") A1 = I - sk[:, np.newaxis] * yk[np.newaxis, :] * rhok A2 = I - yk[:, np.newaxis] * sk[np.newaxis, :] * rhok Hk = np.dot(A1, np.dot( Hk, A2)) + (rhok * sk[:, np.newaxis] * sk[np.newaxis, :]) fval = old_fval if np.isnan(fval): # This can happen if the first call to f returned NaN; # the loop is then never entered. warnflag = 2 if warnflag == 2: msg = _status_message['pr_loss'] elif k >= maxiter: warnflag = 1 msg = _status_message['maxiter'] else: msg = _status_message['success'] history = {"func": all_func, "grad": all_grad, "param_vec": all_param_vec} final_status = { "msg": msg, "warnflag": warnflag, "num_func_call": func_calls[0], "num_grad_call": grad_calls[0], "num_iter": k } last_vars = {"Hk": Hk, "old_old_fval": old_old_fval} record = OptimizeRecord(history, final_status, last_vars, notes) return record
def Customfmin_bfgs(f, x0, fprime=None, args=(), gtol=1e-5, norm=Inf, epsilon= numpy.sqrt(numpy.finfo(float).eps), maxiter=None, full_output=0, disp=1, retall=0, callback=None): testVar = 0 x0 = asarray(x0).squeeze() if x0.ndim == 0: x0.shape = (1,) if maxiter is None: maxiter = len(x0)*200 func_calls, f = wrap_function(f, args) if fprime is None: grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon)) else: grad_calls, myfprime = wrap_function(fprime, args) gfk = myfprime(x0) k = 0 N = len(x0) I = numpy.eye(N,dtype=int) Hk = I old_fval = f(x0) old_old_fval = old_fval + 5000 xk = x0 if retall: allvecs = [x0] sk = [2*gtol] warnflag = 0 gnorm = vecnorm(gfk,ord=norm) while (gnorm > gtol) and (k < maxiter): pk = -numpy.dot(Hk,gfk) alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ linesearch.line_search(f,myfprime,xk,pk,gfk, old_fval,old_old_fval) if alpha_k is None: # line search failed try different one. alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ line_search(f,myfprime,xk,pk,gfk, old_fval,old_old_fval) if alpha_k is None: # This line search also failed to find a better solution. warnflag = 2 break xkp1 = xk + alpha_k * pk if retall: allvecs.append(xkp1) sk = xkp1 - xk xk = xkp1 if gfkp1 is None: gfkp1 = myfprime(xkp1) yk = gfkp1 - gfk gfk = gfkp1 if callback is not None: callback(xk) k += 1 gnorm = vecnorm(gfk,ord=norm) if (gnorm <= gtol): break try: # this was handled in numeric, let it remaines for more safety rhok = 1.0 / (numpy.dot(yk,sk)) except ZeroDivisionError: rhok = 1000.0 print "Divide-by-zero encountered: rhok assumed large" if numpy.isinf(rhok): # this is patch for numpy rhok = 1000.0 print "Divide-by-zero encountered: rhok assumed large" A1 = I - sk[:,numpy.newaxis] * yk[numpy.newaxis,:] * rhok A2 = I - yk[:,numpy.newaxis] * sk[numpy.newaxis,:] * rhok Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \ * sk[numpy.newaxis,:] if disp or full_output: fval = old_fval if warnflag == 2: if disp: print "Warning: Desired error not necessarily achieved" \ "due to precision loss" print " Current function value: %f" % fval print " Iterations: %d" % k print " Function evaluations: %d" % func_calls[0] print " Gradient evaluations: %d" % grad_calls[0] elif k >= maxiter: warnflag = 1 if disp: print "Warning: Maximum number of iterations has been exceeded" print " Current function value: %f" % fval print " Iterations: %d" % k print " Function evaluations: %d" % func_calls[0] print " Gradient evaluations: %d" % grad_calls[0] else: if disp: print "Optimization terminated successfully." print " Current function value: %f" % fval print " Iterations: %d" % k print " Function evaluations: %d" % func_calls[0] print " Gradient evaluations: %d" % grad_calls[0] if full_output: retlist = xk, fval, gfk, Hk, func_calls[0], grad_calls[0], warnflag if retall: retlist += (allvecs,) else: retlist = xk if retall: retlist = (xk, allvecs) return retlist
def _minimize(fun, x0, args=(), jac=None, callback=None, gtol=1e-5, fxtol=1e-09, xtol=1e-09, norm=Inf, eps=_epsilon, maxiter=None, disp=False, return_all=False, **unknown_options): _check_unknown_options(unknown_options) f = fun fprime = jac epsilon = eps retall = return_all x0 = asarray(x0).flatten() if x0.ndim == 0: x0.shape = (1,) if maxiter is None: maxiter = len(x0) * 200 func_calls, f = wrap_function(f, args) grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon)) gfk = myfprime(x0) k = 0 N = len(x0) I = numpy.eye(N, dtype=int) Hk = I old_fval = f(x0) old_old_fval = None xk = x0 if retall: allvecs = [x0] sk = [2 * gtol] warnflag = 0 gnorm = vecnorm(gfk, ord=norm) xnorm = np.Inf fx = np.Inf print_lst = [] while (gnorm > gtol) and (xnorm > xtol) and (fx > fxtol) and (k < maxiter): pk = -numpy.dot(Hk, gfk) try: alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ _line_search_wolfe12(f, myfprime, xk, pk, gfk, old_fval, old_old_fval) except _LineSearchError: # search failed to find a better solution. print_lst.append("Przeszukiwanie liniowe zawiodlo lub nie moze osiagnac lepszego rozwiazania") warnflag = 2 break xkp1 = xk + alpha_k * pk fx = np.absolute(old_old_fval - old_fval) xnorm = vecnorm(xkp1 - xk) if retall: allvecs.append(xkp1) sk = xkp1 - xk xk = xkp1 if gfkp1 is None: gfkp1 = myfprime(xkp1) yk = gfkp1 - gfk gfk = gfkp1 if callback is not None: callback(xk) k += 1 if disp: print_ = ('Iter: ' + str(k) + '\n') print_ += ('x: ' + str(xk) + '\n') print_ += ('f(x): ' + str(f(xk)) + '\n') #zmiana na fx print_ +=('gtol: ' + str(gnorm) + '\n') print_ +=('xtol: ' + str(xnorm) + '\n') print_ +=('fxtol: ' + str(fx) + '\n') print_lst.append(print_) gnorm = vecnorm(gfk, ord=norm) if (gnorm <= gtol): break if not numpy.isfinite(old_fval): # We correctly found +-Inf as optimal value, or something went # wrong. print_lst.append("Zlaneziono +-Inf za optymalna wartosc... lub cos poszlo zle.") warnflag = 2 break try: # this was handled in numeric, let it remaines for more safety rhok = 1.0 / (numpy.dot(yk, sk)) except ZeroDivisionError: rhok = 1000.0 if disp: print_lst.append("Dzielenie przez zero!!") if isinf(rhok): # this is patch for numpy rhok = 1000.0 if disp: print_lst.appedn("Dzielenie przez zero!!") A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok Hk = numpy.dot(A1, numpy.dot(Hk, A2)) + (rhok * sk[:, numpy.newaxis] * sk[numpy.newaxis, :]) fval = old_fval if np.isnan(fval): # This can happen if the first call to f returned NaN; # the loop is then never entered. print_lst.append("Osiagnieto Nan w pierwszym wywolaniem algorytmu.") warnflag = 2 if warnflag == 2: msg = _status_message['pr_loss'] if disp: print_ = ("Ostrzezenie: " + msg) print_ += (" Wartosc funkcji celu: %f" % fval) print_ += (" Iteracje: %d" % k) print_ += (" Wywolania funkcji: %d" % func_calls[0]) print_ += (" Wywolania gradientu: %d" % grad_calls[0]) elif k >= maxiter: warnflag = 1 msg = _status_message['maxiter'] if disp: print_ = ("Ostrzerzenie: " + msg) print_ += (" Wartosc funkcji celu: %f" % fval) print_ += (" Iteracje: %d" % k) print_ += (" Wywolania funkcji: %d" % func_calls[0]) print_ += (" Wywolania gradientu: %d" % grad_calls[0]) print_lst.append(print_) else: msg = _status_message['success'] if disp: print_ = (msg + '\n') print_ += (" Wartosc funkcji celu: %f" % fval) print_ += (" Iteracje: %d" % k) print_ += (" Wywolania funkcji: %d" % func_calls[0]) print_ += (" Wywolania gradientu: %d" % grad_calls[0]) print_lst.append(print_) [print(line) for line in print_lst] result = OptimizeResult(fun=fval,lst=print_lst, jac=gfk, hess_inv=Hk, nfev=func_calls[0], njev=grad_calls[0], status=warnflag, success=(warnflag == 0), message=msg, x=xk, nit=k) if retall: result['allvecs'] = allvecs return result
def _minimize_cg(fun, x0, args=(), jac=None, callback=None, gtol=1e-5, norm=Inf, eps=_epsilon, maxiter=None, disp=False, return_all=False, xtol= 1e-6, **unknown_options): """ Minimization of scalar function of one or more variables using the conjugate gradient algorithm. Options for the conjugate gradient algorithm are: disp : bool Set to True to print convergence messages. maxiter : int Maximum number of iterations to perform. gtol : float Gradient norm must be less than `gtol` before successful termination. norm : float Order of norm (Inf is max, -Inf is min). eps : float or ndarray If `jac` is approximated, use this value for the step size. This function is called by the `minimize` function with `method=CG`. It is not supposed to be called directly. """ _check_unknown_options(unknown_options) f = fun fprime = jac epsilon = eps retall = return_all x0 = asarray(x0).flatten() if maxiter is None: maxiter = len(x0) * 200 func_calls, f = wrap_function(f, args) if fprime is None: grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon)) else: grad_calls, myfprime = wrap_function(fprime, args) gfk = myfprime(x0) k = 0 xk = x0 old_fval = f(xk) old_old_fval = None if retall: allvecs = [xk] warnflag = 0 pk = -gfk gnorm = vecnorm(gfk, ord=norm) while (gnorm > gtol) and (k < maxiter): deltak = numpy.dot(gfk, gfk) try: alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ _line_search_wolfe12(f, myfprime, xk, pk, gfk, old_fval, old_old_fval, c2=0.4, xtol=xtol) except _LineSearchError: # Line search failed to find a better solution. warnflag = 2 break xk = xk + alpha_k * pk if retall: allvecs.append(xk) if gfkp1 is None: gfkp1 = myfprime(xk) yk = gfkp1 - gfk beta_k = max(0, numpy.dot(yk, gfkp1) / deltak) pk = -gfkp1 + beta_k * pk gfk = gfkp1 gnorm = vecnorm(gfk, ord=norm) if callback is not None: callback(xk) k += 1 fval = old_fval if warnflag == 2: msg = _status_message['pr_loss'] if disp: print("Warning: " + msg) print(" Current function value: %f" % fval) print(" Iterations: %d" % k) print(" Function evaluations: %d" % func_calls[0]) print(" Gradient evaluations: %d" % grad_calls[0]) elif k >= maxiter: warnflag = 1 msg = _status_message['maxiter'] if disp: print("Warning: " + msg) print(" Current function value: %f" % fval) print(" Iterations: %d" % k) print(" Function evaluations: %d" % func_calls[0]) print(" Gradient evaluations: %d" % grad_calls[0]) else: msg = _status_message['success'] if disp: print(msg) print(" Current function value: %f" % fval) print(" Iterations: %d" % k) print(" Function evaluations: %d" % func_calls[0]) print(" Gradient evaluations: %d" % grad_calls[0]) result = OptimizeResult(fun=fval, jac=gfk, nfev=func_calls[0], njev=grad_calls[0], status=warnflag, success=(warnflag == 0), message=msg, x=xk) if retall: result['allvecs'] = allvecs return result
def _minimize_lbfgsb( fun, x0, bounds=None, args=(), kwargs={}, jac=None, callback=None, tol={"abs": 1e-05, "rel": 1e-08}, norm=np.Inf, maxiter=None, disp=False, return_all=False, **unknown_options ): """ Minimization of scalar function of one or more variables using the BHHH algorithm. Options ------- disp : bool Set to True to print convergence messages. maxiter : int Maximum number of iterations to perform. tol : dict Absolute and relative tolerance values. norm : float Order of norm (Inf is max, -Inf is min). """ _check_unknown_options(unknown_options) def f(x0): return fun(x0, *args, **kwargs) fprime = jac # epsilon = eps Add functionality retall = return_all k = 0 ns = 0 nsmax = 5 N = len(x0) x0 = np.asarray(x0).flatten() if x0.ndim == 0: x0.shape = (1,) if bounds is None: bounds = np.array([np.inf] * N * 2).reshape((2, N)) bounds[0, :] = -bounds[0, :] if bounds.shape[1] != N: raise ValueError("length of x0 != length of bounds") low = bounds[0, :] up = bounds[1, :] x0 = np.clip(x0, low, up) if maxiter is None: maxiter = len(x0) * 200 if not callable(fprime): def myfprime(x0): return approx_derivative(f, x0, args=args, kwargs=kwargs) else: myfprime = fprime # Setup for iteration old_fval = f(x0) gf0 = myfprime(x0) gfk = gf0 norm_pg0 = vecnorm(x0 - np.clip(x0 - gf0, low, up), ord=norm) xk = x0 norm_pgk = norm_pg0 sstore = np.zeros((maxiter, N)) ystore = sstore.copy() if retall: allvecs = [x0] warnflag = 0 # Calculate indices ofactive and inative set using projected gradient epsilon = min(np.min(up - low) / 2, norm_pgk) activeset = np.logical_or(xk - low <= epsilon, up - xk <= epsilon) inactiveset = np.logical_not(activeset) for _ in range(maxiter): # for loop instead. # Check tolerance of gradient norm if norm_pgk <= tol["abs"] + tol["rel"] * norm_pg0: break pk = -gfk pk = bfgsrecb(ns, sstore, ystore, pk, activeset) gfk_active = gfk.copy() gfk_active[inactiveset] = 0 pk = -gfk_active + pk # Sets the initial step guess to dx ~ 1 old_old_fval = old_fval + np.linalg.norm(gfk) / 2 try: alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = _line_search_wolfe12( f, myfprime, xk, pk, gfk, old_fval, old_old_fval, amin=1e-100, amax=1e100, ) except _LineSearchError: # Line search failed to find a better solution. warnflag = 2 break xkp1 = np.clip(xk + alpha_k * pk, low, up) if retall: allvecs.append(xkp1) yk = myfprime(xkp1) - gfk sk = xkp1 - xk xk = xkp1 gfk = myfprime(xkp1) norm_pgk = vecnorm(xk - np.clip(xk - gfk, low, up), ord=norm) # Calculate indices ofactive and inative set using projected gradient epsilon = min(np.min(up - low) / 2, norm_pgk) activeset = np.logical_or(xk - low <= epsilon, up - xk <= epsilon) inactiveset = np.logical_not(activeset) yk[activeset] = 0 sk[activeset] = 0 # reset storage ytsk = yk.dot(sk) if ytsk <= 0: ns = 0 if ns == nsmax: print("ns reached maximum size") ns = 0 elif ytsk > 0: ns += 1 alpha0 = ytsk ** 0.5 sstore[ns - 1, :] = sk / alpha0 ystore[ns - 1, :] = yk / alpha0 k += 1 if callback is not None: callback(xk) if np.isinf(old_fval): # We correctly found +-Inf as optimal value, or something went # wrong. warnflag = 2 break if np.isnan(xk).any(): warnflag = 3 break fval = old_fval if warnflag == 2: msg = _status_message["pr_loss"] elif k >= maxiter: warnflag = 1 msg = _status_message["maxiter"] elif np.isnan(fval) or np.isnan(xk).any(): warnflag = 3 msg = _status_message["nan"] else: msg = _status_message["success"] if disp: print("{}{}".format("Warning: " if warnflag != 0 else "", msg)) print(" Current function value: %f" % fval) print(" Iterations: %d" % k) result = OptimizeResult( fun=fval, jac=gfk, status=warnflag, success=(warnflag == 0), message=msg, x=xk, nit=k, ) if retall: result["allvecs"] = allvecs return result
def Customfmin_bfgs(f, x0, fprime=None, args=(), gtol=1e-5, norm=Inf, epsilon=numpy.sqrt(numpy.finfo(float).eps), maxiter=None, full_output=0, disp=1, retall=0, callback=None): testVar = 0 x0 = asarray(x0).squeeze() if x0.ndim == 0: x0.shape = (1, ) if maxiter is None: maxiter = len(x0) * 200 func_calls, f = wrap_function(f, args) if fprime is None: grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon)) else: grad_calls, myfprime = wrap_function(fprime, args) gfk = myfprime(x0) k = 0 N = len(x0) I = numpy.eye(N, dtype=int) Hk = I old_fval = f(x0) old_old_fval = old_fval + 5000 xk = x0 if retall: allvecs = [x0] sk = [2 * gtol] warnflag = 0 gnorm = vecnorm(gfk, ord=norm) while (gnorm > gtol) and (k < maxiter): pk = -numpy.dot(Hk, gfk) alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ linesearch.line_search(f,myfprime,xk,pk,gfk, old_fval,old_old_fval) if alpha_k is None: # line search failed try different one. alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \ line_search(f,myfprime,xk,pk,gfk, old_fval,old_old_fval) if alpha_k is None: # This line search also failed to find a better solution. warnflag = 2 break xkp1 = xk + alpha_k * pk if retall: allvecs.append(xkp1) sk = xkp1 - xk xk = xkp1 if gfkp1 is None: gfkp1 = myfprime(xkp1) yk = gfkp1 - gfk gfk = gfkp1 if callback is not None: callback(xk) k += 1 gnorm = vecnorm(gfk, ord=norm) if (gnorm <= gtol): break try: # this was handled in numeric, let it remaines for more safety rhok = 1.0 / (numpy.dot(yk, sk)) except ZeroDivisionError: rhok = 1000.0 print("Divide-by-zero encountered: rhok assumed large") if numpy.isinf(rhok): # this is patch for numpy rhok = 1000.0 print("Divide-by-zero encountered: rhok assumed large") A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \ * sk[numpy.newaxis,:] if disp or full_output: fval = old_fval if warnflag == 2: if disp: print("Warning: Desired error not necessarily achieved" \ "due to precision loss") print(" Current function value: %f" % fval) print(" Iterations: %d" % k) print(" Function evaluations: %d" % func_calls[0]) print(" Gradient evaluations: %d" % grad_calls[0]) elif k >= maxiter: warnflag = 1 if disp: print("Warning: Maximum number of iterations has been exceeded") print(" Current function value: %f" % fval) print(" Iterations: %d" % k) print(" Function evaluations: %d" % func_calls[0]) print(" Gradient evaluations: %d" % grad_calls[0]) else: if disp: print("Optimization terminated successfully.") print(" Current function value: %f" % fval) print(" Iterations: %d" % k) print(" Function evaluations: %d" % func_calls[0]) print(" Gradient evaluations: %d" % grad_calls[0]) if full_output: retlist = xk, fval, gfk, Hk, func_calls[0], grad_calls[0], warnflag if retall: retlist += (allvecs, ) else: retlist = xk if retall: retlist = (xk, allvecs) return retlist
def leon_ncg_python(make_f, w_0, make_fprime=None, gtol=1e-5, norm=numpy.Inf, maxiter=None, full_output=0, disp=1, retall=0, callback=None, direction='hestenes-stiefel', minibatch_size=None, minibatch_offset=None, restart_every=0, normalize=False, constrain_lambda=True, ): """Minimize a function using a nonlinear conjugate gradient algorithm. Parameters ---------- make_f : callable make_f(k0, k1) When called with (k0, k1) as arguments, return a function f such that f(w) is the objective to be minimize at parameter w, on minibatch x_k0 to x_k1. If k1 is None then the minibatch should contain all the remaining data. w_0 : ndarray Initial guess. make_fprime : callable make_f'(k0, k1) Same as `make_f`, but to compute the derivative of f on a minibatch. gtol : float Stop when norm of gradient is less than gtol. norm : float Order of vector norm to use. -Inf is min, Inf is max. size (can be scalar or vector). callback : callable An optional user-supplied function, called after each iteration. Called as callback(w_t, lambda_t), where w_t is the current parameter vector and lambda_t the coefficient for the new direction. direction : string Formula used to computed the new direction, among: - polak-ribiere - hestenes-stiefel minibatch_size : int Size of each minibatch. Use None for batch learning. minibatch_offset : int Shift of the minibatch. Use None to use the minibatch size (i.e. no overlap at all). restart_every : int Force restart every this number of iterations. If <= 0, then never force a restart. normalize : bool If True, then use the normalized gradient instead of the gradient itself to find the next search direction, and always normalize the search direction. constrain_lambda : bool If True, then the `lambda_t` factor used to compute conjugate directions is constrained to be non-negative (it is thus set to zero if the formula given by `direction` computes a negative value). Returns ------- xopt : ndarray Parameters which minimize f, i.e. f(xopt) == fopt. fopt : float Minimum value found, f(xopt). func_calls : int The number of function_calls made. grad_calls : int The number of gradient calls made. warnflag : int 1 : Maximum number of iterations exceeded. 2 : Gradient and/or function calls not changing. allvecs : ndarray If retall is True (see other parameters below), then this vector containing the result at each iteration is returned. Other Parameters ---------------- maxiter : int Maximum number of iterations to perform. full_output : bool If True then return fopt, func_calls, grad_calls, and warnflag in addition to xopt. disp : bool Print convergence message if True. retall : bool Return a list of results at each iteration if True. Notes ----- Optimize the function, f, whose gradient is given by fprime using the nonlinear conjugate gradient algorithm of Polak and Ribiere. See Wright & Nocedal, 'Numerical Optimization', 1999, pg. 120-122. """ if minibatch_offset is None: if minibatch_size is None: # Batch learning: no offset is needed. minibatch_offset = 0 else: # Use the same offset as the minibatch size. minibatch_offset = minibatch_size w_0 = numpy.asarray(w_0).flatten() if maxiter is None: maxiter = len(w_0)*200 k0 = 0 k1 = minibatch_size assert make_fprime is not None f = make_f(k0, k1) fprime = make_fprime(k0, k1) func_calls = [0] grad_calls = [0] tmp_func_calls, f = wrap_function(f, ()) tmp_grad_calls, myfprime = wrap_function(fprime, ()) g_t = myfprime(w_0) t = 0 N = len(w_0) w_t = w_0 if retall: allvecs = [w_t] warnflag = 0 if normalize: d_t = -g_t / numpy.linalg.norm(g_t) else: d_t = -g_t gnorm = vecnorm(g_t, ord=norm) w_t_previous = None while (gnorm > gtol) and (t < maxiter): #print '||g_t|| = %s' % numpy.linalg.norm(g_t) # Since the function changes at each iteration, we cannot re-use # previous function values. old_fval = f(w_t) if w_t_previous is None: old_old_fval = old_fval + 5000 else: old_old_fval = f(w_t_previous) # These values are modified by the line search, even if it fails. old_fval_backup = old_fval old_old_fval_backup = old_old_fval alpha_t, fc, gc, old_fval, old_old_fval, h_t = \ line_search_wolfe1(f, myfprime, w_t, d_t, g_t, old_fval, old_old_fval, c2=0.4) if alpha_t is None: # line search failed -- use different one. print '*********************************** LINE SEARCH FAILURE *********************************' alpha_t, fc, gc, old_fval, old_old_fval, h_t = \ line_search_wolfe2(f, myfprime, w_t, d_t, g_t, old_fval_backup, old_old_fval_backup) print '*********************************** %s *********************************' % alpha_t if alpha_t is None or alpha_t == 0: # This line search also failed to find a better solution. raise AssertionError() warnflag = 2 break print 'alpha_t = %s' % alpha_t # Update weights. w_tp1 = w_t + alpha_t * d_t # Compute derivative after the weight update, if not done already. if h_t is None: h_t = myfprime(w_tp1) else: assert (h_t == myfprime(w_tp1)).all() # Sanity check. # Switch to next minibatch. func_calls[0] += tmp_func_calls[0] grad_calls[0] += tmp_grad_calls[0] k0 += minibatch_offset if minibatch_size is None: k1 = None else: k1 = k0 + minibatch_size tmp_func_calls, f = wrap_function(make_f(k0, k1), ()) tmp_grad_calls, myfprime = wrap_function(make_fprime(k0, k1), ()) # Compute derivative on new minibatch. g_tp1 = myfprime(w_tp1) if normalize: g_tp1_for_dt = g_tp1 / numpy.linalg.norm(g_tp1) else: g_tp1_for_dt = g_tp1 if retall: allvecs.append(w_tp1) h_t_minus_g_t = h_t - g_t if direction == 'polak-ribiere': # Polak-Ribiere. delta_t = numpy.dot(g_t, g_t) lambda_t = numpy.dot(h_t_minus_g_t, g_tp1_for_dt) / delta_t elif direction == 'hestenes-stiefel': # Hestenes-Stiefel. lambda_t = numpy.dot(h_t_minus_g_t, g_tp1_for_dt) / numpy.dot(h_t_minus_g_t, d_t) else: raise NotImplementedError(direction) if constrain_lambda and lambda_t < 0: lambda_t = 0 if restart_every > 0 and (t + 1) % restart_every == 0: lambda_t = 0 if lambda_t == 0: print '*** RESTART ***' else: print 'lambda_t = %s' % lambda_t d_t = -g_tp1_for_dt + lambda_t * d_t if normalize: d_t /= numpy.linalg.norm(d_t) g_t = g_tp1 w_t_previous = w_t w_t = w_tp1 gnorm = vecnorm(g_t, ord=norm) if callback is not None: callback(w_t, lambda_t) t += 1 if disp or full_output: fval = old_fval if warnflag == 2: if disp: print "Warning: Desired error not necessarily achieved due to precision loss" print " Current function value: %f" % fval print " Iterations: %d" % t print " Function evaluations: %d" % func_calls[0] print " Gradient evaluations: %d" % grad_calls[0] elif t >= maxiter: warnflag = 1 if disp: print "Warning: Maximum number of iterations has been exceeded" print " Current function value: %f" % fval print " Iterations: %d" % t print " Function evaluations: %d" % func_calls[0] print " Gradient evaluations: %d" % grad_calls[0] else: if disp: print "Optimization terminated successfully." print " Current function value: %f" % fval print " Iterations: %d" % t print " Function evaluations: %d" % func_calls[0] print " Gradient evaluations: %d" % grad_calls[0] if full_output: retlist = w_t, fval, func_calls[0], grad_calls[0], warnflag if retall: retlist += (allvecs,) else: retlist = w_t if retall: retlist = (w_t, allvecs) return retlist
def _minimize_bhhh(fun, x0, bounds=None, args=(), jac=None, callback=None, tol={ "abs": 1e-05, "rel": 1e-08 }, norm=np.Inf, maxiter=None, disp=False, return_all=False, **unknown_options): """ Minimization of scalar function of one or more variables using the BHHH algorithm. Options ------- disp : bool Set to True to print convergence messages. maxiter : int Maximum number of iterations to perform. tol : dict Absolute and relative tolerance values. norm : float Order of norm (Inf is max, -Inf is min). """ _check_unknown_options(unknown_options) f = fun fprime = jac retall = return_all k = 0 N = len(x0) x0 = np.asarray(x0).flatten() if x0.ndim == 0: x0.shape = (1, ) if bounds is None: bounds = np.array([np.inf] * N * 2).reshape((2, N)) bounds[0, :] = -bounds[0, :] if bounds.shape[1] != N: raise ValueError("length of x0 != length of bounds") low = bounds[0, :] up = bounds[1, :] x0 = np.clip(x0, low, up) if maxiter is None: maxiter = len(x0) * 200 # Need the aggregate functions to take only x0 as an argument func_calls, agg_fun = wrap_function_agg(f, args) if not callable(fprime): grad_calls, myfprime = wrap_function_num_dev(f, args) else: grad_calls, myfprime = wrap_function(fprime, args) def agg_fprime(x0): return myfprime(x0).sum(axis=0) # Setup for iteration old_fval = agg_fun(x0) gf0 = agg_fprime(x0) norm_pg0 = vecnorm(x0 - np.clip(x0 - gf0, low, up), ord=norm) xk = x0 norm_pgk = norm_pg0 if retall: allvecs = [x0] warnflag = 0 for _ in range(maxiter): # for loop instead. # Individual gfk_obs = myfprime(xk) # Aggregate fprime. Might replace by simply summing up gfk_obs gfk = gfk_obs.sum(axis=0) norm_pgk = vecnorm(xk - np.clip(xk - gfk, low, up), ord=norm) # Check tolerance of gradient norm if norm_pgk <= tol["abs"] + tol["rel"] * norm_pg0: break # Sets the initial step guess to dx ~ 1 old_old_fval = old_fval + np.linalg.norm(gfk) / 2 # Calculate BHHH hessian and step Hk = np.dot(gfk_obs.T, gfk_obs) Bk = np.linalg.inv(Hk) pk = np.empty(N) pk = -np.dot(Bk, gfk) try: alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = _line_search_wolfe12( agg_fun, agg_fprime, xk, pk, gfk, old_fval, old_old_fval, amin=1e-100, amax=1e100, ) except _LineSearchError: # Line search failed to find a better solution. warnflag = 2 break xkp1 = np.clip(xk + alpha_k * pk, low, up) if retall: allvecs.append(xkp1) xk = xkp1 if callback is not None: callback(xk) k += 1 if np.isinf(old_fval): # We correctly found +-Inf as optimal value, or something went # wrong. warnflag = 2 break fval = old_fval if warnflag == 2: msg = _status_message["pr_loss"] elif k >= maxiter: warnflag = 1 msg = _status_message["maxiter"] elif np.isnan(fval) or np.isnan(xk).any(): warnflag = 3 msg = _status_message["nan"] else: msg = _status_message["success"] if disp: print("{}{}".format("Warning: " if warnflag != 0 else "", msg)) print(" Current function value: %f" % fval) print(" Iterations: %d" % k) result = OptimizeResult( fun=fval, jac=gfk, hess_inv=Bk, nfev=func_calls[0], njev=grad_calls[0], status=warnflag, success=(warnflag == 0), message=msg, x=xk, nit=k, ) if retall: result["allvecs"] = allvecs return result