def loopThing_BFGS(inputParams):

    for key in inputParams:
        exec(key + " = inputParams['" + key + "']")

    pk = -numpy.dot(Hk, gfk)
    alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
       linesearch.line_search(f,myfprime,xk,pk,gfk,
                              old_fval,old_old_fval)
    if alpha_k is None:  # line search failed try different one.
        alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
                 line_search(f,myfprime,xk,pk,gfk,
                             old_fval,old_old_fval)
        if alpha_k is None:
            # This line search also failed to find a better solution.
            warnflag = 2
            outputParams = getOutputParams(inputParams, list(locals().items()))
            return outputParams

    xkp1 = xk + alpha_k * pk
    if retall:
        allvecs.append(xkp1)
    sk = xkp1 - xk
    xk = xkp1
    if gfkp1 is None:
        gfkp1 = myfprime(xkp1)

    yk = gfkp1 - gfk
    gfk = gfkp1
    if callback is not None:
        callback(xk)
    k += 1
    gnorm = vecnorm(gfk, ord=norm)
    if (gnorm <= gtol):
        outputParams = getOutputParams(inputParams, list(locals().items()))
        return outputParams

    try:  # this was handled in numeric, let it remaines for more safety
        rhok = 1.0 / (numpy.dot(yk, sk))
    except ZeroDivisionError:
        rhok = 1000.0
        print("Divide-by-zero encountered: rhok assumed large")
    if numpy.isinf(rhok):  # this is patch for numpy
        rhok = 1000.0
        print("Divide-by-zero encountered: rhok assumed large")
    A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok
    A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok
    Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
             * sk[numpy.newaxis,:]

    outputParams = getOutputParams(inputParams, list(locals().items()))
    return outputParams
Exemple #2
0
def loopThing(inputParams):

    for key in inputParams:
        exec(key + " = inputParams['" + key + "']")

    pk = -numpy.dot(Hk,gfk)
    alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
       linesearch.line_search(f,myfprime,xk,pk,gfk,
                              old_fval,old_old_fval)
    if alpha_k is None:  # line search failed try different one.
        alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
                 line_search(f,myfprime,xk,pk,gfk,
                             old_fval,old_old_fval)
        if alpha_k is None:
            # This line search also failed to find a better solution.
            warnflag = 2
            outputParams = getOutputParams(inputParams, locals().items())
            return outputParams

    xkp1 = xk + alpha_k * pk
    if retall:
        allvecs.append(xkp1)
    sk = xkp1 - xk
    xk = xkp1
    if gfkp1 is None:
        gfkp1 = myfprime(xkp1)

    yk = gfkp1 - gfk
    gfk = gfkp1
    if callback is not None:
        callback(xk)
    k += 1
    gnorm = vecnorm(gfk,ord=norm)
    if (gnorm <= gtol):
        outputParams = getOutputParams(inputParams, locals().items())
        return outputParams

    try: # this was handled in numeric, let it remaines for more safety
        rhok = 1.0 / (numpy.dot(yk,sk))
    except ZeroDivisionError:
        rhok = 1000.0
        print "Divide-by-zero encountered: rhok assumed large"
    if numpy.isinf(rhok): # this is patch for numpy
        rhok = 1000.0
        print "Divide-by-zero encountered: rhok assumed large"
    A1 = I - sk[:,numpy.newaxis] * yk[numpy.newaxis,:] * rhok
    A2 = I - yk[:,numpy.newaxis] * sk[numpy.newaxis,:] * rhok
    Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
             * sk[numpy.newaxis,:]

    outputParams = getOutputParams(inputParams, locals().items())
    return outputParams
Exemple #3
0
def line_search(f, myfprime, xk, pk, gfk, old_fval, old_old_fval,
                args=(), c1=1e-4, c2=0.9, amax=50):

    try: #XXX: break dependency on scipy.optimize.linesearch
        from scipy.optimize import linesearch
        alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
                         linesearch.line_search(f,myfprime,xk,pk,gfk,\
                         old_fval,old_old_fval,args,c1,c2,amax)
    except ImportError:
        alpha_k = None
        fc = 0
        gc = 0
        gfkp1 = gfk #XXX: or None ?

    return alpha_k, fc, gc, old_fval, old_old_fval, gfkp1
Exemple #4
0
def line_search(f, myfprime, xk, pk, gfk, old_fval, old_old_fval,
                args=(), c1=1e-4, c2=0.9, amax=50):

    try: #XXX: break dependency on scipy.optimize.linesearch
        from scipy.optimize import linesearch
        alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
                         linesearch.line_search(f,myfprime,xk,pk,gfk,\
                         old_fval,old_old_fval,args,c1,c2,amax)
    except ImportError:
        alpha_k = None
        fc = 0
        gc = 0
        gfkp1 = gfk #XXX: or None ?

    return alpha_k, fc, gc, old_fval, old_old_fval, gfkp1
Exemple #5
0
def my_fmin_bfgs(f, x0, fprime=None, args=(), gtol=1e-5, norm=Inf,
                 epsilon=_epsilon, maxiter=None, full_output=0, disp=1,
                 retall=0, callback=None):
    """Minimize a function using the BFGS algorithm.

    :Parameters:

      f : callable f(x,*args)
          Objective function to be minimized.
      x0 : ndarray
          Initial guess.
      fprime : callable f'(x,*args)
          Gradient of f.
      args : tuple
          Extra arguments passed to f and fprime.
      gtol : float
          Gradient norm must be less than gtol before succesful termination.
      norm : float
          Order of norm (Inf is max, -Inf is min)
      epsilon : int or ndarray
          If fprime is approximated, use this value for the step size.
      callback : callable
          An optional user-supplied function to call after each
          iteration.  Called as callback(xk), where xk is the
          current parameter vector.

    :Returns: (xopt, {fopt, gopt, Hopt, func_calls, grad_calls, warnflag}, <allvecs>)

        xopt : ndarray
            Parameters which minimize f, i.e. f(xopt) == fopt.
        fopt : float
            Minimum value.
        gopt : ndarray
            Value of gradient at minimum, f'(xopt), which should be near 0.
        Bopt : ndarray
            Value of 1/f''(xopt), i.e. the inverse hessian matrix.
        func_calls : int
            Number of function_calls made.
        grad_calls : int
            Number of gradient calls made.
        warnflag : integer
            1 : Maximum number of iterations exceeded.
            2 : Gradient and/or function calls not changing.
        allvecs  :  list
            Results at each iteration.  Only returned if retall is True.

    *Other Parameters*:
        maxiter : int
            Maximum number of iterations to perform.
        full_output : bool
            If True,return fopt, func_calls, grad_calls, and warnflag
            in addition to xopt.
        disp : bool
            Print convergence message if True.
        retall : bool
            Return a list of results at each iteration if True.

    :Notes:

        Optimize the function, f, whose gradient is given by fprime
        using the quasi-Newton method of Broyden, Fletcher, Goldfarb,
        and Shanno (BFGS) See Wright, and Nocedal 'Numerical
        Optimization', 1999, pg. 198.

    *See Also*:

      scikits.openopt : SciKit which offers a unified syntax to call
                        this and other solvers.

    """
    x0 = asarray(x0).squeeze()
    if x0.ndim == 0:
        x0.shape = (1,)
    if maxiter is None:
        maxiter = len(x0)*200
    func_calls, f = wrap_function(f, args)
    if fprime is None:
        grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
    else:
        grad_calls, myfprime = wrap_function(fprime, args)

    print "Evaluating initial gradient ..."
    gfk = myfprime(x0)

    k = 0
    N = len(x0)
    I = numpy.eye(N,dtype=int)
    Hk = I

    print "Evaluating initial function value ..."
    fval = f(x0)

    old_fval = fval + 5000
    xk = x0
    if retall:
        allvecs = [x0]
    sk = [2*gtol]
    warnflag = 0
    gnorm = vecnorm(gfk,ord=norm)

    print "gtol  = %g" % gtol
    print "gnorm = %g" % gnorm

    while (gnorm > gtol) and (k < maxiter):
        pk = -numpy.dot(Hk,gfk)

        print "xk =", xk
        print "pk =", pk
        print "Begin iteration %d line search..." % (k + 1)
#        print "  gfk =", gfk
#        print "  Hk = \n", Hk

        # do line search for alpha_k
        old_old_fval = old_fval
        old_fval = fval
        alpha_k, fc, gc, fval, old_fval, gfkp1 = \
           linesearch.line_search(f,myfprime,xk,pk,gfk,
                                  old_fval,old_old_fval)
        if alpha_k is None:  # line search failed try different one.
            print "Begin line search (method 2) ..."
            alpha_k, fc, gc, fval, old_fval, gfkp1 = \
                     line_search(f,myfprime,xk,pk,gfk,
                                 old_fval,old_old_fval)
            if alpha_k is None:
                # This line search also failed to find a better solution.
                print "Line search failed!"
                warnflag = 2
                break
        print "End line search, alpha = %g ..." % alpha_k

        xkp1 = xk + alpha_k * pk
        if retall:
            allvecs.append(xkp1)
        sk = xkp1 - xk
        xk = xkp1
        if gfkp1 is None:
            gfkp1 = myfprime(xkp1)

        yk = gfkp1 - gfk
        gfk = gfkp1
        if callback is not None:
            callback(xk)
        k += 1
        gnorm = vecnorm(gfk,ord=norm)
        print "gnorm = %g" % gnorm
        if (k >= maxiter or gnorm <= gtol):
            break

        # Reset the initial quasi-Newton matrix to a scaled identity aimed
        # at reflecting the size of the inverse true Hessian
        deltaXDeltaGrad = numpy.dot(sk, yk);
        updateOk = deltaXDeltaGrad >= _epsilon * max(_epsilonSq, \
                       vecnorm(sk,ord=2) * vecnorm(yk, ord=2))
        if k == 1 and updateOk:
            Hk = deltaXDeltaGrad / numpy.dot(yk,yk) * numpy.eye(N);
            print "Hscaled =\n", Hk

        try: # this was handled in numeric, let it remain for more safety
            rhok = 1.0 / (numpy.dot(yk,sk))
        except ZeroDivisionError:
            rhok = 1000.0
            print "Divide-by-zero encountered: rhok assumed large"
        if isinf(rhok): # this is patch for numpy
            rhok = 1000.0
            print "Divide-by-zero encountered: rhok assumed large"
        A1 = I - sk[:,numpy.newaxis] * yk[numpy.newaxis,:] * rhok
        A2 = I - yk[:,numpy.newaxis] * sk[numpy.newaxis,:] * rhok
        Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
                 * sk[numpy.newaxis,:]

    if gnorm > gtol:
        warnflag = 1

    if disp:
        if warnflag == 1:
            print "Warning: Maximum number of iterations has been exceeded"
        elif warnflag == 2:
            print "Warning: Desired error not necessarily achieved" \
                  "due to precision loss"
        else:
            print "Optimization terminated successfully."

        print "         Current function value: %g" % fval
        print "         Current gradient norm : %g" % gnorm
        print "         Gradient tolerance    : %g" % gtol
        print "         Iterations: %d" % k
        print "         Function evaluations: %d" % func_calls[0]
        print "         Gradient evaluations: %d" % grad_calls[0]

    if full_output:
        retlist = xk, fval, gfk, Hk, func_calls[0], grad_calls[0], warnflag
        if retall:
            retlist += (allvecs,)
    else:
        retlist = xk
        if retall:
            retlist = (xk, allvecs)

    return retlist
Exemple #6
0
def my_fmin_bfgs(f,
                 x0,
                 fprime=None,
                 args=(),
                 gtol=1e-5,
                 norm=Inf,
                 epsilon=_epsilon,
                 maxiter=None,
                 full_output=0,
                 disp=1,
                 retall=0,
                 callback=None):
    """Minimize a function using the BFGS algorithm.

    :Parameters:

      f : callable f(x,*args)
          Objective function to be minimized.
      x0 : ndarray
          Initial guess.
      fprime : callable f'(x,*args)
          Gradient of f.
      args : tuple
          Extra arguments passed to f and fprime.
      gtol : float
          Gradient norm must be less than gtol before succesful termination.
      norm : float
          Order of norm (Inf is max, -Inf is min)
      epsilon : int or ndarray
          If fprime is approximated, use this value for the step size.
      callback : callable
          An optional user-supplied function to call after each
          iteration.  Called as callback(xk), where xk is the
          current parameter vector.

    :Returns: (xopt, {fopt, gopt, Hopt, func_calls, grad_calls, warnflag}, <allvecs>)

        xopt : ndarray
            Parameters which minimize f, i.e. f(xopt) == fopt.
        fopt : float
            Minimum value.
        gopt : ndarray
            Value of gradient at minimum, f'(xopt), which should be near 0.
        Bopt : ndarray
            Value of 1/f''(xopt), i.e. the inverse hessian matrix.
        func_calls : int
            Number of function_calls made.
        grad_calls : int
            Number of gradient calls made.
        warnflag : integer
            1 : Maximum number of iterations exceeded.
            2 : Gradient and/or function calls not changing.
        allvecs  :  list
            Results at each iteration.  Only returned if retall is True.

    *Other Parameters*:
        maxiter : int
            Maximum number of iterations to perform.
        full_output : bool
            If True,return fopt, func_calls, grad_calls, and warnflag
            in addition to xopt.
        disp : bool
            Print convergence message if True.
        retall : bool
            Return a list of results at each iteration if True.

    :Notes:

        Optimize the function, f, whose gradient is given by fprime
        using the quasi-Newton method of Broyden, Fletcher, Goldfarb,
        and Shanno (BFGS) See Wright, and Nocedal 'Numerical
        Optimization', 1999, pg. 198.

    *See Also*:

      scikits.openopt : SciKit which offers a unified syntax to call
                        this and other solvers.

    """
    x0 = asarray(x0).squeeze()
    if x0.ndim == 0:
        x0.shape = (1, )
    if maxiter is None:
        maxiter = len(x0) * 200
    func_calls, f = wrap_function(f, args)
    if fprime is None:
        grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
    else:
        grad_calls, myfprime = wrap_function(fprime, args)

    print "Evaluating initial gradient ..."
    gfk = myfprime(x0)

    k = 0
    N = len(x0)
    I = numpy.eye(N, dtype=int)
    Hk = I

    print "Evaluating initial function value ..."
    fval = f(x0)

    old_fval = fval + 5000
    xk = x0
    if retall:
        allvecs = [x0]
    sk = [2 * gtol]
    warnflag = 0
    gnorm = vecnorm(gfk, ord=norm)

    print "gtol  = %g" % gtol
    print "gnorm = %g" % gnorm

    while (gnorm > gtol) and (k < maxiter):
        pk = -numpy.dot(Hk, gfk)

        print "xk =", xk
        print "pk =", pk
        print "Begin iteration %d line search..." % (k + 1)
        #        print "  gfk =", gfk
        #        print "  Hk = \n", Hk

        # do line search for alpha_k
        old_old_fval = old_fval
        old_fval = fval
        alpha_k, fc, gc, fval, old_fval, gfkp1 = \
           linesearch.line_search(f,myfprime,xk,pk,gfk,
                                  old_fval,old_old_fval)
        if alpha_k is None:  # line search failed try different one.
            print "Begin line search (method 2) ..."
            alpha_k, fc, gc, fval, old_fval, gfkp1 = \
                     line_search(f,myfprime,xk,pk,gfk,
                                 old_fval,old_old_fval)
            if alpha_k is None:
                # This line search also failed to find a better solution.
                print "Line search failed!"
                warnflag = 2
                break
        print "End line search, alpha = %g ..." % alpha_k

        xkp1 = xk + alpha_k * pk
        if retall:
            allvecs.append(xkp1)
        sk = xkp1 - xk
        xk = xkp1
        if gfkp1 is None:
            gfkp1 = myfprime(xkp1)

        yk = gfkp1 - gfk
        gfk = gfkp1
        if callback is not None:
            callback(xk)
        k += 1
        gnorm = vecnorm(gfk, ord=norm)
        print "gnorm = %g" % gnorm
        if (k >= maxiter or gnorm <= gtol):
            break

        # Reset the initial quasi-Newton matrix to a scaled identity aimed
        # at reflecting the size of the inverse true Hessian
        deltaXDeltaGrad = numpy.dot(sk, yk)
        updateOk = deltaXDeltaGrad >= _epsilon * max(_epsilonSq, \
                       vecnorm(sk,ord=2) * vecnorm(yk, ord=2))
        if k == 1 and updateOk:
            Hk = deltaXDeltaGrad / numpy.dot(yk, yk) * numpy.eye(N)
            print "Hscaled =\n", Hk

        try:  # this was handled in numeric, let it remain for more safety
            rhok = 1.0 / (numpy.dot(yk, sk))
        except ZeroDivisionError:
            rhok = 1000.0
            print "Divide-by-zero encountered: rhok assumed large"
        if isinf(rhok):  # this is patch for numpy
            rhok = 1000.0
            print "Divide-by-zero encountered: rhok assumed large"
        A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok
        A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok
        Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
                 * sk[numpy.newaxis,:]

    if gnorm > gtol:
        warnflag = 1

    if disp:
        if warnflag == 1:
            print "Warning: Maximum number of iterations has been exceeded"
        elif warnflag == 2:
            print "Warning: Desired error not necessarily achieved" \
                  "due to precision loss"
        else:
            print "Optimization terminated successfully."

        print "         Current function value: %g" % fval
        print "         Current gradient norm : %g" % gnorm
        print "         Gradient tolerance    : %g" % gtol
        print "         Iterations: %d" % k
        print "         Function evaluations: %d" % func_calls[0]
        print "         Gradient evaluations: %d" % grad_calls[0]

    if full_output:
        retlist = xk, fval, gfk, Hk, func_calls[0], grad_calls[0], warnflag
        if retall:
            retlist += (allvecs, )
    else:
        retlist = xk
        if retall:
            retlist = (xk, allvecs)

    return retlist
Exemple #7
0
    def __call__( self, x0, conf = None, obj_fun = None, obj_fun_grad = None,
                  status = None, obj_args = None ):
#    def fmin_sd( conf, x0, fn_of, fn_ofg, args = () ):

        conf = get_default( conf, self.conf )
        obj_fun = get_default( obj_fun, self.obj_fun )
        obj_fun_grad = get_default( obj_fun_grad, self.obj_fun_grad )
        status = get_default( status, self.status )
        obj_args = get_default( obj_args, self.obj_args )

        if conf.output:
            globals()['output'] = conf.output

        output( 'entering optimization loop...' )

        nc_of, tt_of, fn_of = wrap_function( obj_fun, obj_args )
        nc_ofg, tt_ofg, fn_ofg = wrap_function( obj_fun_grad, obj_args )

        time_stats = {'of' : tt_of, 'ofg': tt_ofg, 'check' : []}

        ofg = None

        it = 0
        xit = x0.copy()
        while 1:

            of = fn_of( xit )

            if it == 0:
                of0 = ofit0 = of_prev = of
                of_prev_prev = of + 5000.0

            if ofg is None:
                ofg = fn_ofg( xit )

            if conf.check:
                tt = time.clock()
                check_gradient( xit, ofg, fn_of, conf.delta, conf.check )
                time_stats['check'].append( time.clock() - tt )

            ofg_norm = nla.norm( ofg, conf.norm )

            ret = conv_test( conf, it, of, ofit0, ofg_norm )
            if ret >= 0:
                break
            ofit0 = of

            ##
            # Backtrack (on errors).
            alpha = conf.ls0
            can_ls = True
            while 1:
                xit2 = xit - alpha * ofg
                aux = fn_of( xit2 )

                if self.log is not None:
                    self.log(of, ofg_norm, alpha, it)

                if aux is None:
                    alpha *= conf.ls_red_warp
                    can_ls = False
                    output( 'warp: reducing step (%f)' % alpha )
                elif conf.ls and conf.ls_method == 'backtracking':
                    if aux < of * conf.ls_on: break
                    alpha *= conf.ls_red
                    output( 'backtracking: reducing step (%f)' % alpha )
                else:
                    of_prev_prev = of_prev
                    of_prev = aux
                    break

                if alpha < conf.ls_min:
                    if aux is None:
                        raise RuntimeError, 'giving up...'
                    output( 'linesearch failed, continuing anyway' )
                    break

            # These values are modified by the line search, even if it fails
            of_prev_bak = of_prev
            of_prev_prev_bak = of_prev_prev

            if conf.ls and can_ls and conf.ls_method == 'full':
                output( 'full linesearch...' )
                alpha, fc, gc, of_prev, of_prev_prev, ofg1 = \
                       linesearch.line_search(fn_of,fn_ofg,xit,
                                              -ofg,ofg,of_prev,of_prev_prev,
                                              c2=0.4)
                if alpha is None:  # line search failed -- use different one.
                    alpha, fc, gc, of_prev, of_prev_prev, ofg1 = \
                           sopt.line_search(fn_of,fn_ofg,xit,
                                            -ofg,ofg,of_prev_bak,
                                            of_prev_prev_bak)
                    if alpha is None or alpha == 0:
                        # This line search also failed to find a better solution.
                        ret = 3
                        break
                output( ' -> alpha: %.8e' % alpha )
            else:
                if conf.ls_method == 'full':
                    output( 'full linesearch off (%s and %s)' % (conf.ls,
                                                                 can_ls) )
                ofg1 = None

            if self.log is not None:
                self.log.plot_vlines(color='g', linewidth=0.5)

            xit = xit - alpha * ofg
            if ofg1 is None:
                ofg = None
            else:
                ofg = ofg1.copy()

            for key, val in time_stats.iteritems():
                if len( val ):
                    output( '%10s: %7.2f [s]' % (key, val[-1]) )

            it = it + 1

        output( 'status:               %d' % ret )
        output( 'initial value:        %.8e' % of0 )
        output( 'current value:        %.8e' % of )
        output( 'iterations:           %d' % it )
        output( 'function evaluations: %d in %.2f [s]' \
              % (nc_of[0], nm.sum( time_stats['of'] ) ) )
        output( 'gradient evaluations: %d in %.2f [s]' \
              % (nc_ofg[0], nm.sum( time_stats['ofg'] ) ) )

        if self.log is not None:
            self.log(of, ofg_norm, alpha, it)

            if conf.log.plot is not None:
                self.log(save_figure=conf.log.plot,
                         finished=True)
            else:
                self.log(finished=True)
                
        if status is not None:
            status['log'] = self.log
            status['status'] = status
            status['of0'] = of0
            status['of'] = of
            status['it'] = it
            status['nc_of'] = nc_of[0]
            status['nc_ofg'] = nc_ofg[0]
            status['time_stats'] = time_stats

        return xit
Exemple #8
0
    def __call__(self,
                 x0,
                 conf=None,
                 obj_fun=None,
                 obj_fun_grad=None,
                 status=None,
                 obj_args=None):
        conf = get_default(conf, self.conf)
        obj_fun = get_default(obj_fun, self.obj_fun)
        obj_fun_grad = get_default(obj_fun_grad, self.obj_fun_grad)
        status = get_default(status, self.status)
        obj_args = get_default(obj_args, self.obj_args)

        if conf.output:
            globals()['output'] = conf.output

        output('entering optimization loop...')

        nc_of, tt_of, fn_of = wrap_function(obj_fun, obj_args)
        nc_ofg, tt_ofg, fn_ofg = wrap_function(obj_fun_grad, obj_args)

        time_stats = {'of': tt_of, 'ofg': tt_ofg, 'check': []}

        ofg = None

        it = 0
        xit = x0.copy()
        while 1:

            of = fn_of(xit)

            if it == 0:
                of0 = ofit0 = of_prev = of
                of_prev_prev = of + 5000.0

            if ofg is None:
                ofg = fn_ofg(xit)

            if conf.check:
                tt = time.clock()
                check_gradient(xit, ofg, fn_of, conf.delta, conf.check)
                time_stats['check'].append(time.clock() - tt)

            ofg_norm = nla.norm(ofg, conf.norm)

            ret = conv_test(conf, it, of, ofit0, ofg_norm)
            if ret >= 0:
                break
            ofit0 = of

            ##
            # Backtrack (on errors).
            alpha = conf.ls0
            can_ls = True
            while 1:
                xit2 = xit - alpha * ofg
                aux = fn_of(xit2)

                if self.log is not None:
                    self.log(of, ofg_norm, alpha, it)

                if aux is None:
                    alpha *= conf.ls_red_warp
                    can_ls = False
                    output('warp: reducing step (%f)' % alpha)
                elif conf.ls and conf.ls_method == 'backtracking':
                    if aux < of * conf.ls_on: break
                    alpha *= conf.ls_red
                    output('backtracking: reducing step (%f)' % alpha)
                else:
                    of_prev_prev = of_prev
                    of_prev = aux
                    break

                if alpha < conf.ls_min:
                    if aux is None:
                        raise RuntimeError, 'giving up...'
                    output('linesearch failed, continuing anyway')
                    break

            # These values are modified by the line search, even if it fails
            of_prev_bak = of_prev
            of_prev_prev_bak = of_prev_prev

            if conf.ls and can_ls and conf.ls_method == 'full':
                output('full linesearch...')
                alpha, fc, gc, of_prev, of_prev_prev, ofg1 = \
                    linesearch.line_search(fn_of,fn_ofg,xit,
                                           -ofg,ofg,of_prev,of_prev_prev,
                                           c2=0.4)
                if alpha is None:  # line search failed -- use different one.
                    alpha, fc, gc, of_prev, of_prev_prev, ofg1 = \
                        sopt.line_search(fn_of,fn_ofg,xit,
                                         -ofg,ofg,of_prev_bak,
                                         of_prev_prev_bak)
                    if alpha is None or alpha == 0:
                        # This line search also failed to find a better
                        # solution.
                        ret = 3
                        break
                output(' -> alpha: %.8e' % alpha)
            else:
                if conf.ls_method == 'full':
                    output('full linesearch off (%s and %s)' %
                           (conf.ls, can_ls))
                ofg1 = None

            if self.log is not None:
                self.log.plot_vlines(color='g', linewidth=0.5)

            xit = xit - alpha * ofg
            if ofg1 is None:
                ofg = None
            else:
                ofg = ofg1.copy()

            for key, val in time_stats.iteritems():
                if len(val):
                    output('%10s: %7.2f [s]' % (key, val[-1]))

            it = it + 1

        output('status:               %d' % ret)
        output('initial value:        %.8e' % of0)
        output('current value:        %.8e' % of)
        output('iterations:           %d' % it)
        output('function evaluations: %d in %.2f [s]' %
               (nc_of[0], nm.sum(time_stats['of'])))
        output('gradient evaluations: %d in %.2f [s]' %
               (nc_ofg[0], nm.sum(time_stats['ofg'])))

        if self.log is not None:
            self.log(of, ofg_norm, alpha, it)

            if conf.log.plot is not None:
                self.log(save_figure=conf.log.plot, finished=True)

            else:
                self.log(finished=True)

        if status is not None:
            status['log'] = self.log
            status['status'] = status
            status['of0'] = of0
            status['of'] = of
            status['it'] = it
            status['nc_of'] = nc_of[0]
            status['nc_ofg'] = nc_ofg[0]
            status['time_stats'] = time_stats

        return xit
Exemple #9
0
def my_fmin_bfgs(f, x0, fprime=None, args=(), gtol=1e-5, norm=Inf,
              epsilon=_epsilon, maxiter=None, full_output=0, disp=1,
              retall=0, callback=None):
    """Minimize a function using the BFGS algorithm.
    
    :Parameters:

      f : the Python function or method to be minimized.
      x0 : ndarray
        the initial guess for the minimizer.

      fprime : a function to compute the gradient of f.
      args : extra arguments to f and fprime.
      gtol : number
        gradient norm must be less than gtol before succesful termination
      norm : number
        order of norm (Inf is max, -Inf is min)
      epsilon : number
        if fprime is approximated use this value for
                 the step size (can be scalar or vector)
      callback : an optional user-supplied function to call after each
                  iteration.  It is called as callback(xk), where xk is the
                  current parameter vector.

    :Returns: (xopt, {fopt, gopt, Hopt, func_calls, grad_calls, warnflag}, <allvecs>)

      xopt : ndarray
        the minimizer of f.

      fopt : number
        the value of f(xopt).
      gopt : ndarray
        the value of f'(xopt).  (Should be near 0)
      Bopt : ndarray
        the value of 1/f''(xopt).  (inverse hessian matrix)
      func_calls : number
        the number of function_calls.
      grad_calls : number
        the number of gradient calls.
      warnflag : integer
                  1 : 'Maximum number of iterations exceeded.'
                  2 : 'Gradient and/or function calls not changing'
      allvecs  :  a list of all iterates  (only returned if retall==1)

    :OtherParameters:

      maxiter : number
        the maximum number of iterations.
      full_output : number
        if non-zero then return fopt, func_calls, grad_calls,
                     and warnflag in addition to xopt.
      disp : number
        print convergence message if non-zero.
      retall : number
        return a list of results at each iteration if non-zero

    :SeeAlso:

      fmin, fmin_powell, fmin_cg,
             fmin_bfgs, fmin_ncg -- multivariate local optimizers
      leastsq -- nonlinear least squares minimizer

      fmin_l_bfgs_b, fmin_tnc,
             fmin_cobyla -- constrained multivariate optimizers

      anneal, brute -- global optimizers

      fminbound, brent, golden, bracket -- local scalar minimizers

      fsolve -- n-dimenstional root-finding

      brentq, brenth, ridder, bisect, newton -- one-dimensional root-finding

      fixed_point -- scalar fixed-point finder
      
    Notes
    
    ----------------------------------

      Optimize the function, f, whose gradient is given by fprime using the
      quasi-Newton method of Broyden, Fletcher, Goldfarb, and Shanno (BFGS)
      See Wright, and Nocedal 'Numerical Optimization', 1999, pg. 198.
      """
    import numpy
    import scipy.optimize.linesearch as linesearch

    x0 = asarray(x0).squeeze()
    if x0.ndim == 0:
        x0.shape = (1,)
    if maxiter is None:
        maxiter = len(x0)*200
    func_calls, f = wrap_function(f, args)
    if fprime is None:
        grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
    else:
        grad_calls, myfprime = wrap_function(fprime, args)
    gfk = myfprime(x0)
    k = 0
    N = len(x0)
    I = numpy.eye(N,dtype=int)
    Hk = I
    old_fval = f(x0)
    old_old_fval = old_fval + 5000
    xk = x0
    if retall:
        allvecs = [x0]
    sk = [2*gtol]
    warnflag = 0
    gnorm = vecnorm(gfk,ord=norm)
    while (gnorm > gtol) and (k < maxiter):
        pk = -numpy.dot(Hk,gfk)
        if False:
            alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
               linesearch.line_search(f,myfprime,xk,pk,gfk,
                                      old_fval,old_old_fval)
            if alpha_k is None:  # line search failed try different one.
                alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
                         line_search(f,myfprime,xk,pk,gfk,
                                     old_fval,old_old_fval)
                if alpha_k is None:

                    # This line search also failed to find a better solution.
                    warnflag = 2
                    break
        else:
            alpha_k = 0.1
        lg.debug("alpha = {0}".format(alpha_k))
        xkp1 = xk + alpha_k * pk #0.3 added by hcm
        print "--------------------------------\npk =", pk
        dump_mat(Hk)
        if retall:
            allvecs.append(xkp1)
        sk = xkp1 - xk
        xk = xkp1
        #if gfkp1 is None:
        gfkp1 = myfprime(xkp1)

        yk = gfkp1 - gfk
        gfk = gfkp1
        if callback is not None:
            #callback(xk)
            xk = callback(xk) # changed to the following line by hcm
        k += 1
        gnorm = vecnorm(gfk,ord=norm)
        if (gnorm <= gtol):
            break

        try: # this was handled in numeric, let it remaines for more safety
            rhok = 1.0 / (numpy.dot(yk,sk))
        except ZeroDivisionError: 
            rhok = 1000.0
            lg.debug("Divide-by-zero encountered: rhok assumed large")
        if isinf(rhok): # this is patch for numpy
            rhok = 1000.0
            lg.debug("Divide-by-zero encountered: rhok assumed large")
        A1 = I - sk[:,numpy.newaxis] * yk[numpy.newaxis,:] * rhok
        A2 = I - yk[:,numpy.newaxis] * sk[numpy.newaxis,:] * rhok
        Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
                 * sk[numpy.newaxis,:]

    if disp or full_output:
        fval = old_fval
    if warnflag == 2:
        if disp:
            print "Warning: Desired error not necessarily achieved due to precision loss"
            print "         Current function value: %f" % fval
            print "         Iterations: %d" % k
            print "         Function evaluations: %d" % func_calls[0]
            print "         Gradient evaluations: %d" % grad_calls[0]

    elif k >= maxiter:
        warnflag = 1
        if disp:
            print "Warning: Maximum number of iterations has been exceeded"
            print "         Current function value: %f" % fval
            print "         Iterations: %d" % k
            print "         Function evaluations: %d" % func_calls[0]
            print "         Gradient evaluations: %d" % grad_calls[0]
    else:
        if disp:
            print "Optimization terminated successfully."
            print "         Current function value: %f" % fval
            print "         Iterations: %d" % k
            print "         Function evaluations: %d" % func_calls[0]
            print "         Gradient evaluations: %d" % grad_calls[0]

    if full_output:
        retlist = xk, fval, gfk, Hk, func_calls[0], grad_calls[0], warnflag
        if retall:
            retlist += (allvecs,)
    else:
        retlist = xk
        if retall:
            retlist = (xk, allvecs)

    return retlist
Exemple #10
0
def Customfmin_bfgs(f, x0, fprime=None, args=(), gtol=1e-5, norm=Inf,
              epsilon= numpy.sqrt(numpy.finfo(float).eps), maxiter=None, full_output=0, disp=1,
              retall=0, callback=None):


        testVar = 0
        x0 = asarray(x0).squeeze()
        if x0.ndim == 0:
            x0.shape = (1,)
        if maxiter is None:
            maxiter = len(x0)*200
        func_calls, f = wrap_function(f, args)
        if fprime is None:
            grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
        else:
            grad_calls, myfprime = wrap_function(fprime, args)
        gfk = myfprime(x0)
        k = 0
        N = len(x0)
        I = numpy.eye(N,dtype=int)
        Hk = I
        old_fval = f(x0)
        old_old_fval = old_fval + 5000
        xk = x0
        if retall:
            allvecs = [x0]
        sk = [2*gtol]
        warnflag = 0
        gnorm = vecnorm(gfk,ord=norm)
        while (gnorm > gtol) and (k < maxiter):
            pk = -numpy.dot(Hk,gfk)
            alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
               linesearch.line_search(f,myfprime,xk,pk,gfk,
                                      old_fval,old_old_fval)
            if alpha_k is None:  # line search failed try different one.
                alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
                         line_search(f,myfprime,xk,pk,gfk,
                                     old_fval,old_old_fval)
                if alpha_k is None:
                    # This line search also failed to find a better solution.
                    warnflag = 2
                    break
            xkp1 = xk + alpha_k * pk
            if retall:
                allvecs.append(xkp1)
            sk = xkp1 - xk
            xk = xkp1
            if gfkp1 is None:
                gfkp1 = myfprime(xkp1)

            yk = gfkp1 - gfk
            gfk = gfkp1
            if callback is not None:
                callback(xk)
            k += 1
            gnorm = vecnorm(gfk,ord=norm)
            if (gnorm <= gtol):
                break

            try: # this was handled in numeric, let it remaines for more safety
                rhok = 1.0 / (numpy.dot(yk,sk))
            except ZeroDivisionError:
                rhok = 1000.0
                print "Divide-by-zero encountered: rhok assumed large"
            if numpy.isinf(rhok): # this is patch for numpy
                rhok = 1000.0
                print "Divide-by-zero encountered: rhok assumed large"
            A1 = I - sk[:,numpy.newaxis] * yk[numpy.newaxis,:] * rhok
            A2 = I - yk[:,numpy.newaxis] * sk[numpy.newaxis,:] * rhok
            Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
                     * sk[numpy.newaxis,:]

        if disp or full_output:
            fval = old_fval
        if warnflag == 2:
            if disp:
                print "Warning: Desired error not necessarily achieved" \
                      "due to precision loss"
                print "         Current function value: %f" % fval
                print "         Iterations: %d" % k
                print "         Function evaluations: %d" % func_calls[0]
                print "         Gradient evaluations: %d" % grad_calls[0]

        elif k >= maxiter:
            warnflag = 1
            if disp:
                print "Warning: Maximum number of iterations has been exceeded"
                print "         Current function value: %f" % fval
                print "         Iterations: %d" % k
                print "         Function evaluations: %d" % func_calls[0]
                print "         Gradient evaluations: %d" % grad_calls[0]
        else:
            if disp:
                print "Optimization terminated successfully."
                print "         Current function value: %f" % fval
                print "         Iterations: %d" % k
                print "         Function evaluations: %d" % func_calls[0]
                print "         Gradient evaluations: %d" % grad_calls[0]

        if full_output:
            retlist = xk, fval, gfk, Hk, func_calls[0], grad_calls[0], warnflag
            if retall:
                retlist += (allvecs,)
        else:
            retlist = xk
            if retall:
                retlist = (xk, allvecs)

        return retlist
Exemple #11
0
def my_fmin_bfgs(f,
                 x0,
                 fprime=None,
                 args=(),
                 gtol=1e-5,
                 norm=Inf,
                 epsilon=_epsilon,
                 maxiter=None,
                 full_output=0,
                 disp=1,
                 retall=0,
                 callback=None):
    """Minimize a function using the BFGS algorithm.
    
    :Parameters:

      f : the Python function or method to be minimized.
      x0 : ndarray
        the initial guess for the minimizer.

      fprime : a function to compute the gradient of f.
      args : extra arguments to f and fprime.
      gtol : number
        gradient norm must be less than gtol before succesful termination
      norm : number
        order of norm (Inf is max, -Inf is min)
      epsilon : number
        if fprime is approximated use this value for
                 the step size (can be scalar or vector)
      callback : an optional user-supplied function to call after each
                  iteration.  It is called as callback(xk), where xk is the
                  current parameter vector.

    :Returns: (xopt, {fopt, gopt, Hopt, func_calls, grad_calls, warnflag}, <allvecs>)

      xopt : ndarray
        the minimizer of f.

      fopt : number
        the value of f(xopt).
      gopt : ndarray
        the value of f'(xopt).  (Should be near 0)
      Bopt : ndarray
        the value of 1/f''(xopt).  (inverse hessian matrix)
      func_calls : number
        the number of function_calls.
      grad_calls : number
        the number of gradient calls.
      warnflag : integer
                  1 : 'Maximum number of iterations exceeded.'
                  2 : 'Gradient and/or function calls not changing'
      allvecs  :  a list of all iterates  (only returned if retall==1)

    :OtherParameters:

      maxiter : number
        the maximum number of iterations.
      full_output : number
        if non-zero then return fopt, func_calls, grad_calls,
                     and warnflag in addition to xopt.
      disp : number
        print convergence message if non-zero.
      retall : number
        return a list of results at each iteration if non-zero

    :SeeAlso:

      fmin, fmin_powell, fmin_cg,
             fmin_bfgs, fmin_ncg -- multivariate local optimizers
      leastsq -- nonlinear least squares minimizer

      fmin_l_bfgs_b, fmin_tnc,
             fmin_cobyla -- constrained multivariate optimizers

      anneal, brute -- global optimizers

      fminbound, brent, golden, bracket -- local scalar minimizers

      fsolve -- n-dimenstional root-finding

      brentq, brenth, ridder, bisect, newton -- one-dimensional root-finding

      fixed_point -- scalar fixed-point finder
      
    Notes
    
    ----------------------------------

      Optimize the function, f, whose gradient is given by fprime using the
      quasi-Newton method of Broyden, Fletcher, Goldfarb, and Shanno (BFGS)
      See Wright, and Nocedal 'Numerical Optimization', 1999, pg. 198.
      """
    import numpy
    import scipy.optimize.linesearch as linesearch

    x0 = asarray(x0).squeeze()
    if x0.ndim == 0:
        x0.shape = (1, )
    if maxiter is None:
        maxiter = len(x0) * 200
    func_calls, f = wrap_function(f, args)
    if fprime is None:
        grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
    else:
        grad_calls, myfprime = wrap_function(fprime, args)
    gfk = myfprime(x0)
    k = 0
    N = len(x0)
    I = numpy.eye(N, dtype=int)
    Hk = I
    old_fval = f(x0)
    old_old_fval = old_fval + 5000
    xk = x0
    if retall:
        allvecs = [x0]
    sk = [2 * gtol]
    warnflag = 0
    gnorm = vecnorm(gfk, ord=norm)
    while (gnorm > gtol) and (k < maxiter):
        pk = -numpy.dot(Hk, gfk)
        if False:
            alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
               linesearch.line_search(f,myfprime,xk,pk,gfk,
                                      old_fval,old_old_fval)
            if alpha_k is None:  # line search failed try different one.
                alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
                         line_search(f,myfprime,xk,pk,gfk,
                                     old_fval,old_old_fval)
                if alpha_k is None:

                    # This line search also failed to find a better solution.
                    warnflag = 2
                    break
        else:
            alpha_k = 0.1
        lg.debug("alpha = {0}".format(alpha_k))
        xkp1 = xk + alpha_k * pk  #0.3 added by hcm
        print "--------------------------------\npk =", pk
        dump_mat(Hk)
        if retall:
            allvecs.append(xkp1)
        sk = xkp1 - xk
        xk = xkp1
        #if gfkp1 is None:
        gfkp1 = myfprime(xkp1)

        yk = gfkp1 - gfk
        gfk = gfkp1
        if callback is not None:
            #callback(xk)
            xk = callback(xk)  # changed to the following line by hcm
        k += 1
        gnorm = vecnorm(gfk, ord=norm)
        if (gnorm <= gtol):
            break

        try:  # this was handled in numeric, let it remaines for more safety
            rhok = 1.0 / (numpy.dot(yk, sk))
        except ZeroDivisionError:
            rhok = 1000.0
            lg.debug("Divide-by-zero encountered: rhok assumed large")
        if isinf(rhok):  # this is patch for numpy
            rhok = 1000.0
            lg.debug("Divide-by-zero encountered: rhok assumed large")
        A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok
        A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok
        Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
                 * sk[numpy.newaxis,:]

    if disp or full_output:
        fval = old_fval
    if warnflag == 2:
        if disp:
            print "Warning: Desired error not necessarily achieved due to precision loss"
            print "         Current function value: %f" % fval
            print "         Iterations: %d" % k
            print "         Function evaluations: %d" % func_calls[0]
            print "         Gradient evaluations: %d" % grad_calls[0]

    elif k >= maxiter:
        warnflag = 1
        if disp:
            print "Warning: Maximum number of iterations has been exceeded"
            print "         Current function value: %f" % fval
            print "         Iterations: %d" % k
            print "         Function evaluations: %d" % func_calls[0]
            print "         Gradient evaluations: %d" % grad_calls[0]
    else:
        if disp:
            print "Optimization terminated successfully."
            print "         Current function value: %f" % fval
            print "         Iterations: %d" % k
            print "         Function evaluations: %d" % func_calls[0]
            print "         Gradient evaluations: %d" % grad_calls[0]

    if full_output:
        retlist = xk, fval, gfk, Hk, func_calls[0], grad_calls[0], warnflag
        if retall:
            retlist += (allvecs, )
    else:
        retlist = xk
        if retall:
            retlist = (xk, allvecs)

    return retlist
def Customfmin_bfgs(f,
                    x0,
                    fprime=None,
                    args=(),
                    gtol=1e-5,
                    norm=Inf,
                    epsilon=numpy.sqrt(numpy.finfo(float).eps),
                    maxiter=None,
                    full_output=0,
                    disp=1,
                    retall=0,
                    callback=None):

    testVar = 0
    x0 = asarray(x0).squeeze()
    if x0.ndim == 0:
        x0.shape = (1, )
    if maxiter is None:
        maxiter = len(x0) * 200
    func_calls, f = wrap_function(f, args)
    if fprime is None:
        grad_calls, myfprime = wrap_function(approx_fprime, (f, epsilon))
    else:
        grad_calls, myfprime = wrap_function(fprime, args)
    gfk = myfprime(x0)
    k = 0
    N = len(x0)
    I = numpy.eye(N, dtype=int)
    Hk = I
    old_fval = f(x0)
    old_old_fval = old_fval + 5000
    xk = x0
    if retall:
        allvecs = [x0]
    sk = [2 * gtol]
    warnflag = 0
    gnorm = vecnorm(gfk, ord=norm)
    while (gnorm > gtol) and (k < maxiter):
        pk = -numpy.dot(Hk, gfk)
        alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
           linesearch.line_search(f,myfprime,xk,pk,gfk,
                                  old_fval,old_old_fval)
        if alpha_k is None:  # line search failed try different one.
            alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
                     line_search(f,myfprime,xk,pk,gfk,
                                 old_fval,old_old_fval)
            if alpha_k is None:
                # This line search also failed to find a better solution.
                warnflag = 2
                break
        xkp1 = xk + alpha_k * pk
        if retall:
            allvecs.append(xkp1)
        sk = xkp1 - xk
        xk = xkp1
        if gfkp1 is None:
            gfkp1 = myfprime(xkp1)

        yk = gfkp1 - gfk
        gfk = gfkp1
        if callback is not None:
            callback(xk)
        k += 1
        gnorm = vecnorm(gfk, ord=norm)
        if (gnorm <= gtol):
            break

        try:  # this was handled in numeric, let it remaines for more safety
            rhok = 1.0 / (numpy.dot(yk, sk))
        except ZeroDivisionError:
            rhok = 1000.0
            print("Divide-by-zero encountered: rhok assumed large")
        if numpy.isinf(rhok):  # this is patch for numpy
            rhok = 1000.0
            print("Divide-by-zero encountered: rhok assumed large")
        A1 = I - sk[:, numpy.newaxis] * yk[numpy.newaxis, :] * rhok
        A2 = I - yk[:, numpy.newaxis] * sk[numpy.newaxis, :] * rhok
        Hk = numpy.dot(A1,numpy.dot(Hk,A2)) + rhok * sk[:,numpy.newaxis] \
                 * sk[numpy.newaxis,:]

    if disp or full_output:
        fval = old_fval
    if warnflag == 2:
        if disp:
            print("Warning: Desired error not necessarily achieved" \
                  "due to precision loss")
            print("         Current function value: %f" % fval)
            print("         Iterations: %d" % k)
            print("         Function evaluations: %d" % func_calls[0])
            print("         Gradient evaluations: %d" % grad_calls[0])

    elif k >= maxiter:
        warnflag = 1
        if disp:
            print("Warning: Maximum number of iterations has been exceeded")
            print("         Current function value: %f" % fval)
            print("         Iterations: %d" % k)
            print("         Function evaluations: %d" % func_calls[0])
            print("         Gradient evaluations: %d" % grad_calls[0])
    else:
        if disp:
            print("Optimization terminated successfully.")
            print("         Current function value: %f" % fval)
            print("         Iterations: %d" % k)
            print("         Function evaluations: %d" % func_calls[0])
            print("         Gradient evaluations: %d" % grad_calls[0])

    if full_output:
        retlist = xk, fval, gfk, Hk, func_calls[0], grad_calls[0], warnflag
        if retall:
            retlist += (allvecs, )
    else:
        retlist = xk
        if retall:
            retlist = (xk, allvecs)

    return retlist
Exemple #13
0
    def __call__(self, x0, conf=None, obj_fun=None, obj_fun_grad=None, status=None, obj_args=None):
        #    def fmin_sd( conf, x0, fn_of, fn_ofg, args = () ):

        conf = get_default(conf, self.conf)
        obj_fun = get_default(obj_fun, self.obj_fun)
        obj_fun_grad = get_default(obj_fun_grad, self.obj_fun_grad)
        status = get_default(status, self.status)
        obj_args = get_default(obj_args, self.obj_args)

        if conf.output:
            globals()["output"] = conf.output

        output("entering optimization loop...")

        nc_of, tt_of, fn_of = wrap_function(obj_fun, obj_args)
        nc_ofg, tt_ofg, fn_ofg = wrap_function(obj_fun_grad, obj_args)

        time_stats = {"of": tt_of, "ofg": tt_ofg, "check": []}

        if conf.log:
            log = Log.from_conf(conf, ([r"of"], [r"$||$ofg$||$"], [r"alpha"]))
        else:
            log = None

        ofg = None

        it = 0
        xit = x0.copy()
        while 1:

            of = fn_of(xit)

            if it == 0:
                of0 = ofit0 = of_prev = of
                of_prev_prev = of + 5000.0

            if ofg is None:
                #            ofg = 1
                ofg = fn_ofg(xit)

            if conf.check:
                tt = time.clock()
                check_gradient(xit, ofg, fn_of, conf.delta, conf.check)
                time_stats["check"].append(time.clock() - tt)

            ofg_norm = nla.norm(ofg, conf.norm)

            ret = conv_test(conf, it, of, ofit0, ofg_norm)
            if ret >= 0:
                break
            ofit0 = of

            ##
            # Backtrack (on errors).
            alpha = conf.ls0
            can_ls = True
            while 1:
                xit2 = xit - alpha * ofg
                aux = fn_of(xit2)
                if aux is None:
                    alpha *= conf.ls_red_warp
                    can_ls = False
                    output("warp: reducing step (%f)" % alpha)
                elif conf.ls and conf.ls_method == "backtracking":
                    if aux < of * conf.ls_on:
                        break
                    alpha *= conf.ls_red
                    output("backtracking: reducing step (%f)" % alpha)
                else:
                    of_prev_prev = of_prev
                    of_prev = aux
                    break

                if alpha < conf.ls_min:
                    if aux is None:
                        raise RuntimeError, "giving up..."
                    output("linesearch failed, continuing anyway")
                    break

            # These values are modified by the line search, even if it fails
            of_prev_bak = of_prev
            of_prev_prev_bak = of_prev_prev

            if conf.ls and can_ls and conf.ls_method == "full":
                output("full linesearch...")
                alpha, fc, gc, of_prev, of_prev_prev, ofg1 = linesearch.line_search(
                    fn_of, fn_ofg, xit, -ofg, ofg, of_prev, of_prev_prev, c2=0.4
                )
                if alpha is None:  # line search failed -- use different one.
                    alpha, fc, gc, of_prev, of_prev_prev, ofg1 = sopt.line_search(
                        fn_of, fn_ofg, xit, -ofg, ofg, of_prev_bak, of_prev_prev_bak
                    )
                    if alpha is None or alpha == 0:
                        # This line search also failed to find a better solution.
                        ret = 3
                        break
                output(" -> alpha: %.8e" % alpha)
            else:
                if conf.ls_method == "full":
                    output("full linesearch off (%s and %s)" % (conf.ls, can_ls))
                ofg1 = None

            if conf.log:
                log(of, ofg_norm, alpha)

            xit = xit - alpha * ofg
            if ofg1 is None:
                ofg = None
            else:
                ofg = ofg1.copy()

            for key, val in time_stats.iteritems():
                if len(val):
                    output("%10s: %7.2f [s]" % (key, val[-1]))

            it = it + 1

        output("status:               %d" % ret)
        output("initial value:        %.8e" % of0)
        output("current value:        %.8e" % of)
        output("iterations:           %d" % it)
        output("function evaluations: %d in %.2f [s]" % (nc_of[0], nm.sum(time_stats["of"])))
        output("gradient evaluations: %d in %.2f [s]" % (nc_ofg[0], nm.sum(time_stats["ofg"])))

        if conf.log:
            log(of, ofg_norm, alpha, finished=True)

        if status is not None:
            status["log"] = log
            status["status"] = status
            status["of0"] = of0
            status["of"] = of
            status["it"] = it
            status["nc_of"] = nc_of[0]
            status["nc_ofg"] = nc_ofg[0]
            status["time_stats"] = time_stats

        return xit