def sqp(func,
grad=None,
hessian=None,
x0=None,
lb=None,
ub=None,
G=None,
h=None,
A=None,
b=None,
maxiter=100,
method='trust',
disp=0,
full_output=False):
if method.lower() == 'trust' or method.lower() == 'line':
pass
else:
raise Exception("Input method not recognized")
z, G, h, y, A, b = _setup(lb, ub, G, h, A, b)
x = _checkInitialValue(x0, G, h, A, b)
p = len(x)
if hessian is None:
approxH = BFGS
if grad is None:
def finiteForward(x, func, p):
def finiteForward1(x):
return forward(func, x.ravel())
return finiteForward1
grad = finiteForward(x, func, p)
g = numpy.zeros((p, 1))
H = numpy.zeros((p, p))
oldFx = numpy.inf
oldOldFx = numpy.inf
oldGrad = None
update = True
deltaX = numpy.zeros((p, 1))
fx = func(x)
dispObj = Disp(disp)
i = 0
innerI = 0
step = 1.0
radius = 1.0
if hessian is None:
H = numpy.eye(len(x))
while maxiter >= i:
g[:] = grad(x.ravel()).reshape(p, 1)
if hessian is None:
if oldGrad is not None:
if update: # update is always true for line search
diffG = (g - oldGrad).ravel()
H = approxH(H, diffG, step * deltaX.ravel())
else:
H = hessian(x.ravel())
if method == 'trust':
if hessian is None:
# we assume that the hessian is always a PSD
x, update, radius, deltaX, z, y, fx, oldFx, oldGrad, innerIter = _updateTrustRegion(
x, fx, oldFx, deltaX, p, radius, g, oldGrad, H, func, grad,
z, G, h, y, A, b)
else:
x, update, radius, deltaX, z, y, fx, oldFx, oldGrad, innerIter = _updateTrustRegionSOCP(
x, fx, oldFx, deltaX, p, radius, g, oldGrad, H, func, grad,
z, G, h, y, A, b)
else:
x, deltaX, z, y, fx, oldFx, oldOldFx, oldGrad, step, innerIter = _updateLineSearch(
x, fx, oldFx, oldOldFx, deltaX, g, H, func, grad, z, G, h, y,
A, b)
innerI += innerIter
# print qpOut
# print "b Temp"
# print bTemp
# print "b"
# print b - A.dot(x)
i += 1
dispObj.d(i, x, fx, deltaX.ravel(), g.ravel(), radius)
# print "s"
# print h - G.dot(x)
# print "z"
# print numpy.array(qpOut['z']).ravel()
if sufficientNewtonDecrement(deltaX.ravel(), g.ravel()):
break
if abs(fx - oldFx) <= EPSILON:
break
# TODO: full_output- dual variables
if full_output:
output = dict()
output['H'] = H
output['g'] = g.flatten()
output['fx'] = fx
output['iter'] = i
output['innerIter'] = innerI
if G is not None:
output['z'] = z.flatten()
output['s'] = (h - G.dot(x)).flatten()
if A is not None:
output['y'] = y.flatten()
return x, output
else:
return x
def ipBar(func,
grad,
hessian=None,
x0=None,
lb=None,
ub=None,
G=None,
h=None,
A=None,
b=None,
maxiter=100,
disp=0,
full_output=False):
z, G, h, y, A, b = _setup(lb, ub, G, h, A, b)
x = _checkInitialValue(x0, G, h, A, b)
p = len(x)
if hessian is None:
approxH = BFGS
elif type(hessian) is str:
if hessian.lower() == 'bfgs':
approxH = BFGS
elif hessian.lower() == 'sr1':
approxH = SR1
elif hessian.lower() == 'dfp':
approxH = DFP
else:
raise Exception("Input name of hessian is not recognizable")
hessian = None
if grad is None:
def finiteForward(x, func, p):
def finiteForward1(x):
return forward(func, x.ravel())
return finiteForward1
grad = finiteForward(x, func, p)
if G is not None:
m = G.shape[0]
else:
m = 1
fx = None
oldFx = None
oldOldFx = None
oldGrad = None
deltaX = numpy.zeros((p, 1))
g = numpy.zeros((p, 1))
H = numpy.zeros((p, p))
Haug = numpy.zeros((p, p))
dispObj = Disp(disp)
i = 0
t = 0.01
mu = 20.0
step0 = 1.0 # back tracking search step maximum value
step = 0.0
j = 0
while maxiter >= j:
oldFx = numpy.inf
# define the barrier function given t. Note that
# t is adjusted at each outer iteration
barrierFunc = _logBarrier(func, t, G, h)
if j == 0:
fx = barrierFunc(x)
#print "barrier = " +str(fx)
update = True
#while (abs(fx-oldFx)/fx)>=rtol and abs(fx-oldFx)>=atol:
# for i in range(1):
while update:
# print abs(fx-oldFx)
# print abs(fx-oldFx)/fx
# print fx
# print oldFx
gOrig = grad(x.ravel()).reshape(p, 1)
if hessian is None:
if oldGrad is None:
H = numpy.eye(len(x))
else:
diffG = numpy.array(gOrig - oldGrad).ravel()
H = approxH(H, diffG, step * deltaX.ravel())
else:
H = hessian(x.ravel())
## standard log barrier
if G is not None:
s = h - G.dot(x)
Gs = G / s
s2 = s**2
Dphi = Gs.sum(axis=0).reshape(p, 1)
if j == 0:
t = _findInitialBarrier(gOrig, Dphi, A)
# print "initial barrier = " +str(t)
# print "fake barrier = "+str(_findInitialBarrier(gOrig,Dphi,A))
Haug = t * H + numpy.einsum('ji,ik->jk', G.T, G / s2)
g = t * gOrig + Dphi
else:
Haug = t * H
g = t * gOrig
## solving the QP to get the descent direction
if A is not None:
# re-adjust the bounds
bTemp = b - A.dot(x)
LHS = scipy.sparse.bmat([[Haug, A.T], [A, None]], 'csc')
RHS = numpy.append(g, -bTemp, axis=0)
if LHS.size >= (LHS.shape[0] * LHS.shape[1]) / 2:
deltaTemp = scipy.linalg.solve(LHS.todense(),
-RHS).reshape(len(RHS), 1)
else:
deltaTemp = scipy.sparse.linalg.spsolve(LHS, -RHS).reshape(
len(RHS), 1)
deltaX = deltaTemp[:p]
y = deltaTemp[p::]
else:
deltaX = scipy.linalg.solve(Haug, -g)
oldOldFxTemp = oldFx
oldFx = fx
oldGrad = gOrig
lineFunc = lineSearch(x, deltaX, barrierFunc)
#step, fx = exactLineSearch2(step0, lineFunc, deltaX.ravel().dot(g.ravel()), oldFx)
# step, fx = backTrackingLineSearch(step0, lineFunc, deltaX.ravel().dot(g.ravel()), alpha=0.0001,beta=0.8)
barrierGrad = _logBarrierGrad(func, gOrig, t, G, h)
step, fc, gc, fx, oldFx, new_slope = scipy.optimize.line_search(
barrierFunc, barrierGrad, x.ravel(), deltaX.ravel(), g.ravel(),
oldFx, oldOldFx)
# print "fx = " +str(fx)
# print "step= " +str(step)
# if step is not None:
# print "step = "+str(step)+ " with fx" +str(fx)+ " and barrier = " +str(barrierFunc(x + step * deltaX))
# print "s"
# print h - G.dot(x + step * deltaX)
if step is None:
# step, fx = exactLineSearch2(step0, lineFunc, deltaX.ravel().dot(g.ravel()), oldFx)
step, fx = backTrackingLineSearch(step0,
lineFunc,
deltaX.ravel().dot(
g.ravel()),
alpha=0.0001,
beta=0.8)
# print "fail wolfe = " +str(step)+ " maxStep = " +str(step0)
# print "fx = " +str(fx)
# print "step= " +str(step)
update = False
oldOldFx = oldOldFxTemp
x += step * deltaX
# print "stepped func = "+str(func(x))
j += 1
# dispObj.d(j, x.ravel() , fx, deltaX.ravel(), g.ravel(), step)
dispObj.d(j, x.ravel(), func(x.ravel()), deltaX.ravel(), g.ravel(),
step)
# end of inner iteration
i += 1
# obtain the missing Lagrangian multiplier
if G is not None:
s = h - G.dot(x)
z = 1.0 / (t * s)
if m / t < atol:
if sufficientNewtonDecrement(deltaX.ravel(), g.ravel()):
break
else:
t *= mu
# print scipy.linalg.norm(_rDualFunc(x, grad, z, G, y, A))
if scipy.linalg.norm(_rDualFunc(x, grad, z, G, y, A)) <= EPSILON:
break
# end of outer iteration
# TODO: full_output- dual variables
if full_output:
output = dict()
output['t'] = t
output['outerIter'] = i
output['innerIter'] = j
if G is not None:
s = h - G.dot(x)
z = 1.0 / (t * s)
output['s'] = s.ravel()
output['z'] = z.ravel()
if A is not None:
y = y / t
output['y'] = y.ravel()
gap = _dualityGap(func, x, z, G, h, y, A, b)
output['subopt'] = m / t
output['dgap'] = gap
output['fx'] = func(x)
output['H'] = H
output['g'] = gOrig.ravel()
output['rDual'] = _rDualFunc(x, grad, z, G, y, A)
return x.ravel(), output
else:
return x.ravel()
def sqp(func, grad=None, hessian=None, x0=None,
lb=None, ub=None,
G=None, h=None,
A=None, b=None,
maxiter=100,
method='trust',
disp=0, full_output=False):
if method.lower()=='trust' or method.lower()=='line':
pass
else:
raise Exception("Input method not recognized")
z, G, h, y, A, b = _setup(lb, ub, G, h, A, b)
x = _checkInitialValue(x0, G, h, A, b)
p = len(x)
if hessian is None:
approxH = BFGS
if grad is None:
def finiteForward(x,func,p):
def finiteForward1(x):
return forward(func,x.ravel())
return finiteForward1
grad = finiteForward(x,func,p)
g = numpy.zeros((p,1))
H = numpy.zeros((p,p))
oldFx = numpy.inf
oldOldFx = numpy.inf
oldGrad = None
update = True
deltaX = numpy.zeros((p,1))
fx = func(x)
dispObj = Disp(disp)
i = 0
innerI = 0
step = 1.0
radius = 1.0
if hessian is None:
H = numpy.eye(len(x))
while maxiter>=i:
g[:] = grad(x.ravel()).reshape(p,1)
if hessian is None:
if oldGrad is not None:
if update: # update is always true for line search
diffG = (g - oldGrad).ravel()
H = approxH(H, diffG, step * deltaX.ravel())
else:
H = hessian(x.ravel())
if method=='trust':
if hessian is None:
# we assume that the hessian is always a PSD
x, update, radius, deltaX, z, y, fx, oldFx, oldGrad, innerIter = _updateTrustRegion(x, fx, oldFx, deltaX, p, radius, g, oldGrad, H, func, grad, z, G, h, y, A, b)
else:
x, update, radius, deltaX, z, y, fx, oldFx, oldGrad, innerIter = _updateTrustRegionSOCP(x, fx, oldFx, deltaX, p, radius, g, oldGrad, H, func, grad, z, G, h, y, A, b)
else:
x, deltaX, z, y, fx, oldFx, oldOldFx, oldGrad, step, innerIter = _updateLineSearch(x, fx, oldFx, oldOldFx, deltaX, g, H, func, grad, z, G, h, y, A, b)
innerI += innerIter
# print qpOut
# print "b Temp"
# print bTemp
# print "b"
# print b - A.dot(x)
i += 1
dispObj.d(i, x , fx, deltaX.ravel(), g.ravel(), radius)
# print "s"
# print h - G.dot(x)
# print "z"
# print numpy.array(qpOut['z']).ravel()
if sufficientNewtonDecrement(deltaX.ravel(),g.ravel()):
break
if abs(fx-oldFx)<=EPSILON:
break
# TODO: full_output- dual variables
if full_output:
output = dict()
output['H'] = H
output['g'] = g.flatten()
output['fx'] = fx
output['iter'] = i
output['innerIter'] = innerI
if G is not None:
output['z'] = z.flatten()
output['s'] = (h - G.dot(x)).flatten()
if A is not None:
output['y'] = y.flatten()
return x, output
else:
return x
def ipBar(func, grad, hessian=None, x0=None,
lb=None, ub=None,
G=None, h=None,
A=None, b=None,
maxiter=100,
disp=0, full_output=False):
z, G, h, y, A, b = _setup(lb, ub, G, h, A, b)
x = _checkInitialValue(x0, G, h, A, b)
p = len(x)
if hessian is None:
approxH = BFGS
elif type(hessian) is str:
if hessian.lower()=='bfgs':
approxH = BFGS
elif hessian.lower()=='sr1':
approxH = SR1
elif hessian.lower()=='dfp':
approxH = DFP
else:
raise Exception("Input name of hessian is not recognizable")
hessian = None
if grad is None:
def finiteForward(x,func,p):
def finiteForward1(x):
return forward(func,x.ravel())
return finiteForward1
grad = finiteForward(x,func,p)
if G is not None:
m = G.shape[0]
else:
m = 1
fx = None
oldFx = None
oldOldFx = None
oldGrad = None
deltaX = numpy.zeros((p,1))
g = numpy.zeros((p,1))
H = numpy.zeros((p,p))
Haug = numpy.zeros((p,p))
dispObj = Disp(disp)
i = 0
t = 0.01
mu = 20.0
step0 = 1.0 # back tracking search step maximum value
step = 0.0
j = 0
while maxiter>=j:
oldFx = numpy.inf
# define the barrier function given t. Note that
# t is adjusted at each outer iteration
barrierFunc = _logBarrier(func, t, G, h)
if j==0:
fx = barrierFunc(x)
#print "barrier = " +str(fx)
update = True
#while (abs(fx-oldFx)/fx)>=rtol and abs(fx-oldFx)>=atol:
# for i in range(1):
while update:
# print abs(fx-oldFx)
# print abs(fx-oldFx)/fx
# print fx
# print oldFx
gOrig = grad(x.ravel()).reshape(p,1)
if hessian is None:
if oldGrad is None:
H = numpy.eye(len(x))
else:
diffG = numpy.array(gOrig - oldGrad).ravel()
H = approxH(H, diffG, step * deltaX.ravel())
else:
H = hessian(x.ravel())
## standard log barrier
if G is not None:
s = h - G.dot(x)
Gs = G/s
s2 = s**2
Dphi = Gs.sum(axis=0).reshape(p,1)
if j==0:
t = _findInitialBarrier(gOrig,Dphi,A)
# print "initial barrier = " +str(t)
# print "fake barrier = "+str(_findInitialBarrier(gOrig,Dphi,A))
Haug = t*H + numpy.einsum('ji,ik->jk',G.T, G/s2)
g = t*gOrig + Dphi
else:
Haug = t*H
g = t*gOrig
## solving the QP to get the descent direction
if A is not None:
# re-adjust the bounds
bTemp = b - A.dot(x)
LHS = scipy.sparse.bmat([
[Haug,A.T],
[A,None]
], 'csc')
RHS = numpy.append(g,-bTemp,axis=0)
if LHS.size>= (LHS.shape[0] * LHS.shape[1])/2:
deltaTemp = scipy.linalg.solve(LHS.todense(),-RHS).reshape(len(RHS),1)
else:
deltaTemp = scipy.sparse.linalg.spsolve(LHS,-RHS).reshape(len(RHS),1)
deltaX = deltaTemp[:p]
y = deltaTemp[p::]
else:
deltaX = scipy.linalg.solve(Haug,-g)
oldOldFxTemp = oldFx
oldFx = fx
oldGrad = gOrig
lineFunc = lineSearch(x, deltaX, barrierFunc)
#step, fx = exactLineSearch2(step0, lineFunc, deltaX.ravel().dot(g.ravel()), oldFx)
# step, fx = backTrackingLineSearch(step0, lineFunc, deltaX.ravel().dot(g.ravel()), alpha=0.0001,beta=0.8)
barrierGrad = _logBarrierGrad(func, gOrig, t, G, h)
step, fc, gc, fx, oldFx, new_slope = scipy.optimize.line_search(barrierFunc,
barrierGrad,
x.ravel(),
deltaX.ravel(),
g.ravel(),
oldFx,
oldOldFx
)
# print "fx = " +str(fx)
# print "step= " +str(step)
# if step is not None:
# print "step = "+str(step)+ " with fx" +str(fx)+ " and barrier = " +str(barrierFunc(x + step * deltaX))
# print "s"
# print h - G.dot(x + step * deltaX)
if step is None:
# step, fx = exactLineSearch2(step0, lineFunc, deltaX.ravel().dot(g.ravel()), oldFx)
step, fx = backTrackingLineSearch(step0, lineFunc, deltaX.ravel().dot(g.ravel()), alpha=0.0001,beta=0.8)
# print "fail wolfe = " +str(step)+ " maxStep = " +str(step0)
# print "fx = " +str(fx)
# print "step= " +str(step)
update = False
oldOldFx = oldOldFxTemp
x += step * deltaX
# print "stepped func = "+str(func(x))
j += 1
# dispObj.d(j, x.ravel() , fx, deltaX.ravel(), g.ravel(), step)
dispObj.d(j, x.ravel() , func(x.ravel()), deltaX.ravel(), g.ravel(), step)
# end of inner iteration
i += 1
# obtain the missing Lagrangian multiplier
if G is not None:
s = h - G.dot(x)
z = 1.0 / (t * s)
if m/t < atol:
if sufficientNewtonDecrement(deltaX.ravel(),g.ravel()):
break
else:
t *= mu
# print scipy.linalg.norm(_rDualFunc(x, grad, z, G, y, A))
if scipy.linalg.norm(_rDualFunc(x, grad, z, G, y, A))<=EPSILON:
break
# end of outer iteration
# TODO: full_output- dual variables
if full_output:
output = dict()
output['t'] = t
output['outerIter'] = i
output['innerIter'] = j
if G is not None:
s = h - G.dot(x)
z = 1.0 / (t * s)
output['s'] = s.ravel()
output['z'] = z.ravel()
if A is not None:
y = y/t
output['y'] = y.ravel()
gap = _dualityGap(func, x,
z, G, h,
y, A, b)
output['subopt'] = m/t
output['dgap'] = gap
output['fx'] = func(x)
output['H'] = H
output['g'] = gOrig.ravel()
output['rDual'] = _rDualFunc(x, grad, z, G, y, A)
return x.ravel(), output
else:
return x.ravel()