def __init__(self, nlp, **kwargs):
self.nlp = nlp
self.npairs = kwargs.get('npairs', 5)
self.silent = kwargs.get('silent', False)
self.abstol = kwargs.get('abstol', 1.0e-6)
self.reltol = kwargs.get('reltol', self.nlp.stop_d)
self.iter = 0
self.nresets = 0
self.converged = False
self.lbfgs = InverseLBFGS(self.nlp.n, **kwargs)
# Code for testingLBFGS
self.alt_lbfgs = LBFGS(self.nlp.n, **kwargs)
self.x = kwargs.get('x0', self.nlp.x0)
self.f = self.nlp.obj(self.x)
self.g = self.nlp.grad(self.x)
self.gnorm = norms.norm2(self.g)
self.f0 = self.f
self.g0 = self.gnorm
# Optional arguments
self.maxiter = kwargs.get('maxiter', max(10*self.nlp.n, 1000))
self.tsolve = 0.0
def solve(self):
tstart = cputime()
# Initial LBFGS matrix is the identity. In other words,
# the initial search direction is the steepest descent direction.
# This is the original L-BFGS stopping condition.
#stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x))
stoptol = max(self.abstol, self.reltol * self.g0)
while self.gnorm > stoptol and self.iter < self.maxiter:
if not self.silent:
print '%-5d %-12g %-12g' % (self.iter, self.f, self.gnorm)
# Obtain search direction
d = self.lbfgs.matvec(-self.g)
# Prepare for modified More-Thuente linesearch
if self.iter == 0:
stp0 = 1.0/self.gnorm
else:
stp0 = 1.0
SWLS = StrongWolfeLineSearch(self.f,
self.x,
self.g,
d,
lambda z: self.nlp.obj(z),
lambda z: self.nlp.grad(z),
stp = stp0)
# Perform linesearch
SWLS.search()
# SWLS.x contains the new iterate
# SWLS.g contains the objective gradient at the new iterate
# SWLS.f contains the objective value at the new iterate
s = SWLS.x - self.x
self.x = SWLS.x
y = SWLS.g - self.g
self.g = SWLS.g
self.gnorm = norms.norm2(self.g)
self.f = SWLS.f
#stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x))
# Update inverse Hessian approximation using the most recent pair
self.lbfgs.store(s, y)
self.iter += 1
# Code for testing the LBFGS implementation
self.alt_lbfgs.store(s, y)
self.tsolve = cputime() - tstart
self.converged = (self.iter < self.maxiter)
def solve(self):
tstart = cputime()
# Initial LBFGS matrix is the identity. In other words,
# the initial search direction is the steepest descent direction.
# This is the original L-BFGS stopping condition.
#stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x))
stoptol = max(self.abstol, self.reltol * self.g0)
while self.gnorm > stoptol and self.iter < self.maxiter:
if not self.silent:
print '%-5d %-12g %-12g' % (self.iter, self.f, self.gnorm)
# Obtain search direction
d = self.lbfgs.matvec(-self.g)
# Prepare for modified More-Thuente linesearch
if self.iter == 0:
stp0 = 1.0 / self.gnorm
else:
stp0 = 1.0
SWLS = StrongWolfeLineSearch(self.f,
self.x,
self.g,
d,
lambda z: self.nlp.obj(z),
lambda z: self.nlp.grad(z),
stp=stp0)
# Perform linesearch
SWLS.search()
# SWLS.x contains the new iterate
# SWLS.g contains the objective gradient at the new iterate
# SWLS.f contains the objective value at the new iterate
s = SWLS.x - self.x
self.x = SWLS.x
y = SWLS.g - self.g
self.g = SWLS.g
self.gnorm = norms.norm2(self.g)
self.f = SWLS.f
#stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x))
# Update inverse Hessian approximation using the most recent pair
self.lbfgs.store(s, y)
self.iter += 1
self.tsolve = cputime() - tstart
self.converged = (self.iter < self.maxiter)
def solveSystem(self, rhs, itref_threshold=1.0e-5, nitrefmax=5):
"""
Solve the augmented system with right-hand side `rhs` and optionally
perform iterative refinement.
Return the solution vector (as a reference), the 2-norm of the residual
and the number of negative eigenvalues of the coefficient matrix.
"""
self.log.debug('Solving linear system')
self.LBL.solve(rhs)
self.LBL.refine(rhs, tol=itref_threshold, nitref=nitrefmax)
# Collect statistics on the linear system solve.
self.cond_history.append((self.LBL.cond, self.LBL.cond2))
self.berr_history.append((self.LBL.berr, self.LBL.berr2))
self.derr_history.append(self.LBL.dirError)
self.nrms_history.append((self.LBL.matNorm, self.LBL.xNorm))
self.lres_history.append(self.LBL.relRes)
nr = norm2(self.LBL.residual)
return (self.LBL.x, nr, self.LBL.neig)
def solveSystem(self, rhs, itref_threshold=1.0e-5, nitrefmax=5):
"""
Solve the augmented system with right-hand side `rhs` and optionally
perform iterative refinement.
Return the solution vector (as a reference), the 2-norm of the residual
and the number of negative eigenvalues of the coefficient matrix.
"""
self.log.debug('Solving linear system')
self.LBL.solve(rhs)
self.LBL.refine(rhs, tol=itref_threshold, nitref=nitrefmax)
# Collect statistics on the linear system solve.
self.cond_history.append((self.LBL.cond, self.LBL.cond2))
self.berr_history.append((self.LBL.berr, self.LBL.berr2))
self.derr_history.append(self.LBL.dirError)
self.nrms_history.append((self.LBL.matNorm, self.LBL.xNorm))
self.lres_history.append(self.LBL.relRes)
nr = norm2(self.LBL.residual)
return (self.LBL.x, nr, self.LBL.neig)
def SolveSystem(A, rhs, itref_threshold=1.0e-6, nitrefmax=5, **kwargs):
# Obtain Sils context object
t = cputime()
LBL = LBLContext(A, **kwargs)
t_analyze = cputime() - t
# Solve system and compute residual
t = cputime()
LBL.solve(rhs)
t_solve = cputime() - t_analyze
# Compute residual norm
nrhsp1 = norms.norm_infty(rhs) + 1
nr = norms.norm2(LBL.residual)/nrhsp1
# If residual not small, perform iterative refinement
LBL.refine(rhs, tol = itref_threshold, nitref=nitrefmax)
nr1 = norms.norm_infty(LBL.residual)/nrhsp1
return (LBL.x, LBL.residual, nr, nr1, t_analyze, t_solve, LBL.neig)
def __init__(self, nlp, **kwargs):
self.nlp = nlp
self.npairs = kwargs.get('npairs', 5)
self.silent = kwargs.get('silent', False)
self.abstol = kwargs.get('abstol', 1.0e-6)
self.reltol = kwargs.get('reltol', self.nlp.stop_d)
self.iter = 0
self.nresets = 0
self.converged = False
self.lbfgs = InverseLBFGS(self.nlp.n, **kwargs)
self.x = kwargs.get('x0', self.nlp.x0)
self.f = self.nlp.obj(self.x)
self.g = self.nlp.grad(self.x)
self.gnorm = norms.norm2(self.g)
self.f0 = self.f
self.g0 = self.gnorm
# Optional arguments
self.maxiter = kwargs.get('maxiter', max(10 * self.nlp.n, 1000))
self.tsolve = 0.0
def __init__(self, lp, **kwargs):
"""
Solve a linear program of the form::
minimize c' x subject to A1 x + A2 s = b and s >= 0, (LP)
where the variables x are the original problem variables and s are
slack variables. Any linear program may be converted to the above form
by instantiation of the `SlackFramework` class. The conversion to the
slack formulation is mandatory in this implementation.
The method is a variant of Mehrotra's predictor-corrector method where
steps are computed by solving the primal-dual system in augmented form.
Primal and dual regularization parameters may be specified by the user
via the opional keyword arguments `regpr` and `regdu`. Both should be
positive real numbers and should not be "too large". By default they are
set to 1.0 and updated at each iteration.
If `scale` is set to `True`, (LP) is scaled automatically prior to
solution so as to equilibrate the rows and columns of the constraint
matrix [A1 A2].
Advantages of this method are that it is not sensitive to dense columns
in A, no special treatment of the unbounded variables x is required, and
a sparse symmetric quasi-definite system of equations is solved at each
iteration. The latter, although indefinite, possesses a Cholesky-like
factorization. Those properties makes the method typically more robust
that a standard predictor-corrector implementation and the linear system
solves are often much faster than in a traditional interior-point method
in augmented form.
:keywords:
:scale: Perform row and column equilibration of the constraint
matrix [A1 A2] prior to solution (default: `True`).
:stabilize: Scale the linear system to be solved at each iteration
(default: `True`).
:regpr: Initial value of primal regularization parameter
(default: `1.0`).
:regdu: Initial value of dual regularization parameter
(default: `1.0`).
:verbose: Turn on verbose mode (default `False`).
"""
if not isinstance(lp, SlackFramework):
msg = 'Input problem must be an instance of SlackFramework'
raise ValueError, msg
scale = kwargs.get('scale', True)
self.verbose = kwargs.get('verbose', True)
self.stabilize = kwargs.get('stabilize', True)
self.lp = lp
self.A = lp.A() # Constraint matrix
if not isinstance(self.A, PysparseMatrix):
self.A = PysparseMatrix(matrix=self.A)
m, n = self.A.shape
# Record number of slack variables in LP
self.nSlacks = lp.n - lp.original_n
# Constant vectors
zero = np.zeros(n)
self.b = -lp.cons(zero) # Right-hand side
self.c0 = lp.obj(zero) # Constant term in objective
self.c = lp.grad(zero[:lp.original_n]) #lp.cost() # Cost vector
# Apply in-place problem scaling if requested.
self.prob_scaled = False
if scale:
self.t_scale = cputime()
self.scale()
self.t_scale = cputime() - self.t_scale
self.normb = norm2(self.b)
self.normc = norm2(self.c)
self.normbc = 1 + max(self.normb, self.normc)
# Initialize augmented matrix
self.H = PysparseMatrix(size=n+m,
sizeHint=n+m+self.A.nnz,
symmetric=True)
# We perform the analyze phase on the augmented system only once.
# self.LBL will be initialized in set_initial_guess().
self.LBL = None
self.regpr = kwargs.get('regpr', 1.0) ; self.regpr_min = 1.0e-8
self.regdu = kwargs.get('regdu', 1.0) ; self.regdu_min = 1.0e-8
# Check input parameters.
if self.regpr < 0.0: self.regpr = 0.0
if self.regdu < 0.0: self.regdu = 0.0
# Dual regularization is necessary for stabilization.
if self.regdu == 0.0:
sys.stderr.write('Warning: No dual regularization in effect\n')
sys.stderr.write(' Stabilization has been turned off\n')
self.stabilize = False
# Initialize format strings for display
fmt_hdr = '%-4s %9s' + ' %-8s'*6 + ' %-7s %-4s %-4s' + ' %-8s'*8
self.header = fmt_hdr % ('Iter', 'Cost', 'pResid', 'dResid', 'cResid',
'rGap', 'qNorm', 'rNorm', 'Mu', 'AlPr', 'AlDu',
'LS Resid', 'RegPr', 'RegDu', 'Rho q', 'Del r',
'Min(s)', 'Min(z)', 'Max(s)')
self.format1 = '%-4d %9.2e'
self.format1 += ' %-8.2e' * 6
self.format2 = ' %-7.1e %-4.2f %-4.2f'
self.format2 += ' %-8.2e' * 8 + '\n'
if self.verbose: self.display_stats()
return
# Call projected CG to solve problem
# min <g,d> + 1/2 <d, Hd>
# s.t Jd = 0, |d| <= delta
CG = Ppcg(g,
SimpleLinearOperator(nlp.n,
nlp.n,
lambda p: nlp.hprod(nlp.x0, nlp.pi0, p),
symmetric=True),
A=J,
radius=delta,
debug=True)
#CG = Ppcg(g, A=J, rhs=-c, matvec=hprod, debug=True)
CG.Solve()
Jd = numpy.empty(nlp.m, 'd')
J.matvec(CG.x, Jd)
feas = norms.norm_infty(Jd)
#feas = norms.norm_infty( c + Jd ) # for nonzero rhs constraint
snorm = norms.norm2(CG.x) # The trust-region is in the l2 norm
sys.stdout.write('Number of variables : %-d\n' % nlp.n)
sys.stdout.write('Number of constraints : %-d\n' % nlp.m)
sys.stdout.write('Converged : %-s\n' % repr(CG.converged))
sys.stdout.write('Trust-region radius : %8.1e\n' % delta)
sys.stdout.write('Solution norm : %8.1e\n' % snorm)
sys.stdout.write('Final/Initial Residual : ')
sys.stdout.write('%8.1e / %8.1e\n' % (CG.residNorm, CG.residNorm0))
sys.stdout.write('Feasibility error : %8.1e\n' % feas)
sys.stdout.write('Number of iterations : %-d\n' % CG.iter)
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations (default: 10n)
:tolerance: Stopping tolerance (default: 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default: `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
Upon exit, the following members of the class instance are set:
x..............final iterate
y..............final value of the Lagrange multipliers associated
to A1 x + A2 s = b
z..............final value of the Lagrange multipliers associated
to s>=0
obj_value......final cost
iter...........total number of iterations
kktResid.......final relative residual
solve_time.....time to solve the LP
status.........string describing the exit status
short_status...short version of status, used for printing.
"""
lp = self.lp
itermax = kwargs.get('itermax', max(100,10*lp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape ; on = lp.original_n
A = self.A ; b = self.b ; c = self.c ; H = self.H
regpr = self.regpr ; regdu = self.regdu
regpr_min = self.regpr_min ; regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
# set_initial_guess() initializes self.LBL which is reused below.
(x,y,z) = self.set_initial_guess(self.lp, **kwargs)
# Slack variables are the trailing variables in x.
s = x[on:] ; ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
col_scale = np.empty(n)
# Allocate room for right-hand side of linear systems.
rhs = np.zeros(n+m)
finished = False
iter = 0
# Acceptance thresholds for primal and dual reg parameters.
#t1 = t2 = 0.99
solve_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if self.verbose and iter % 20 == 0:
sys.stdout.write('\n' + self.header + '\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Compute residuals.
pFeas = A*x - b
comp = s*z ; sz = sum(comp) # comp = S z
dFeas = y*A ; dFeas[:on] -= self.c # dFeas1 = A1'y - c
dFeas[on:] += z # dFeas2 = A2'y + z
mu = sz/ns
# Compute residual norms and scaled residual norms.
# We don't need to keep both the scaled and unscaled residuals
# store.
#pResid = norm_infty(pFeas + regdu * r)/(1+self.normc)
#dResid = norm_infty(dFeas - regpr * q)/(1+self.normb)
pResid = norm2(pFeas) ; spResid = pResid/(1+self.normc)
cResid = norm2(comp) ; scResid = cResid/self.normbc
dResid = norm2(dFeas) ; sdResid = dResid/(1+self.normb)
# Compute relative duality gap.
cx = np.dot(c,x[:on])
by = np.dot(b,y)
rgap = cx - by
#rgap += regdu * (rNorm**2 + np.dot(r,y))
rgap = abs(rgap) / (1 + abs(cx))
rgap2 = mu /(1 + abs(cx))
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
#kktResid = max(pResid, cResid, dResid)
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status= 'iter'
finished = True
continue
# Adjust regularization parameters
#mu = sum(comp)/ns
#if mu < 1:
# regpr = sqrt(mu)
# regdu = sqrt(mu)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
if iter == 0:
regpr = self.regpr ; regdu = self.regdu
if regpr > 0:
q = dFeas/regpr ; qNorm = norm2(q) ; rho_q = regpr * qNorm
else:
q = dFeas ; qNorm = norm2(q) ; rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas/regdu ; rNorm = norm2(r) ; del_r = regdu * rNorm
else:
r = -pFeas ; rNorm = norm2(r) ; del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
# Adjust regularization parameters.
#regpr = max(min(regpr/10, regpr**(1.1)), regpr_min)
#regdu = max(min(regdu/10, regdu**(1.1)), regdu_min)
# 1) rho+ |dx| <= const * s'z
# 2) del+ |dy| <= const * s'z
if regdu > 0:
regdu = min(regdu/10, sz/normdy/10, (sz/normdy)**(1.1))
regdu = max(regdu, regdu_min)
if regpr > 0:
regpr = min(regpr/10, sz/normdx/10, (sz/normdx)**(1.1))
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
#if mu < 1.0e-8 * mu0 and rho_q > 1.0e+3 * kktResid * self.normbc: #* mu * self.normbc:
if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter-1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
#if mu < 1.0e-8 * mu0 and del_r > 1.0e+3 * kktResid * self.normbc: # * mu * self.normbc:
if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter-1:
if du_infeas_count > 6:
status='Problem seems to be (locally) primal infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
# Display objective and residual data.
if self.verbose:
sys.stdout.write(self.format1 % (iter, cx, pResid, dResid,
cResid, rgap, qNorm, rNorm))
# Record some quantities for display
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
# Repeatedly assemble system and compute step until primal and
# dual regularization parameters have appropriate values.
# Reset primal and dual regularization parameters to best guess
#if iter > 0:
# regpr = max(regpr_min, 0.5*sigma*dResid/normds)
# regdu = max(regdu_min, 0.5*sigma*pResid/normdy)
step_acceptable = False
while not step_acceptable:
# Solve the linear system
#
# [-pI 0 A1'] [∆x] [c - A1' y ]
# [ 0 -(S^{-1} Z + pI) A2'] [∆s] = [ - A2' y - µ S^{-1} e]
# [ A1 A2 dI ] [∆y] [b - A1 x - A2 s ]
#
# where s are the slack variables, p is the primal
# regularization parameter, d is the dual regularization
# parameter, and A = [ A1 A2 ] where the columns of A1
# correspond to the original problem variables and those of A2
# correspond to slack variables.
#
# We recover ∆z = -z - S^{-1} (Z ∆s + µ e).
# Compute augmented matrix and factorize it.
factorized = False
nb_bump = 0
while not factorized and nb_bump < 5:
if self.stabilize:
col_scale[:on] = sqrt(regpr)
col_scale[on:] = np.sqrt(z/s + regpr)
H.put(-sqrt(regdu), range(n))
H.put( sqrt(regdu), range(n,n+m))
AA = self.A.copy()
AA.col_scale(1/col_scale)
H[n:,:n] = AA
else:
if regpr > 0: H.put(-regpr, range(on))
H.put(-z/s - regpr, range(on,n))
if regdu > 0: H.put(regdu, range(n,n+m))
#if iter == 5:
# # Export current matrix to file for futher inspection.
# import os
# name = os.path.basename(self.lp.name)
# fname = '.'.join(name.split('.')[:-1]) + '.mtx'
# H.exportMmf(fname)
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
sys.stderr.write('Primal-Dual Matrix ')
sys.stderr.write('Rank Deficient')
if regdu == 0.0:
sys.stderr.write('... No regularization in effect')
sys.stderr.write('... bailing out\n')
factorized = False
nb_bump = 5
continue
else:
sys.stderr.write('... bumping up reg parameters\n')
regpr *= 10 ; regdu *= 10
nb_bump += 1
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and nb_bump >= 5:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue # Does this get us out of the outer while?
# Compute duality measure.
mu = sz/ns
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
rhs[:n] = -dFeas
rhs[on:n] += z
rhs[n:] = -pFeas
# if 'stabilize' is on, must scale right-hand side.
if self.stabilize:
rhs[:n] /= col_scale
rhs[n:] /= sqrt(regdu)
(step, nres, neig) = self.solveSystem(rhs)
# Unscale step if 'stabilize' is on.
if self.stabilize:
step[:n] *= sqrt(regdu) / col_scale
# Recover dx and dz.
dx = step[:n]
ds = dx[on:]
dz = -z * (1 + ds/s)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
sigma = (muAff/mu)**3
# Incorporate predictor information for corrector step.
comp += ds*dz
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100*mu)
# Assemble right-hand side with centering information.
comp -= sigma * mu
if PredictorCorrector:
# Only update rhs[on:n]; the rest of rhs did not change.
if self.stabilize:
rhs[on:n] += (comp/s - z)/col_scale[on:n]
else:
rhs[on:n] += comp/s - z
else:
rhs[:n] = -dFeas
rhs[on:n] += comp/s
rhs[n:] = -pFeas
# If 'stabilize' is on, must scale right-hand side.
# In the predictor-corrector method, this has already been
# done.
if self.stabilize:
rhs[:n] /= col_scale
rhs[n:] /= sqrt(regdu)
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Unscale step if 'stabilize' is on.
if self.stabilize:
step[:n] *= sqrt(regdu) / col_scale
# Recover step.
dx = step[:n]
ds = dx[on:]
dy = step[n:]
normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx)
step_acceptable = True # Must get rid of this
# End while not step_acceptable
# Recover step in z.
dz = -(comp + z*ds)/s
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0-mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip)
alpha_p *= max(1-mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid)
alpha_d *= max(1-mult, gamma_d)
if ip==id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
if self.verbose:
sys.stdout.write(self.format2 % (mu, alpha_p, alpha_d,
nres, regpr, regdu, rho_q,
del_r, mins, minz, maxs))
# Update primal variables and slacks.
x += alpha_p * dx
# Update dual variables.
y += alpha_d * dy
z += alpha_d * dz
# Update perturbation vectors.
q *= (1-alpha_p) ; q += alpha_p * dx
r *= (1-alpha_d) ; r += alpha_d * dy
qNorm = norm2(q) ; rNorm = norm2(r)
rho_q = regpr * qNorm/(1+self.normc) ; rho_q_min = min(rho_q_min, rho_q)
del_r = regdu * rNorm/(1+self.normb) ; del_r_min = min(del_r_min, del_r)
iter += 1
solve_time = cputime() - solve_time
if self.verbose:
sys.stdout.write('\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = np.dot(self.c, x[:on]) + self.c0
return
#output_line += format2 % (regqp.mu, regqp.alpha_p, regqp.alpha_d,
#regqp.nres, regqp.regpr, regqp.regdu, regqp.rho_q,
#regqp.del_w, regqp.mins, regqp.minz, regqp.maxs)
#log.info(output_line)
log.info('-' * len(regqp.header))
#sys.stdout.write(fmt % (probname, regqp.iter, regqp.obj_value,
#regqp.pResid, regqp.dResid, regqp.rgap,
#t_setup, regqp.solve_time, regqp.short_status))
#if regqp.short_status == 'degn':
#sys.stdout.write(' F') # Could not regularize sufficiently.
#sys.stdout.write('\n')
x = regqp.x[0:n]
x0 = 0.1*np.ones([1,n])
x0[0] = -0.1
norm_x0_x = norm2(x0-x)
nnz = nnz_elements(x,1e-3)
Probname = str(m)+'--'+str(n)
numpy.set_printoptions(threshold=5)
numpy.set_printoptions(threshold='nan')
print x[0:10]
log.info('difference between initialize point and minimizer %6.f'%norm_x0_x)
x = regqp.x[0:n]
log.info('#Iterations: %-d' % regqp.iter)
log.info('RelResidual: %7.1e' % regqp.kktResid)
log.info('Final cost : %21.15e' % regqp.obj_value)
log.info('Setup time : %6.2fs' % t_setup)
# J.matvec(e, c)
# Call projected CG to solve problem
# min <g,d> + 1/2 <d, Hd>
# s.t Jd = 0, |d| <= delta
CG = Ppcg(
g,
SimpleLinearOperator(nlp.n, nlp.n, lambda p: nlp.hprod(nlp.x0, nlp.pi0, p), symmetric=True),
A=J,
radius=delta,
debug=True,
)
# CG = Ppcg(g, A=J, rhs=-c, matvec=hprod, debug=True)
CG.Solve()
Jd = numpy.empty(nlp.m, "d")
J.matvec(CG.x, Jd)
feas = norms.norm_infty(Jd)
# feas = norms.norm_infty( c + Jd ) # for nonzero rhs constraint
snorm = norms.norm2(CG.x) # The trust-region is in the l2 norm
sys.stdout.write("Number of variables : %-d\n" % nlp.n)
sys.stdout.write("Number of constraints : %-d\n" % nlp.m)
sys.stdout.write("Converged : %-s\n" % repr(CG.converged))
sys.stdout.write("Trust-region radius : %8.1e\n" % delta)
sys.stdout.write("Solution norm : %8.1e\n" % snorm)
sys.stdout.write("Final/Initial Residual : ")
sys.stdout.write("%8.1e / %8.1e\n" % (CG.residNorm, CG.residNorm0))
sys.stdout.write("Feasibility error : %8.1e\n" % feas)
sys.stdout.write("Number of iterations : %-d\n" % CG.iter)
if probname[-3:] == '.nl': probname = probname[:-3]
if not options.verbose:
log.info(fmt % (probname, regqp.iter, regqp.obj_value,
regqp.pResid, regqp.dResid, regqp.rgap,
t_setup, regqp.solve_time, regqp.short_status))
if regqp.short_status == 'degn':
log.info(' F') # Could not regularize sufficiently.
qp.close()
log.info('-'*len(hdr))
if not multiple_problems:
x = regqp.x[:qp.original_n]
log.info('Final x: %s, |x| = %7.1e' % (repr(x),norm2(x)))
log.info('Final y: %s, |y| = %7.1e' % (repr(regqp.y),norm2(regqp.y)))
log.info('Final z: %s, |z| = %7.1e' % (repr(regqp.z),norm2(regqp.z)))
log.info(regqp.status)
log.info('#Iterations: %-d' % regqp.iter)
log.info('RelResidual: %7.1e' % regqp.kktResid)
log.info('Final cost : %21.15e' % regqp.obj_value)
log.info('Setup time : %6.2fs' % t_setup)
log.info('Solve time : %6.2fs' % regqp.solve_time)
# Plot linear system statistics.
import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.gca()
ax.semilogy(regqp.lres_history)
#regqp.dResid, regqp.cResid, regqp.rgap, regqp.qNorm,
#regqp.wNorm)
#output_line += format2 % (regqp.mu, regqp.alpha_p, regqp.alpha_d,
#regqp.nres, regqp.regpr, regqp.regdu, regqp.rho_q,
#regqp.del_w, regqp.mins, regqp.minz, regqp.maxs)
#log.info(output_line)
log.info('-' * len(regqp.header))
#sys.stdout.write(fmt % (probname, regqp.iter, regqp.obj_value,
#regqp.pResid, regqp.dResid, regqp.rgap,
#t_setup, regqp.solve_time, regqp.short_status))
#if regqp.short_status == 'degn':
#sys.stdout.write(' F') # Could not regularize sufficiently.
#sys.stdout.write('\n')
x = regqp.x[0:n]
norm_x0_x = norm2(.1*np.ones([1,n])-x)
nnz = nnz_elements(x,1e-3)
Probname = str(n)+'--'+str(m)
numpy.set_printoptions(threshold=5)
numpy.set_printoptions(threshold='nan')
#print x[1:10]
log.info('Non zero elements in minimizer %6.f'%nnz)
x = regqp.x[0:n]
log.info('#Iterations: %-d' % regqp.iter)
log.info('RelResidual: %7.1e' % regqp.kktResid)
def SolveInner(self, **kwargs):
"""
Perform a series of inner iterations so as to minimize the primal-dual
merit function with the current value of the barrier parameter to within
some given tolerance. The only optional argument recognized is
stopTol stopping tolerance (default: muerrfact * mu).
"""
merit = self.merit
nx = merit.nlp.n
nz = merit.nz
rho = 1 # Dummy initial value for rho
niter = 0 # Dummy initial value for number of inners
status = '' # Dummy initial step status
alpha = 0.0 # Fraction-to-the-boundary step size
if self.inexact:
cgtol = 1.0
else:
cgtol = -1.0
inner_iter = 0 # Inner iteration counter
# Obtain starting point.
(x, z) = (self.x, self.z)
# Obtain first-order data at starting point.
if self.iter == 0:
f = nlp.obj(x)
gf = nlp.grad(x)
else:
f = self.f
gf = self.gf
psi = merit.obj(x, z, f=f)
g = merit.grad(x, z, g=gf, check_optimal=True)
gNorm = norm2(g)
if self.optimal: return
# Reset initial trust-region radius
self.TR.Delta = 0.1 * gNorm
# Set inner iteration stopping tolerance
stopTol = kwargs.get('stopTol', self.muerrfact * self.mu)
finished = (gNorm <= stopTol) or (self.iter >= self.maxiter)
while not finished:
# Print out header every so many iterations
if self.verbose:
if self.iter % self.printFrequency == 0:
sys.stdout.write(self.hline)
sys.stdout.write(self.header)
sys.stdout.write(self.hline)
if inner_iter == 0:
sys.stdout.write(('*' + self.itFormat) % self.iter)
else:
sys.stdout.write((' ' + self.itFormat) % self.iter)
sys.stdout.write(
self.format %
(f,
max(norm_infty(self.dRes), norm_infty(self.cRes),
norm_infty(self.pRes)), gNorm, self.mu, alpha, niter,
rho, self.TR.Delta, status))
# Set stopping tolerance for trust-region subproblem.
if self.inexact:
cgtol = max(1.0e-8, min(0.1 * cgtol, sqrt(gNorm)))
if self.debug: self._debugMsg('cgtol = ' + str(cgtol))
# Update Hessian matrix with current iteration information.
# self.PDHess(x,z)
if self.debug:
self._debugMsg('g = ' + np.str(g))
self._debugMsg('gNorm = ' + str(gNorm))
self._debugMsg('stopTol = ' + str(stopTol))
self._debugMsg('dRes = ' + np.str(self.dRes))
self._debugMsg('cRes = ' + np.str(self.cRes))
self._debugMsg('pRes = ' + np.str(self.pRes))
self._debugMsg('optimal = ' + str(self.optimal))
# Set up the preconditioner if applicable.
self.SetupPrecon()
# Iteratively minimize the quadratic model in the trust region
# m(s) = <g, s> + 1/2 <s, Hs>
# Note that m(s) does not include f(x): m(0) = 0.
H = SimpleLinearOperator(nx + nz,
nx + nz,
lambda v: merit.hprod(v),
symmetric=True)
subsolver = self.TrSolver(
g,
H,
prec=self.Precon,
radius=self.TR.Delta,
reltol=cgtol,
#fraction = 0.5,
itmax=2 * (n + nz),
#debug=True,
#btol=.9,
#cur_iter=np.concatenate((x,z))
)
subsolver.Solve()
if self.debug:
self._debugMsg('x = ' + np.str(x))
self._debugMsg('z = ' + np.str(z))
self._debugMsg('step = ' + np.str(solver.step))
self._debugMsg('step norm = ' + str(solver.stepNorm))
# Record total number of CG iterations.
niter = subsolver.niter
self.cgiter += subsolver.niter
# Compute maximal step to the boundary and next candidate.
alphax, alphaz = self.ftb(x, z, solver.step)
alpha = min(alphax, alphaz)
dx = subsolver.step[:n]
dz = subsolver.step[n:]
x_trial = x + alphax * dx
z_trial = z + alphaz * dz
f_trial = merit.nlp.obj(x_trial)
psi_trial = merit.obj(x_trial, z_trial, f=f_trial)
# Compute ratio of predicted versus achieved reduction.
rho = self.TR.Rho(psi, psi_trial, subsolver.m)
if self.debug:
self._debugMsg('m = ' + str(solver.m))
self._debugMsg('x_trial = ' + np.str(x_trial))
self._debugMsg('z_trial = ' + np.str(z_trial))
self._debugMsg('psi_trial = ' + str(psi_trial))
self._debugMsg('rho = ' + str(rho))
# Accept or reject next candidate
status = 'Rej'
if rho >= self.TR.eta1:
self.TR.UpdateRadius(rho, solver.stepNorm)
x = x_trial
z = z_trial
f = f_trial
psi = psi_trial
gf = nlp.grad(x)
g = self.GradPDMerit(x, z, g=gf, check_optimal=True)
gNorm = norm2(g)
if self.optimal:
finished = True
continue
status = 'Acc'
else:
if self.ny: # Backtracking linesearch a la "Nocedal & Yuan"
slope = np.dot(g, solver.step)
target = psi + 1.0e-4 * alpha * slope
j = 0
while (psi_trial >= target) and (j < self.nyMax):
alphax /= 1.2
alphaz /= 1.2
alpha = min(alphax, alphaz)
target = psi + 1.0e-4 * alpha * slope
x_trial = x + alphax * dx
z_trial = z + alphaz * dz
f_trial = nlp.obj(x_trial)
psi_trial = self.PDMerit(x_trial, z_trial, f=f_trial)
j += 1
if self.opportunistic or (j < self.nyMax):
x = x_trial
z = z_trial #self.PrimalMultipliers(x)
f = f_trial
psi = psi_trial
gf = nlp.grad(x)
g = self.GradPDMerit(x, z, g=gf, check_optimal=True)
gNorm = norm2(g)
if self.optimal:
finished = True
continue
self.TR.Delta = alpha * solver.stepNorm
status = 'N-Y'
else:
self.TR.UpdateRadius(rho, solver.stepNorm)
else:
self.TR.UpdateRadius(rho, solver.stepNorm)
self.UpdatePrecon()
self.iter += 1
inner_iter += 1
finished = (gNorm <= stopTol) or (self.iter >= self.maxiter)
if self.debug: sys.stderr.write('\n')
# Store final iterate
(self.x, self.z) = (x, z)
self.f = f
self.gf = gf
self.g = g
self.gNorm = gNorm
self.psi = psi
return
if not options.verbose:
log.info(
fmt %
(probname, regqp.iter, regqp.obj_value, regqp.pResid, regqp.dResid,
regqp.rgap, t_setup, regqp.solve_time, regqp.short_status))
if regqp.short_status == 'degn':
log.info(' F') # Could not regularize sufficiently.
qp.close()
log.info('-' * len(hdr))
if not multiple_problems:
x = regqp.x[:qp.original_n]
log.info('Final x: %s, |x| = %7.1e' % (repr(x), norm2(x)))
log.info('Final y: %s, |y| = %7.1e' % (repr(regqp.y), norm2(regqp.y)))
log.info('Final z: %s, |z| = %7.1e' % (repr(regqp.z), norm2(regqp.z)))
log.info(regqp.status)
log.info('#Iterations: %-d' % regqp.iter)
log.info('RelResidual: %7.1e' % regqp.kktResid)
log.info('Final cost : %21.15e' % regqp.obj_value)
log.info('Setup time : %6.2fs' % t_setup)
log.info('Solve time : %6.2fs' % regqp.solve_time)
# Plot linear system statistics.
import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.gca()
ax.semilogy(regqp.lres_history)
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations
(default 10n)
:tolerance: Stopping tolerance (default 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
:return:
:x: final iterate
:y: final value of the Lagrange multipliers associated
to `A1 x + A2 s = b`
:z: final value of the Lagrange multipliers associated
to `s >= 0`
:obj_value: final cost
:iter: total number of iterations
:kktResid: final relative residual
:solve_time: time to solve the LP
:status: string describing the exit status
:short_status: short version of status, used for printing.
"""
lp = self.lp
itermax = kwargs.get('itermax', max(100,10*lp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape ; on = lp.original_n
A = self.A ; b = self.b ; c = self.c ; H = self.H
regpr = self.regpr ; regdu = self.regdu
regpr_min = self.regpr_min ; regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
# set_initial_guess() initializes self.LBL which is reused below.
(x,y,z) = self.set_initial_guess(self.lp, **kwargs)
# Slack variables are the trailing variables in x.
s = x[on:] ; ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
col_scale = np.empty(n)
# Allocate room for right-hand side of linear systems.
rhs = np.zeros(n+m)
finished = False
iter = 0
# Acceptance thresholds for primal and dual reg parameters.
#t1 = t2 = 0.99
solve_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if self.verbose and iter % 20 == 0:
sys.stdout.write('\n' + self.header + '\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Compute residuals.
pFeas = A*x - b
comp = s*z ; sz = sum(comp) # comp = S z
dFeas = y*A ; dFeas[:on] -= self.c # dFeas1 = A1'y - c
dFeas[on:] += z # dFeas2 = A2'y + z
mu = sz/ns
# Compute residual norms and scaled residual norms.
pResid = norm2(pFeas) ; spResid = pResid/(1+self.normb+self.normA)
cResid = norm2(comp) ; scResid = cResid/self.normbc
dResid = norm2(dFeas) ; sdResid = dResid/(1+self.normc+self.normA)
# Compute relative duality gap.
cx = np.dot(c,x[:on])
by = np.dot(b,y)
rgap = cx - by
rgap = abs(rgap) / (1 + abs(cx))
rgap2 = mu /(1 + abs(cx))
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
#kktResid = max(pResid, cResid, dResid)
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status= 'iter'
finished = True
continue
# Adjust regularization parameters
#mu = sum(comp)/ns
#if mu < 1:
# regpr = sqrt(mu)
# regdu = sqrt(mu)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
if iter == 0:
regpr = self.regpr ; regdu = self.regdu
if regpr > 0:
q = dFeas/regpr ; qNorm = norm2(q) ; rho_q = regpr * qNorm
else:
q = dFeas ; qNorm = norm2(q) ; rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas/regdu ; rNorm = norm2(r) ; del_r = regdu * rNorm
else:
r = -pFeas ; rNorm = norm2(r) ; del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
# Adjust regularization parameters.
#regpr = max(min(regpr/10, regpr**(1.1)), regpr_min)
#regdu = max(min(regdu/10, regdu**(1.1)), regdu_min)
# 1) rho+ |dx| <= const * s'z
# 2) del+ |dy| <= const * s'z
if regdu > 0:
regdu = min(regdu/10, sz/normdy/10, (sz/normdy)**(1.1))
regdu = max(regdu, regdu_min)
if regpr > 0:
regpr = min(regpr/10, sz/normdx/10, (sz/normdx)**(1.1))
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
#if mu < 1.0e-8 * mu0 and rho_q > 1.0e+3 * kktResid * self.normbc: #* mu * self.normbc:
if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter-1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
#if mu < 1.0e-8 * mu0 and del_r > 1.0e+3 * kktResid * self.normbc: # * mu * self.normbc:
if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter-1:
if du_infeas_count > 6:
status='Problem seems to be (locally) primal infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
# Display objective and residual data.
if self.verbose:
sys.stdout.write(self.format1 % (iter, cx, pResid, dResid,
cResid, rgap, qNorm, rNorm))
# Record some quantities for display
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
# Repeatedly assemble system and compute step until primal and
# dual regularization parameters have appropriate values.
# Reset primal and dual regularization parameters to best guess
#if iter > 0:
# regpr = max(regpr_min, 0.5*sigma*dResid/normds)
# regdu = max(regdu_min, 0.5*sigma*pResid/normdy)
step_acceptable = False
while not step_acceptable:
# Solve the linear system
#
# [-pI 0 A1'] [∆x] [c - A1' y ]
# [ 0 -(S^{-1} Z + pI) A2'] [∆s] = [ - A2' y - µ S^{-1} e]
# [ A1 A2 dI ] [∆y] [b - A1 x - A2 s ]
#
# where s are the slack variables, p is the primal
# regularization parameter, d is the dual regularization
# parameter, and A = [ A1 A2 ] where the columns of A1
# correspond to the original problem variables and those of A2
# correspond to slack variables.
#
# We recover ∆z = -z - S^{-1} (Z ∆s + µ e).
# Compute augmented matrix and factorize it.
factorized = False
nb_bump = 0
while not factorized and nb_bump < 5:
if self.stabilize:
col_scale[:on] = sqrt(regpr)
col_scale[on:] = np.sqrt(z/s + regpr)
H.put(-sqrt(regdu), range(n))
H.put( sqrt(regdu), range(n,n+m))
AA = self.A.copy()
AA.col_scale(1/col_scale)
H[n:,:n] = AA
else:
if regpr > 0: H.put(-regpr, range(on))
H.put(-z/s - regpr, range(on,n))
if regdu > 0: H.put(regdu, range(n,n+m))
#if iter == 5:
# # Export current matrix to file for futher inspection.
# import os
# name = os.path.basename(self.lp.name)
# fname = '.'.join(name.split('.')[:-1]) + '.mtx'
# H.exportMmf(fname)
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
sys.stderr.write('Primal-Dual Matrix ')
sys.stderr.write('Rank Deficient')
if regdu == 0.0:
sys.stderr.write('... No regularization in effect')
sys.stderr.write('... bailing out\n')
factorized = False
nb_bump = 5
continue
else:
sys.stderr.write('... bumping up reg parameters\n')
regpr *= 10 ; regdu *= 10
nb_bump += 1
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and nb_bump >= 5:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue # Does this get us out of the outer while?
# Compute duality measure.
mu = sz/ns
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
rhs[:n] = -dFeas
rhs[on:n] += z
rhs[n:] = -pFeas
# if 'stabilize' is on, must scale right-hand side.
if self.stabilize:
rhs[:n] /= col_scale
rhs[n:] /= sqrt(regdu)
(step, nres, neig) = self.solveSystem(rhs)
# Unscale step if 'stabilize' is on.
if self.stabilize:
step[:n] *= sqrt(regdu) / col_scale
# Recover dx and dz.
dx = step[:n]
ds = dx[on:]
dz = -z * (1 + ds/s)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
sigma = (muAff/mu)**3
# Incorporate predictor information for corrector step.
comp += ds*dz
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100*mu)
# Assemble right-hand side with centering information.
comp -= sigma * mu
if PredictorCorrector:
# Only update rhs[on:n]; the rest of rhs did not change.
if self.stabilize:
rhs[on:n] += (comp/s - z)/col_scale[on:n]
else:
rhs[on:n] += comp/s - z
else:
rhs[:n] = -dFeas
rhs[on:n] += comp/s
rhs[n:] = -pFeas
# If 'stabilize' is on, must scale right-hand side.
# In the predictor-corrector method, this has already been
# done.
if self.stabilize:
rhs[:n] /= col_scale
rhs[n:] /= sqrt(regdu)
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Unscale step if 'stabilize' is on.
if self.stabilize:
step[:n] *= sqrt(regdu) / col_scale
# Recover step.
dx = step[:n]
ds = dx[on:]
dy = step[n:]
normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx)
step_acceptable = True # Must get rid of this
# End while not step_acceptable
# Recover step in z.
dz = -(comp + z*ds)/s
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0-mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip)
alpha_p *= max(1-mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid)
alpha_d *= max(1-mult, gamma_d)
if ip==id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
if self.verbose:
sys.stdout.write(self.format2 % (mu, alpha_p, alpha_d,
nres, regpr, regdu, rho_q,
del_r, mins, minz, maxs))
# Update primal variables and slacks.
x += alpha_p * dx
# Update dual variables.
y += alpha_d * dy
z += alpha_d * dz
# Update perturbation vectors.
q *= (1-alpha_p) ; q += alpha_p * dx
r *= (1-alpha_d) ; r += alpha_d * dy
qNorm = norm2(q) ; rNorm = norm2(r)
rho_q = regpr * qNorm/(1+self.normc) ; rho_q_min = min(rho_q_min, rho_q)
del_r = regdu * rNorm/(1+self.normb) ; del_r_min = min(del_r_min, del_r)
iter += 1
solve_time = cputime() - solve_time
if self.verbose:
sys.stdout.write('\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = np.dot(self.c, x[:on]) + self.c0
return
def scale(self, **kwargs):
"""
Equilibrate the constraint matrix of the linear program. Equilibration
is done by first dividing every row by its largest element in absolute
value and then by dividing every column by its largest element in
absolute value. In effect the original problem::
minimize c' x + 1/2 x' Q x
subject to A1 x + A2 s = b, x >= 0
is converted to::
minimize (Cc)' x + 1/2 x' (CQC') x
subject to R A1 C x + R A2 C s = Rb, x >= 0,
where the diagonal matrices R and C operate row and column scaling
respectively.
Upon return, the matrix A and the right-hand side b are scaled and the
members `row_scale` and `col_scale` are set to the row and column
scaling factors.
The scaling may be undone by subsequently calling :meth:`unscale`. It is
necessary to unscale the problem in order to unscale the final dual
variables. Normally, the :meth:`solve` method takes care of unscaling
the problem upon termination.
"""
log = self.log
m, n = self.A.shape
row_scale = np.zeros(m)
col_scale = np.zeros(n)
(values,irow,jcol) = self.A.find()
if self.verbose:
log.info('Smallest and largest elements of A prior to scaling: ')
log.info('%8.2e %8.2e' % (np.min(np.abs(values)),
np.max(np.abs(values))))
# Find row scaling.
for k in range(len(values)):
row = irow[k]
val = abs(values[k])
row_scale[row] = max(row_scale[row], val)
row_scale[row_scale == 0.0] = 1.0
if self.verbose:
log.info('Max row scaling factor = %8.2e' % np.max(row_scale))
# Apply row scaling to A and b.
values /= row_scale[irow]
self.b /= row_scale
# Find column scaling.
for k in range(len(values)):
col = jcol[k]
val = abs(values[k])
col_scale[col] = max(col_scale[col], val)
col_scale[col_scale == 0.0] = 1.0
if self.verbose:
log.info('Max column scaling factor = %8.2e' % np.max(col_scale))
# Apply column scaling to A and c.
values /= col_scale[jcol]
self.c[:self.qp.original_n] /= col_scale[:self.qp.original_n]
if self.verbose:
log.info('Smallest and largest elements of A after scaling: ')
log.info('%8.2e %8.2e' % (np.min(np.abs(values)),
np.max(np.abs(values))))
# Overwrite A with scaled values.
self.A.put(values,irow,jcol)
self.normA = norm2(values) # Frobenius norm of A.
# Apply scaling to Hessian matrix Q.
(values,irow,jcol) = self.Q.find()
values /= col_scale[irow]
values /= col_scale[jcol]
self.Q.put(values,irow,jcol)
self.normQ = norm2(values) # Frobenius norm of Q
# Save row and column scaling.
self.row_scale = row_scale
self.col_scale = col_scale
self.prob_scaled = True
return
hdr_fmt = '%-13s %-11s %-11s %-11s %-7s %-7s %-5s\n'
res_fmt = '%-13s %-11.5e %-11.5e %-11.5e %-7.3f %-7.3f %-5d\n'
hdrs = ('Name', 'Rel. resid.', 'Residual', 'Resid itref', 'Analyze',
'Solve', 'neig')
header = hdr_fmt % hdrs
lhead = len(header)
sys.stderr.write('-' * lhead + '\n')
sys.stderr.write(header)
sys.stderr.write('-' * lhead + '\n')
# Solve example from the spec sheet
(A, rhs) = Ma27SpecSheet()
(x, r, nr, nr1, t_an, t_sl, neig) = SolveSystem(A, rhs)
exact = numpy.arange(5, dtype = 'd') + 1
relres = norms.norm2(x - exact) / norms.norm2(exact)
sys.stdout.write(res_fmt % ('Spec sheet',relres,nr,nr1,t_an,t_sl,neig))
# Solve example with Hilbert matrix
n = 10
H = Hilbert(n)
e = numpy.ones(n, 'd')
rhs = numpy.empty(n, 'd')
H.matvec(e, rhs)
(x, r, nr, nr1, t_an, t_sl, neig) = SolveSystem(H, rhs)
relres = norms.norm2(x - e) / norms.norm2(e)
sys.stdout.write(res_fmt % ('Hilbert', relres, nr, nr1, t_an, t_sl, neig))
# Process matrices given on the command line
for matrix in matrices:
M = spmatrix.ll_mat_from_mtx(matrix)
**opts_solve)
# Display summary line.
probname=os.path.basename(probname)
if probname[-3:] == '.nl': probname = probname[:-3]
if not options.verbose:
sys.stdout.write(fmt % (probname, regqp.iter, regqp.obj_value,
regqp.pResid, regqp.dResid, regqp.rgap,
t_setup, regqp.solve_time, regqp.short_status))
if regqp.short_status == 'degn':
sys.stdout.write(' F') # Could not regularize sufficiently.
sys.stdout.write('\n')
qp.close()
if not options.verbose:
sys.stderr.write('-'*len(hdr) + '\n')
else:
x = regqp.x[:qp.original_n]
print 'Final x: ', x, ', |x| = %7.1e' % norm2(x)
print 'Final y: ', regqp.y, ', |y| = %7.1e' % norm2(regqp.y)
print 'Final z: ', regqp.z, ', |z| = %7.1e' % norm2(regqp.z)
sys.stdout.write('\n' + regqp.status + '\n')
sys.stdout.write(' #Iterations: %-d\n' % regqp.iter)
sys.stdout.write(' RelResidual: %7.1e\n' % regqp.kktResid)
sys.stdout.write(' Final cost : %21.15e\n' % regqp.obj_value)
sys.stdout.write(' Setup time : %6.2fs\n' % t_setup)
sys.stdout.write(' Solve time : %6.2fs\n' % regqp.solve_time)
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations (default: 10n)
:tolerance: Stopping tolerance (default: 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default: `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
:returns:
:x: final iterate
:y: final value of the Lagrange multipliers associated
to `A1 x + A2 s = b`
:z: final value of the Lagrange multipliers associated
to `s >= 0`
:obj_value: final cost
:iter: total number of iterations
:kktResid: final relative residual
:solve_time: time to solve the QP
:status: string describing the exit status.
:short_status: short version of status, used for printing.
"""
qp = self.qp
itermax = kwargs.get('itermax', max(100, 10 * qp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape
on = qp.original_n
A = self.A
b = self.b
c = self.c
Q = self.Q
diagQ = self.diagQ
H = self.H
regpr = self.regpr
regdu = self.regdu
regpr_min = self.regpr_min
regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
(x, y, z) = self.set_initial_guess(**kwargs)
# Slack variables are the trailing variables in x.
s = x[on:]
ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
# Allocate room for right-hand side of linear systems.
rhs = self.initialize_rhs()
finished = False
iter = 0
setup_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if iter % 50 == 0:
self.log.info(self.header)
self.log.info('-' * len(self.header))
# Compute residuals.
pFeas = A * x - b
comp = s * z
sz = sum(comp) # comp = Sz
Qx = Q * x[:on]
dFeas = y * A
dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx
dFeas[on:] += z # dFeas2 = A2'y + z
# Compute duality measure.
if ns > 0:
mu = sz / ns
else:
mu = 0.0
# Compute residual norms and scaled residual norms.
pResid = norm2(pFeas)
spResid = pResid / (1 + self.normb + self.normA + self.normQ)
dResid = norm2(dFeas)
sdResid = dResid / (1 + self.normc + self.normA + self.normQ)
if ns > 0:
cResid = norm_infty(comp) / (self.normbc + self.normA +
self.normQ)
else:
cResid = 0.0
# Compute relative duality gap.
cx = np.dot(c, x[:on])
xQx = np.dot(x[:on], Qx)
by = np.dot(b, y)
rgap = cx + xQx - by
rgap = abs(rgap) / (1 + abs(cx) + self.normA + self.normQ)
rgap2 = mu / (1 + abs(cx) + self.normA + self.normQ)
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
# Should probably get rid of q when regpr=0 and of r when regdu=0.
if iter == 0:
if regpr > 0:
q = dFeas / regpr
qNorm = dResid / regpr
rho_q = dResid
else:
q = dFeas
qNorm = dResid
rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas / regdu
rNorm = pResid / regdu
del_r = pResid
else:
r = -pFeas
rNorm = pResid
del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
if regdu > 0:
regdu = regdu / 10
regdu = max(regdu, regdu_min)
if regpr > 0:
regpr = regpr / 10
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
if mu < tolerance/100 * mu0 and \
rho_q > 1./tolerance/1.0e+6 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter - 1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual'
status += ' infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
else:
pr_infeas_count = 0
if mu < tolerance/100 * mu0 and \
del_r > 1./tolerance/1.0e+6 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter - 1:
if du_infeas_count > 6:
status = 'Problem seems to be (locally) primal'
status += ' infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
else:
du_infeas_count = 0
# Display objective and residual data.
output_line = self.format1 % (iter, cx + 0.5 * xQx, pResid, dResid,
cResid, rgap, qNorm, rNorm)
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status = 'iter'
finished = True
continue
# Record some quantities for display
if ns > 0:
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
else:
mins = minz = maxs = 0
# Compute augmented matrix and factorize it.
factorized = False
degenerate = False
nb_bump = 0
while not factorized and not degenerate:
self.update_linear_system(s, z, regpr, regdu)
self.log.debug('Factorizing')
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up the
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
self.log.info('Primal-Dual Matrix Rank Deficient' + \
'... bumping up reg parameters')
if regpr == 0. and regdu == 0.:
degenerate = True
else:
if regpr > 0:
regpr *= 100
if regdu > 0:
regdu *= 100
nb_bump += 1
degenerate = nb_bump > self.bump_max
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and degenerate:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
self.set_affine_scaling_rhs(rhs, pFeas, dFeas, s, z)
(step, nres, neig) = self.solveSystem(rhs)
# Recover dx and dz.
dx, ds, dy, dz = self.get_affine_scaling_dxsyz(
step, x, s, y, z)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns
sigma = (muAff / mu)**3
# Incorporate predictor information for corrector step.
# Only update rhs[on:n]; the rest of the vector did not change.
comp += ds * dz
comp -= sigma * mu
self.update_corrector_rhs(rhs, s, z, comp)
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100 * mu)
comp -= sigma * mu
# Assemble rhs.
self.update_long_step_rhs(rhs, pFeas, dFeas, comp, s)
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Recover step.
dx, ds, dy, dz = self.get_dxsyz(step, x, s, y, z, comp)
normds = norm2(ds)
normdy = norm2(dy)
normdx = norm2(dx)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0 - mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic.
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult * mu_tmp - s[ip] * zip) / (alpha_p *
ds[ip] * zip)
alpha_p *= max(1 - mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult * mu_tmp - z[id] * sid) / (alpha_d *
dz[id] * sid)
alpha_d *= max(1 - mult, gamma_d)
if ip == id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
output_line += self.format2 % (mu, alpha_p, alpha_d, nres, regpr,
regdu, rho_q, del_r, mins, minz,
maxs)
self.log.info(output_line)
# Update iterates and perturbation vectors.
x += alpha_p * dx # This also updates slack variables.
y += alpha_d * dy
z += alpha_d * dz
q *= (1 - alpha_p)
q += alpha_p * dx
r *= (1 - alpha_d)
r += alpha_d * dy
qNorm = norm2(q)
rNorm = norm2(r)
if regpr > 0:
rho_q = regpr * qNorm / (1 + self.normc)
rho_q_min = min(rho_q_min, rho_q)
else:
rho_q = 0.0
if regdu > 0:
del_r = regdu * rNorm / (1 + self.normb)
del_r_min = min(del_r_min, del_r)
else:
del_r = 0.0
iter += 1
solve_time = cputime() - setup_time
self.log.info('-' * len(self.header))
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid
self.cResid = cResid
self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = self.c0 + cx + 0.5 * xQx
return
def solveSystem(self, rhs, itref_threshold=1.0e-5, nitrefmax=3):
self.LBL.solve(rhs)
#nr = norm2(self.LBL.residual)
self.LBL.refine(rhs, tol=itref_threshold, nitref=nitrefmax)
nr = norm2(self.LBL.residual)
return (self.LBL.x, nr, self.LBL.neig)
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations (default: 10n)
:tolerance: Stopping tolerance (default: 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default: `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
Upon exit, the following members of the class instance are set:
* x..............final iterate
* y..............final value of the Lagrange multipliers associated
to A1 x + A2 s = b
* z..............final value of the Lagrange multipliers associated
to s>=0
* obj_value......final cost
* iter...........total number of iterations
* kktResid.......final relative residual
* solve_time.....time to solve the QP
* status.........string describing the exit status.
* short_status...short version of status, used for printing.
"""
qp = self.qp
itermax = kwargs.get('itermax', max(100, 10 * qp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape
on = qp.original_n
A = self.A
b = self.b
c = self.c
Q = self.Q
diagQ = self.diagQ
H = self.H
regpr = self.regpr
regdu = self.regdu
regpr_min = self.regpr_min
regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
(x, y, z) = self.set_initial_guess(self.qp, **kwargs)
# Slack variables are the trailing variables in x.
s = x[on:]
ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
col_scale = np.empty(n)
# Allocate room for right-hand side of linear systems.
rhs = np.zeros(n + m)
finished = False
iter = 0
setup_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if self.verbose and iter % 20 == 0:
sys.stdout.write('\n' + self.header + '\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Compute residuals.
pFeas = A * x - b
comp = s * z
sz = sum(comp) # comp = S z
Qx = Q * x[:on]
dFeas = y * A
dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx
dFeas[on:] += z # dFeas2 = A2'y + z
# Compute duality measure.
if ns > 0:
mu = sz / ns
else:
mu = 0.0
# Compute residual norms and scaled residual norms.
#pResid = norm_infty(pFeas + regdu * r)/(1+self.normc+self.normQ)
#dResid = norm_infty(dFeas - regpr * q)/(1+self.normb+self.normQ)
pResid = norm2(pFeas)
spResid = pResid / (1 + self.normc + self.normQ)
dResid = norm2(dFeas)
sdResid = dResid / (1 + self.normb + self.normQ)
if ns > 0:
cResid = norm_infty(comp) / (self.normbc + self.normQ)
else:
cResid = 0.0
# Compute relative duality gap.
cx = np.dot(c, x[:on])
xQx = np.dot(x[:on], Qx)
by = np.dot(b, y)
rgap = cx + xQx - by
#rgap += regdu * (rNorm**2 + np.dot(r,y))
rgap = abs(rgap) / (1 + abs(cx) + self.normQ)
rgap2 = mu / (1 + abs(cx) + self.normQ)
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
#kktResid = max(pResid, cResid, dResid)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
# Should probably get rid of q when regpr=0 and of r when regdu=0.
if iter == 0:
if regpr > 0:
q = dFeas / regpr
qNorm = dResid / regpr
rho_q = dResid
else:
q = dFeas
qNorm = dResid
rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas / regdu
rNorm = pResid / regdu
del_r = pResid
else:
r = -pFeas
rNorm = pResid
del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
regdu = min(regdu / 10, sz / normdy / 10, (sz / normdy)**(1.1))
regdu = max(regdu, regdu_min)
regpr = min(regpr / 10, sz / normdx / 10, (sz / normdx)**(1.1))
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter - 1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter - 1:
if du_infeas_count > 6:
status = 'Problem seems to be (locally) primal infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
# Display objective and residual data.
if self.verbose:
sys.stdout.write(self.format1 %
(iter, cx + 0.5 * xQx, pResid, dResid, cResid,
rgap, qNorm, rNorm))
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status = 'iter'
finished = True
continue
# Record some quantities for display
if ns > 0:
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
else:
mins = minz = maxs = 0
# Solve the linear system
#
# [ -(Q+pI) 0 A1' ] [∆x] = [c + Q x - A1' y ]
# [ 0 -(S^{-1} Z + pI) A2' ] [∆s] [ - A2' y - µ S^{-1} e]
# [ A1 A2 dI ] [∆y] [ b - A1 x - A2 s ]
#
# where s are the slack variables, p is the primal regularization
# parameter, d is the dual regularization parameter, and
# A = [ A1 A2 ] where the columns of A1 correspond to the original
# problem variables and those of A2 correspond to slack variables.
#
# We recover ∆z = -z - S^{-1} (Z ∆s + µ e).
# Compute augmented matrix and factorize it.
factorized = False
nb_bump = 0
while not factorized and nb_bump < 5:
H.put(-diagQ - regpr, range(on))
H.put(-z / s - regpr, range(on, n))
H.put(regdu, range(n, n + m))
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up the
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
sys.stderr.write('Primal-Dual Matrix Rank Deficient')
sys.stderr.write('... bumping up reg parameters\n')
regpr *= 100
regdu *= 100
nb_bump += 1
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and nb_bump == 5:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
rhs[:n] = -dFeas
rhs[on:n] += z
rhs[n:] = -pFeas
(step, nres, neig) = self.solveSystem(rhs)
# Recover dx and dz.
dx = step[:n]
ds = dx[on:]
dz = -z * (1 + ds / s)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns
sigma = (muAff / mu)**3
# Incorporate predictor information for corrector step.
comp += ds * dz
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100 * mu)
# Assemble right-hand side with centering information.
comp -= sigma * mu
if PredictorCorrector:
# Only update rhs[on:n]; the rest of the vector did not change.
rhs[on:n] += comp / s - z
else:
rhs[:n] = -dFeas
rhs[on:n] += comp / s
rhs[n:] = -pFeas
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Recover step.
dx = step[:n]
ds = dx[on:]
dy = step[n:]
dz = -(comp + z * ds) / s
normds = norm2(ds)
normdy = norm2(dy)
normdx = norm2(dx)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0 - mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult * mu_tmp - s[ip] * zip) / (alpha_p *
ds[ip] * zip)
alpha_p *= max(1 - mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult * mu_tmp - z[id] * sid) / (alpha_d *
dz[id] * sid)
alpha_d *= max(1 - mult, gamma_d)
if ip == id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
if self.verbose:
sys.stdout.write(self.format2 %
(mu, alpha_p, alpha_d, nres, regpr, regdu,
rho_q, del_r, mins, minz, maxs))
# Update iterates and perturbation vectors.
x += alpha_p * dx # This also updates slack variables.
y += alpha_d * dy
z += alpha_d * dz
q *= (1 - alpha_p)
q += alpha_p * dx
r *= (1 - alpha_d)
r += alpha_d * dy
qNorm = norm2(q)
rNorm = norm2(r)
rho_q = regpr * qNorm / (1 + self.normc)
rho_q_min = min(rho_q_min, rho_q)
del_r = regdu * rNorm / (1 + self.normb)
del_r_min = min(del_r_min, del_r)
iter += 1
solve_time = cputime() - setup_time
if self.verbose:
sys.stdout.write('\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid
self.cResid = cResid
self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = self.c0 + cx + 0.5 * xQx
return
def SolveInner(self, **kwargs):
"""
Perform a series of inner iterations so as to minimize the primal-dual
merit function with the current value of the barrier parameter to within
some given tolerance. The only optional argument recognized is
stopTol stopping tolerance (default: muerrfact * mu).
"""
merit = self.merit ; nx = merit.nlp.n ; nz = merit.nz
rho = 1 # Dummy initial value for rho
niter = 0 # Dummy initial value for number of inners
status = '' # Dummy initial step status
alpha = 0.0 # Fraction-to-the-boundary step size
if self.inexact:
cgtol = 1.0
else:
cgtol = -1.0
inner_iter = 0 # Inner iteration counter
# Obtain starting point.
(x,z) = (self.x, self.z)
# Obtain first-order data at starting point.
if self.iter == 0:
f = nlp.obj(x) ; gf = nlp.grad(x)
else:
f = self.f ; gf = self.gf
psi = merit.obj(x, z, f=f)
g = merit.grad(x, z, g=gf, check_optimal=True)
gNorm = norm2(g)
if self.optimal: return
# Reset initial trust-region radius
self.TR.Delta = 0.1*gNorm
# Set inner iteration stopping tolerance
stopTol = kwargs.get('stopTol', self.muerrfact * self.mu)
finished = (gNorm <= stopTol) or (self.iter >= self.maxiter)
while not finished:
# Print out header every so many iterations
if self.verbose:
if self.iter % self.printFrequency == 0:
sys.stdout.write(self.hline)
sys.stdout.write(self.header)
sys.stdout.write(self.hline)
if inner_iter == 0:
sys.stdout.write(('*' + self.itFormat) % self.iter)
else:
sys.stdout.write((' ' + self.itFormat) % self.iter)
sys.stdout.write(self.format % (f,
max(norm_infty(self.dRes),
norm_infty(self.cRes),
norm_infty(self.pRes)),
gNorm,
self.mu, alpha, niter, rho,
self.TR.Delta, status))
# Set stopping tolerance for trust-region subproblem.
if self.inexact:
cgtol = max(1.0e-8, min(0.1 * cgtol, sqrt(gNorm)))
if self.debug: self._debugMsg('cgtol = ' + str(cgtol))
# Update Hessian matrix with current iteration information.
# self.PDHess(x,z)
if self.debug:
self._debugMsg('g = ' + np.str(g))
self._debugMsg('gNorm = ' + str(gNorm))
self._debugMsg('stopTol = ' + str(stopTol))
self._debugMsg('dRes = ' + np.str(self.dRes))
self._debugMsg('cRes = ' + np.str(self.cRes))
self._debugMsg('pRes = ' + np.str(self.pRes))
self._debugMsg('optimal = ' + str(self.optimal))
# Set up the preconditioner if applicable.
self.SetupPrecon()
# Iteratively minimize the quadratic model in the trust region
# m(s) = <g, s> + 1/2 <s, Hs>
# Note that m(s) does not include f(x): m(0) = 0.
H = SimpleLinearOperator(nx+nz, nx+nz, lambda v: merit.hprod(v),
symmetric=True)
subsolver = self.TrSolver(g, H,
prec = self.Precon,
radius = self.TR.Delta,
reltol = cgtol,
#fraction = 0.5,
itmax = 2*(n+nz),
#debug=True,
#btol=.9,
#cur_iter=np.concatenate((x,z))
)
subsolver.Solve()
if self.debug:
self._debugMsg('x = ' + np.str(x))
self._debugMsg('z = ' + np.str(z))
self._debugMsg('step = ' + np.str(solver.step))
self._debugMsg('step norm = ' + str(solver.stepNorm))
# Record total number of CG iterations.
niter = subsolver.niter
self.cgiter += subsolver.niter
# Compute maximal step to the boundary and next candidate.
alphax, alphaz = self.ftb(x, z, solver.step)
alpha = min(alphax, alphaz)
dx = subsolver.step[:n] ; dz = subsolver.step[n:]
x_trial = x + alphax * dx
z_trial = z + alphaz * dz
f_trial = merit.nlp.obj(x_trial)
psi_trial = merit.obj(x_trial, z_trial, f=f_trial)
# Compute ratio of predicted versus achieved reduction.
rho = self.TR.Rho(psi, psi_trial, subsolver.m)
if self.debug:
self._debugMsg('m = ' + str(solver.m))
self._debugMsg('x_trial = ' + np.str(x_trial))
self._debugMsg('z_trial = ' + np.str(z_trial))
self._debugMsg('psi_trial = ' + str(psi_trial))
self._debugMsg('rho = ' + str(rho))
# Accept or reject next candidate
status = 'Rej'
if rho >= self.TR.eta1:
self.TR.UpdateRadius(rho, solver.stepNorm)
x = x_trial
z = z_trial
f = f_trial
psi = psi_trial
gf = nlp.grad(x)
g = self.GradPDMerit(x, z, g=gf, check_optimal=True)
gNorm = norm2(g)
if self.optimal:
finished = True
continue
status = 'Acc'
else:
if self.ny: # Backtracking linesearch a la "Nocedal & Yuan"
slope = np.dot(g, solver.step)
target = psi + 1.0e-4 * alpha * slope
j = 0
while (psi_trial >= target) and (j < self.nyMax):
alphax /= 1.2 ; alphaz /= 1.2
alpha = min(alphax, alphaz)
target = psi + 1.0e-4 * alpha * slope
x_trial = x + alphax * dx
z_trial = z + alphaz * dz
f_trial = nlp.obj(x_trial)
psi_trial = self.PDMerit(x_trial, z_trial, f=f_trial)
j += 1
if self.opportunistic or (j < self.nyMax):
x = x_trial
z = z_trial #self.PrimalMultipliers(x)
f = f_trial
psi = psi_trial
gf = nlp.grad(x)
g = self.GradPDMerit(x, z, g=gf, check_optimal=True)
gNorm = norm2(g)
if self.optimal:
finished = True
continue
self.TR.Delta = alpha * solver.stepNorm
status = 'N-Y'
else:
self.TR.UpdateRadius(rho, solver.stepNorm)
else:
self.TR.UpdateRadius(rho, solver.stepNorm)
self.UpdatePrecon()
self.iter += 1
inner_iter += 1
finished = (gNorm <= stopTol) or (self.iter >= self.maxiter)
if self.debug: sys.stderr.write('\n')
# Store final iterate
(self.x, self.z) = (x, z)
self.f = f
self.gf = gf
self.g = g
self.gNorm = gNorm
self.psi = psi
return
# Display summary line.
probname=os.path.basename(probname)
if probname[-3:] == '.nl': probname = probname[:-3]
if not options.verbose:
sys.stdout.write(fmt % (probname, reglp.iter, reglp.obj_value,
reglp.pResid, reglp.dResid, reglp.rgap,
t_setup, reglp.solve_time, reglp.short_status))
if reglp.short_status == 'degn':
sys.stdout.write(' F') # Could not regularize sufficiently.
sys.stdout.write('\n')
lp.close()
if islp:
if not options.verbose:
sys.stderr.write('-'*len(hdr) + '\n')
else:
x = reglp.x[:lp.original_n]
print 'Final x: ', x, ', |x| = %7.1e' % norm2(x)
print 'Final y: ', reglp.y, ', |y| = %7.1e' % norm2(reglp.y)
print 'Final z: ', reglp.z, ', |z| = %7.1e' % norm2(reglp.z)
sys.stdout.write('\n' + reglp.status + '\n')
sys.stdout.write(' #Iterations: %-d\n' % reglp.iter)
sys.stdout.write(' RelResidual: %7.1e\n' % reglp.kktResid)
sys.stdout.write(' Final cost : %21.15e\n' % reglp.obj_value)
sys.stdout.write(' Setup time : %6.2fs\n' % t_setup)
sys.stdout.write(' Solve time : %6.2fs\n' % reglp.solve_time)
def solveSystem(self, rhs, itref_threshold=1.0e-5, nitrefmax=3):
self.LBL.solve(rhs)
#nr = norm2(self.LBL.residual)
self.LBL.refine(rhs, tol=itref_threshold, nitref=nitrefmax)
nr = norm2(self.LBL.residual)
return (self.LBL.x, nr, self.LBL.neig)
def Solve(self, **kwargs):
nlp = self.nlp
# Gather initial information.
self.f = self.nlp.obj(self.x)
self.f0 = self.f
self.g = self.nlp.grad(self.x) # Current gradient
self.g_old = self.g # Previous gradient
self.gnorm = norms.norm2(self.g)
self.g0 = self.gnorm
# Reset initial trust-region radius.
# self.TR.Delta = 0.1 * self.g0
if self.inexact:
cgtol = 1.0
else:
cgtol = -1.0
stoptol = max(self.abstol, self.reltol * self.g0)
step_status = None
exitOptimal = exitIter = exitUser = False
# Initialize non-monotonicity parameters.
if not self.monotone:
fMin = fRef = fCan = self.f0
l = 0
sigRef = sigCan = 0
t = cputime()
# Print out header and initial log.
if self.iter % 20 == 0 and self.verbose:
self.log.info(self.hline)
self.log.info(self.header)
self.log.info(self.hline)
self.log.info(
self.format0 %
(self.iter, self.f, self.gnorm, '', '', self.TR.Delta, ''))
while not (exitUser or exitOptimal or exitIter):
self.iter += 1
self.alpha = 1.0
# Save current gradient
if self.save_g:
self.g_old = self.g.copy()
# Iteratively minimize the quadratic model in the trust region
# m(s) = <g, s> + 1/2 <s, Hs>
# Note that m(s) does not include f(x): m(0) = 0.
if self.inexact:
cgtol = max(1.0e-6, min(0.5 * cgtol, sqrt(self.gnorm)))
H = SimpleLinearOperator(nlp.n,
nlp.n,
lambda v: self.hprod(v),
symmetric=True)
self.solver = self.TrSolver(self.g, H)
self.solver.Solve(
prec=self.precon,
radius=self.TR.Delta,
reltol=cgtol,
#debug=True
)
step = self.solver.step
snorm = self.solver.stepNorm
cgiter = self.solver.niter
# Obtain model value at next candidate
m = self.solver.m
if m is None:
m = numpy.dot(self.g, step) + 0.5 * numpy.dot(step, H * step)
self.total_cgiter += cgiter
x_trial = self.x + step
f_trial = nlp.obj(x_trial)
rho = self.TR.Rho(self.f, f_trial, m)
if not self.monotone:
rhoHis = (fRef - f_trial) / (sigRef - m)
rho = max(rho, rhoHis)
step_status = 'Rej'
if rho >= self.TR.eta1:
# Trust-region step is accepted.
self.TR.UpdateRadius(rho, snorm)
self.x = x_trial
self.f = f_trial
self.g = nlp.grad(self.x)
self.gnorm = norms.norm2(self.g)
step_status = 'Acc'
# Update non-monotonicity parameters.
if not self.monotone:
sigRef = sigRef - m
sigCan = sigCan - m
if f_trial < fMin:
fCan = f_trial
fMin = f_trial
sigCan = 0
l = 0
else:
l = l + 1
if f_trial > fCan:
fCan = f_trial
sigCan = 0
if l == self.nIterNonMono:
fRef = fCan
sigRef = sigCan
else:
# Trust-region step is rejected.
if self.ny: # Backtracking linesearch following "Nocedal & Yuan"
slope = numpy.dot(self.g, step)
bk = 0
while bk < self.nbk and \
f_trial >= self.f + 1.0e-4 * self.alpha * slope:
bk = bk + 1
self.alpha /= 1.2
x_trial = self.x + self.alpha * step
f_trial = nlp.obj(x_trial)
self.x = x_trial
self.f = f_trial
self.g = nlp.grad(self.x)
self.gnorm = norms.norm2(self.g)
self.TR.Delta = self.alpha * snorm
step_status = 'N-Y'
else:
self.TR.UpdateRadius(rho, snorm)
self.step_status = step_status
self.radii.append(self.TR.Delta)
status = ''
try:
self.PostIteration()
except UserExitRequest:
status = 'usr'
# Print out header, say, every 20 iterations
if self.iter % 20 == 0 and self.verbose:
self.log.info(self.hline)
self.log.info(self.header)
self.log.info(self.hline)
if self.verbose:
pstatus = step_status if step_status != 'Acc' else ''
self.log.info(self.format %
(self.iter, self.f, self.gnorm, cgiter, rho,
self.TR.Delta, pstatus))
exitOptimal = self.gnorm < stoptol
exitIter = self.iter > self.maxiter
exitUser = status == 'usr'
self.tsolve = cputime() - t # Solve time
# Set final solver status.
if status == 'usr':
pass
elif self.gnorm <= stoptol:
status = 'opt'
else: # self.iter > self.maxiter:
status = 'itr'
self.status = status
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations (default: 10n)
:tolerance: Stopping tolerance (default: 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default: `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
:returns:
:x: final iterate
:y: final value of the Lagrange multipliers associated
to `A1 x + A2 s = b`
:z: final value of the Lagrange multipliers associated
to `s >= 0`
:obj_value: final cost
:iter: total number of iterations
:kktResid: final relative residual
:solve_time: time to solve the QP
:status: string describing the exit status.
:short_status: short version of status, used for printing.
"""
qp = self.qp
itermax = kwargs.get('itermax', max(100,10*qp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape ; on = qp.original_n
A = self.A ; b = self.b ; c = self.c ; Q = self.Q ; diagQ = self.diagQ
H = self.H
regpr = self.regpr ; regdu = self.regdu
regpr_min = self.regpr_min ; regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
(x,y,z) = self.set_initial_guess(**kwargs)
# Slack variables are the trailing variables in x.
s = x[on:] ; ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
# Allocate room for right-hand side of linear systems.
rhs = self.initialize_rhs()
finished = False
iter = 0
setup_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if iter % 50 == 0:
self.log.info(self.header)
self.log.info('-' * len(self.header))
# Compute residuals.
pFeas = A*x - b
comp = s*z ; sz = sum(comp) # comp = Sz
Qx = Q*x[:on]
dFeas = y*A ; dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx
dFeas[on:] += z # dFeas2 = A2'y + z
# Compute duality measure.
if ns > 0:
mu = sz/ns
else:
mu = 0.0
# Compute residual norms and scaled residual norms.
pResid = norm2(pFeas)
spResid = pResid/(1+self.normb+self.normA+self.normQ)
dResid = norm2(dFeas)
sdResid = dResid/(1+self.normc+self.normA+self.normQ)
if ns > 0:
cResid = norm_infty(comp)/(self.normbc+self.normA+self.normQ)
else:
cResid = 0.0
# Compute relative duality gap.
cx = np.dot(c,x[:on])
xQx = np.dot(x[:on],Qx)
by = np.dot(b,y)
rgap = cx + xQx - by
rgap = abs(rgap) / (1 + abs(cx) + self.normA + self.normQ)
rgap2 = mu / (1 + abs(cx) + self.normA + self.normQ)
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
# Should probably get rid of q when regpr=0 and of r when regdu=0.
if iter == 0:
if regpr > 0:
q = dFeas/regpr ; qNorm = dResid/regpr ; rho_q = dResid
else:
q = dFeas ; qNorm = dResid ; rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas/regdu ; rNorm = pResid/regdu ; del_r = pResid
else:
r = -pFeas ; rNorm = pResid ; del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
if regdu > 0:
regdu = regdu/10
regdu = max(regdu, regdu_min)
if regpr > 0:
regpr = regpr/10
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
if mu < tolerance/100 * mu0 and \
rho_q > 1./tolerance/1.0e+6 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter-1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual'
status += ' infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
else:
pr_infeas_count = 0
if mu < tolerance/100 * mu0 and \
del_r > 1./tolerance/1.0e+6 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter-1:
if du_infeas_count > 6:
status = 'Problem seems to be (locally) primal'
status += ' infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
else:
du_infeas_count = 0
# Display objective and residual data.
output_line = self.format1 % (iter, cx + 0.5 * xQx, pResid,
dResid, cResid, rgap, qNorm,
rNorm)
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status = 'iter'
finished = True
continue
# Record some quantities for display
if ns > 0:
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
else:
mins = minz = maxs = 0
# Compute augmented matrix and factorize it.
factorized = False
degenerate = False
nb_bump = 0
while not factorized and not degenerate:
self.update_linear_system(s, z, regpr, regdu)
self.log.debug('Factorizing')
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up the
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
self.log.info('Primal-Dual Matrix Rank Deficient' + \
'... bumping up reg parameters')
if regpr == 0. and regdu == 0.:
degenerate = True
else:
if regpr > 0:
regpr *= 100
if regdu > 0:
regdu *= 100
nb_bump += 1
degenerate = nb_bump > self.bump_max
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and degenerate:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
self.set_affine_scaling_rhs(rhs, pFeas, dFeas, s, z)
(step, nres, neig) = self.solveSystem(rhs)
# Recover dx and dz.
dx, ds, dy, dz = self.get_affine_scaling_dxsyz(step, x, s, y, z)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
sigma = (muAff/mu)**3
# Incorporate predictor information for corrector step.
# Only update rhs[on:n]; the rest of the vector did not change.
comp += ds*dz
comp -= sigma * mu
self.update_corrector_rhs(rhs, s, z, comp)
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100*mu)
comp -= sigma * mu
# Assemble rhs.
self.update_long_step_rhs(rhs, pFeas, dFeas, comp, s)
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Recover step.
dx, ds, dy, dz = self.get_dxsyz(step, x, s, y, z, comp)
normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0-mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic.
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip)
alpha_p *= max(1-mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid)
alpha_d *= max(1-mult, gamma_d)
if ip==id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
output_line += self.format2 % (mu, alpha_p, alpha_d,
nres, regpr, regdu, rho_q,
del_r, mins, minz, maxs)
self.log.info(output_line)
# Update iterates and perturbation vectors.
x += alpha_p * dx # This also updates slack variables.
y += alpha_d * dy
z += alpha_d * dz
q *= (1-alpha_p) ; q += alpha_p * dx
r *= (1-alpha_d) ; r += alpha_d * dy
qNorm = norm2(q) ; rNorm = norm2(r)
if regpr > 0:
rho_q = regpr * qNorm/(1+self.normc)
rho_q_min = min(rho_q_min, rho_q)
else:
rho_q = 0.0
if regdu > 0:
del_r = regdu * rNorm/(1+self.normb)
del_r_min = min(del_r_min, del_r)
else:
del_r = 0.0
iter += 1
solve_time = cputime() - setup_time
self.log.info('-' * len(self.header))
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = self.c0 + cx + 0.5 * xQx
return
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations (default: 10n)
:tolerance: Stopping tolerance (default: 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default: `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
Upon exit, the following members of the class instance are set:
* x..............final iterate
* y..............final value of the Lagrange multipliers associated
to A1 x + A2 s = b
* z..............final value of the Lagrange multipliers associated
to s>=0
* obj_value......final cost
* iter...........total number of iterations
* kktResid.......final relative residual
* solve_time.....time to solve the QP
* status.........string describing the exit status.
* short_status...short version of status, used for printing.
"""
qp = self.qp
itermax = kwargs.get('itermax', max(100,10*qp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape ; on = qp.original_n
A = self.A ; b = self.b ; c = self.c ; Q = self.Q ; diagQ = self.diagQ
H = self.H
regpr = self.regpr ; regdu = self.regdu
regpr_min = self.regpr_min ; regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
(x,y,z) = self.set_initial_guess(self.qp, **kwargs)
# Slack variables are the trailing variables in x.
s = x[on:] ; ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
col_scale = np.empty(n)
# Allocate room for right-hand side of linear systems.
rhs = np.zeros(n+m)
finished = False
iter = 0
setup_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if self.verbose and iter % 20 == 0:
sys.stdout.write('\n' + self.header + '\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Compute residuals.
pFeas = A*x - b
comp = s*z ; sz = sum(comp) # comp = S z
Qx = Q*x[:on]
dFeas = y*A ; dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx
dFeas[on:] += z # dFeas2 = A2'y + z
# Compute duality measure.
if ns > 0:
mu = sz/ns
else:
mu = 0.0
# Compute residual norms and scaled residual norms.
#pResid = norm_infty(pFeas + regdu * r)/(1+self.normc+self.normQ)
#dResid = norm_infty(dFeas - regpr * q)/(1+self.normb+self.normQ)
pResid = norm2(pFeas) ; spResid = pResid/(1+self.normc+self.normQ)
dResid = norm2(dFeas) ; sdResid = dResid/(1+self.normb+self.normQ)
if ns > 0:
cResid = norm_infty(comp)/(self.normbc+self.normQ)
else:
cResid = 0.0
# Compute relative duality gap.
cx = np.dot(c,x[:on])
xQx = np.dot(x[:on],Qx)
by = np.dot(b,y)
rgap = cx + xQx - by
#rgap += regdu * (rNorm**2 + np.dot(r,y))
rgap = abs(rgap) / (1 + abs(cx) + self.normQ)
rgap2 = mu / (1 + abs(cx) + self.normQ)
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
#kktResid = max(pResid, cResid, dResid)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
# Should probably get rid of q when regpr=0 and of r when regdu=0.
if iter == 0:
if regpr > 0:
q = dFeas/regpr ; qNorm = dResid/regpr ; rho_q = dResid
else:
q = dFeas ; qNorm = dResid ; rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas/regdu ; rNorm = pResid/regdu ; del_r = pResid
else:
r = -pFeas ; rNorm = pResid ; del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
regdu = min(regdu/10, sz/normdy/10, (sz/normdy)**(1.1))
regdu = max(regdu, regdu_min)
regpr = min(regpr/10, sz/normdx/10, (sz/normdx)**(1.1))
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter-1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter-1:
if du_infeas_count > 6:
status='Problem seems to be (locally) primal infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
# Display objective and residual data.
if self.verbose:
sys.stdout.write(self.format1 % (iter, cx + 0.5 * xQx, pResid,
dResid, cResid, rgap, qNorm,
rNorm))
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status = 'iter'
finished = True
continue
# Record some quantities for display
if ns > 0:
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
else:
mins = minz = maxs = 0
# Solve the linear system
#
# [ -(Q+pI) 0 A1' ] [∆x] = [c + Q x - A1' y ]
# [ 0 -(S^{-1} Z + pI) A2' ] [∆s] [ - A2' y - µ S^{-1} e]
# [ A1 A2 dI ] [∆y] [ b - A1 x - A2 s ]
#
# where s are the slack variables, p is the primal regularization
# parameter, d is the dual regularization parameter, and
# A = [ A1 A2 ] where the columns of A1 correspond to the original
# problem variables and those of A2 correspond to slack variables.
#
# We recover ∆z = -z - S^{-1} (Z ∆s + µ e).
# Compute augmented matrix and factorize it.
factorized = False
nb_bump = 0
while not factorized and nb_bump < 5:
H.put(-diagQ - regpr, range(on))
H.put(-z/s - regpr, range(on,n))
H.put(regdu, range(n,n+m))
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up the
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
sys.stderr.write('Primal-Dual Matrix Rank Deficient')
sys.stderr.write('... bumping up reg parameters\n')
regpr *= 100 ; regdu *= 100
nb_bump += 1
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and nb_bump == 5:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
rhs[:n] = -dFeas
rhs[on:n] += z
rhs[n:] = -pFeas
(step, nres, neig) = self.solveSystem(rhs)
# Recover dx and dz.
dx = step[:n]
ds = dx[on:]
dz = -z * (1 + ds/s)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
sigma = (muAff/mu)**3
# Incorporate predictor information for corrector step.
comp += ds*dz
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100*mu)
# Assemble right-hand side with centering information.
comp -= sigma * mu
if PredictorCorrector:
# Only update rhs[on:n]; the rest of the vector did not change.
rhs[on:n] += comp/s - z
else:
rhs[:n] = -dFeas
rhs[on:n] += comp/s
rhs[n:] = -pFeas
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Recover step.
dx = step[:n]
ds = dx[on:]
dy = step[n:]
dz = -(comp + z*ds)/s
normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0-mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip)
alpha_p *= max(1-mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid)
alpha_d *= max(1-mult, gamma_d)
if ip==id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
if self.verbose:
sys.stdout.write(self.format2 % (mu, alpha_p, alpha_d,
nres, regpr, regdu, rho_q,
del_r, mins, minz, maxs))
# Update iterates and perturbation vectors.
x += alpha_p * dx # This also updates slack variables.
y += alpha_d * dy
z += alpha_d * dz
q *= (1-alpha_p) ; q += alpha_p * dx
r *= (1-alpha_d) ; r += alpha_d * dy
qNorm = norm2(q) ; rNorm = norm2(r)
rho_q = regpr * qNorm/(1+self.normc) ; rho_q_min = min(rho_q_min, rho_q)
del_r = regdu * rNorm/(1+self.normb) ; del_r_min = min(del_r_min, del_r)
iter += 1
solve_time = cputime() - setup_time
if self.verbose:
sys.stdout.write('\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = self.c0 + cx + 0.5 * xQx
return
def Solve(self, **kwargs):
nlp = self.nlp
# Gather initial information.
self.f = self.nlp.obj(self.x)
self.f0 = self.f
self.g = self.nlp.grad(self.x) # Current gradient
self.g_old = self.g # Previous gradient
self.gnorm = norms.norm2(self.g)
self.g0 = self.gnorm
# Reset initial trust-region radius.
# self.TR.Delta = 0.1 * self.g0
if self.inexact:
cgtol = 1.0
else:
cgtol = -1.0
stoptol = max(self.abstol, self.reltol * self.g0)
step_status = None
exitOptimal = exitIter = exitUser = False
# Initialize non-monotonicity parameters.
if not self.monotone:
fMin = fRef = fCan = self.f0
l = 0
sigRef = sigCan = 0
t = cputime()
# Print out header and initial log.
if self.iter % 20 == 0 and self.verbose:
self.log.info(self.hline)
self.log.info(self.header)
self.log.info(self.hline)
self.log.info(self.format0 % (self.iter, self.f,
self.gnorm, '', '',
self.TR.Delta, ''))
while not (exitUser or exitOptimal or exitIter):
self.iter += 1
self.alpha = 1.0
# Save current gradient
if self.save_g:
self.g_old = self.g.copy()
# Iteratively minimize the quadratic model in the trust region
# m(s) = <g, s> + 1/2 <s, Hs>
# Note that m(s) does not include f(x): m(0) = 0.
if self.inexact:
cgtol = max(1.0e-6, min(0.5 * cgtol, sqrt(self.gnorm)))
H = SimpleLinearOperator(nlp.n, nlp.n,
lambda v: self.hprod(v),
symmetric=True)
self.solver = self.TrSolver(self.g, H)
self.solver.Solve(prec=self.precon,
radius=self.TR.Delta,
reltol=cgtol,
#debug=True
)
step = self.solver.step
snorm = self.solver.stepNorm
cgiter = self.solver.niter
# Obtain model value at next candidate
m = self.solver.m
if m is None:
m = numpy.dot(self.g, step) + 0.5*numpy.dot(step, H * step)
self.total_cgiter += cgiter
x_trial = self.x + step
f_trial = nlp.obj(x_trial)
rho = self.TR.Rho(self.f, f_trial, m)
if not self.monotone:
rhoHis = (fRef - f_trial)/(sigRef - m)
rho = max(rho, rhoHis)
step_status = 'Rej'
if rho >= self.TR.eta1:
# Trust-region step is accepted.
self.TR.UpdateRadius(rho, snorm)
self.x = x_trial
self.f = f_trial
self.g = nlp.grad(self.x)
self.gnorm = norms.norm2(self.g)
step_status = 'Acc'
# Update non-monotonicity parameters.
if not self.monotone:
sigRef = sigRef - m
sigCan = sigCan - m
if f_trial < fMin:
fCan = f_trial
fMin = f_trial
sigCan = 0
l = 0
else:
l = l + 1
if f_trial > fCan:
fCan = f_trial
sigCan = 0
if l == self.nIterNonMono:
fRef = fCan
sigRef = sigCan
else:
# Trust-region step is rejected.
if self.ny: # Backtracking linesearch following "Nocedal & Yuan"
slope = numpy.dot(self.g, step)
bk = 0
while bk < self.nbk and \
f_trial >= self.f + 1.0e-4 * self.alpha * slope:
bk = bk + 1
self.alpha /= 1.2
x_trial = self.x + self.alpha * step
f_trial = nlp.obj(x_trial)
self.x = x_trial
self.f = f_trial
self.g = nlp.grad(self.x)
self.gnorm = norms.norm2(self.g)
self.TR.Delta = self.alpha * snorm
step_status = 'N-Y'
else:
self.TR.UpdateRadius(rho, snorm)
self.step_status = step_status
self.radii.append(self.TR.Delta)
status = ''
try:
self.PostIteration()
except UserExitRequest:
status = 'usr'
# Print out header, say, every 20 iterations
if self.iter % 20 == 0 and self.verbose:
self.log.info(self.hline)
self.log.info(self.header)
self.log.info(self.hline)
if self.verbose:
pstatus = step_status if step_status != 'Acc' else ''
self.log.info(self.format % (self.iter, self.f,
self.gnorm, cgiter, rho,
self.TR.Delta, pstatus))
exitOptimal = self.gnorm < stoptol
exitIter = self.iter > self.maxiter
exitUser = status == 'usr'
self.tsolve = cputime() - t # Solve time
# Set final solver status.
if status == 'usr':
pass
elif self.gnorm <= stoptol:
status = 'opt'
else: # self.iter > self.maxiter:
status = 'itr'
self.status = status
def scale(self, **kwargs):
"""
Equilibrate the constraint matrix of the linear program. Equilibration
is done by first dividing every row by its largest element in absolute
value and then by dividing every column by its largest element in
absolute value. In effect the original problem::
minimize c' x + 1/2 x' Q x
subject to A1 x + A2 s = b, x >= 0
is converted to::
minimize (Cc)' x + 1/2 x' (CQC') x
subject to R A1 C x + R A2 C s = Rb, x >= 0,
where the diagonal matrices R and C operate row and column scaling
respectively.
Upon return, the matrix A and the right-hand side b are scaled and the
members `row_scale` and `col_scale` are set to the row and column
scaling factors.
The scaling may be undone by subsequently calling :meth:`unscale`. It is
necessary to unscale the problem in order to unscale the final dual
variables. Normally, the :meth:`solve` method takes care of unscaling
the problem upon termination.
"""
w = sys.stdout.write
m, n = self.A.shape
row_scale = np.zeros(m)
col_scale = np.zeros(n)
(values, irow, jcol) = self.A.find()
if self.verbose:
w('Smallest and largest elements of A prior to scaling: ')
w('%8.2e %8.2e\n' %
(np.min(np.abs(values)), np.max(np.abs(values))))
# Find row scaling.
for k in range(len(values)):
row = irow[k]
val = abs(values[k])
row_scale[row] = max(row_scale[row], val)
row_scale[row_scale == 0.0] = 1.0
if self.verbose:
w('Largest row scaling factor = %8.2e\n' % np.max(row_scale))
# Apply row scaling to A and b.
values /= row_scale[irow]
self.b /= row_scale
# Find column scaling.
for k in range(len(values)):
col = jcol[k]
val = abs(values[k])
col_scale[col] = max(col_scale[col], val)
col_scale[col_scale == 0.0] = 1.0
if self.verbose:
w('Largest column scaling factor = %8.2e\n' % np.max(col_scale))
# Apply column scaling to A and c.
values /= col_scale[jcol]
self.c[:self.qp.original_n] /= col_scale[:self.qp.original_n]
if self.verbose:
w('Smallest and largest elements of A after scaling: ')
w('%8.2e %8.2e\n' %
(np.min(np.abs(values)), np.max(np.abs(values))))
# Overwrite A with scaled values.
self.A.put(values, irow, jcol)
# Apply scaling to Hessian matrix Q.
(values, irow, jcol) = self.Q.find()
self.normQ = norm2(values) # Frobenius norm of Q
values /= col_scale[irow]
values /= col_scale[jcol]
self.Q.put(values, irow, jcol)
# Save row and column scaling.
self.row_scale = row_scale
self.col_scale = col_scale
self.prob_scaled = True
return
