def SolveOuter(self, **kwargs):
nlp = self.nlp
n = nlp.n
err = min(nlp.stop_d, nlp.stop_c, nlp.stop_p)
self.mu_min = min(self.mu_min, (err / (1 + self.muerrfact)) / 2)
# Measure solve time
t = cputime()
# Solve sequence of inner iterations
while (not self.optimal) and (self.mu >= self.mu_min) and \
(self.iter < self.maxiter):
self.SolveInner() #stopTol=max(1.0e-7, 5*self.mu))
#self.z = self.PrimalMultipliers(self.x)
#res, self.optimal = self.AtOptimality(self.x, self.z)
self.UpdateMu()
self.tsolve = cputime() - t # Solve time
if self.optimal:
print 'First-order optimal solution found'
elif self.iter >= self.maxiter:
print 'Maximum number of iterations reached'
else:
print 'Reached smallest allowed value of barrier parameter'
#print '(mu = %8.2e, mu_min = %8.2e)' % (self.mu, self.mu_min)
return
def gltr_implicit(H, g, **kwargs):
G = pygltr.PyGltrContext(g, **kwargs)
H.to_csr()
t = cputime()
G.implicit_solve(lambda v: MatVecProd(H,v))
t = cputime() - t
return (G.m, G.mult, G.snorm, G.niter, G.nc, G.ierr, t)
def gltr_implicit(H, g, **kwargs):
G = pygltr.PyGltrContext(g, **kwargs)
H.to_csr()
t = cputime()
G.implicit_solve(lambda v: MatVecProd(H, v))
t = cputime() - t
return (G.m, G.mult, G.snorm, G.niter, G.nc, G.ierr, t)
def SolveOuter(self, **kwargs):
nlp = self.nlp
n = nlp.n
err = min(nlp.stop_d, nlp.stop_c, nlp.stop_p)
self.mu_min = min(self.mu_min, (err/(1 + self.muerrfact))/2)
# Measure solve time
t = cputime()
# Solve sequence of inner iterations
while (not self.optimal) and (self.mu >= self.mu_min) and \
(self.iter < self.maxiter):
self.SolveInner() #stopTol=max(1.0e-7, 5*self.mu))
#self.z = self.PrimalMultipliers(self.x)
#res, self.optimal = self.AtOptimality(self.x, self.z)
self.UpdateMu()
self.tsolve = cputime() - t # Solve time
if self.optimal:
print 'First-order optimal solution found'
elif self.iter >= self.maxiter:
print 'Maximum number of iterations reached'
else:
print 'Reached smallest allowed value of barrier parameter'
#print '(mu = %8.2e, mu_min = %8.2e)' % (self.mu, self.mu_min)
return
def solve(self):
tstart = cputime()
# Initial LBFGS matrix is the identity. In other words,
# the initial search direction is the steepest descent direction.
# This is the original L-BFGS stopping condition.
#stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x))
stoptol = max(self.abstol, self.reltol * self.g0)
while self.gnorm > stoptol and self.iter < self.maxiter:
if not self.silent:
print '%-5d %-12g %-12g' % (self.iter, self.f, self.gnorm)
# Obtain search direction
d = self.lbfgs.matvec(-self.g)
# Prepare for modified More-Thuente linesearch
if self.iter == 0:
stp0 = 1.0/self.gnorm
else:
stp0 = 1.0
SWLS = StrongWolfeLineSearch(self.f,
self.x,
self.g,
d,
lambda z: self.nlp.obj(z),
lambda z: self.nlp.grad(z),
stp = stp0)
# Perform linesearch
SWLS.search()
# SWLS.x contains the new iterate
# SWLS.g contains the objective gradient at the new iterate
# SWLS.f contains the objective value at the new iterate
s = SWLS.x - self.x
self.x = SWLS.x
y = SWLS.g - self.g
self.g = SWLS.g
self.gnorm = norms.norm2(self.g)
self.f = SWLS.f
#stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x))
# Update inverse Hessian approximation using the most recent pair
self.lbfgs.store(s, y)
self.iter += 1
# Code for testing the LBFGS implementation
self.alt_lbfgs.store(s, y)
self.tsolve = cputime() - tstart
self.converged = (self.iter < self.maxiter)
def solve(self):
tstart = cputime()
# Initial LBFGS matrix is the identity. In other words,
# the initial search direction is the steepest descent direction.
# This is the original L-BFGS stopping condition.
#stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x))
stoptol = max(self.abstol, self.reltol * self.g0)
while self.gnorm > stoptol and self.iter < self.maxiter:
if not self.silent:
print '%-5d %-12g %-12g' % (self.iter, self.f, self.gnorm)
# Obtain search direction
d = self.lbfgs.matvec(-self.g)
# Prepare for modified More-Thuente linesearch
if self.iter == 0:
stp0 = 1.0 / self.gnorm
else:
stp0 = 1.0
SWLS = StrongWolfeLineSearch(self.f,
self.x,
self.g,
d,
lambda z: self.nlp.obj(z),
lambda z: self.nlp.grad(z),
stp=stp0)
# Perform linesearch
SWLS.search()
# SWLS.x contains the new iterate
# SWLS.g contains the objective gradient at the new iterate
# SWLS.f contains the objective value at the new iterate
s = SWLS.x - self.x
self.x = SWLS.x
y = SWLS.g - self.g
self.g = SWLS.g
self.gnorm = norms.norm2(self.g)
self.f = SWLS.f
#stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x))
# Update inverse Hessian approximation using the most recent pair
self.lbfgs.store(s, y)
self.iter += 1
self.tsolve = cputime() - tstart
self.converged = (self.iter < self.maxiter)
def pass_to_trunk(nlp, **kwargs):
if nlp.nbounds > 0 or nlp.m > 0: # Check for unconstrained problem
sys.stderr.write('%s has %d bounds and %d general constraints\n' % (ProblemName, nlp.nbounds, nlp.m))
return None
verbose = kwargs.get('verbose', False)
t = cputime()
tr = TR(Delta=1.0, eta1=0.05, eta2=0.9, gamma1=0.25, gamma2=2.5)
# When instantiating TrunkFramework of TrunkLbfgsFramework,
# we select a trust-region subproblem solver of our choice.
TRNK = solver(nlp, tr, TRSolver, **kwargs)
TRNK.TR.Delta = 0.1 * TRNK.gnorm # Reset initial trust-region radius
t_setup = cputime() - t # Setup time
if verbose:
print
print '------------------------------------------'
print 'LDFP: Solving problem %-s with parameters' % ProblemName
hdr = 'eta1 = %-g eta2 = %-g gamma1 = %-g gamma2 = %-g Delta0 = %-g'
print hdr % (tr.eta1, tr.eta2, tr.gamma1, tr.gamma2, tr.Delta)
print '------------------------------------------'
print
TRNK.Solve()
# Output final statistics
if verbose:
print
print 'Final variables:', TRNK.x
print
print '-------------------------------'
print 'Trunk: End of Execution'
print ' Problem : %-s' % ProblemName
print ' Dimension : %-d' % nlp.n
print ' Initial/Final Objective : %-g/%-g' % (TRNK.f0, TRNK.f)
print ' Initial/Final Gradient Norm : %-g/%-g' % (TRNK.g0, TRNK.gnorm)
print ' Number of iterations : %-d' % TRNK.iter
print ' Number of function evals : %-d' % TRNK.nlp.feval
print ' Number of gradient evals : %-d' % TRNK.nlp.geval
print ' Number of Hessian evals : %-d' % TRNK.nlp.Heval
print ' Number of matvec products : %-d' % TRNK.nlp.Hprod
print ' Total/Average Lanczos iter : %-d/%-g' % (TRNK.cgiter, (float(TRNK.cgiter)/TRNK.iter))
print ' Setup/Solve time : %-gs/%-gs' % (t_setup, TRNK.tsolve)
print ' Total time : %-gs' % (t_setup + TRNK.tsolve)
print '-------------------------------'
return (t_setup, TRNK)
def FindFeasible(self):
"""
If rhs was specified, obtain x_feasible satisfying the constraints
"""
n = self.n
if self.debug: self._write('Obtaining feasible solution...\n')
self.t_feasible = cputime()
self.rhs[n:] = self.b
self.Proj.solve(self.rhs)
self.x_feasible = self.Proj.x[:n].copy()
self.t_feasible = cputime() - self.t_feasible
self.CheckAccurate()
if self.debug:
self._write(' done (%-5.2fs)\n' % self.t_feasible)
return
def FindFeasible(self):
"""
If rhs was specified, obtain x_feasible satisfying the constraints
"""
n = self.n
if self.debug: self._write('Obtaining feasible solution...\n')
self.t_feasible = cputime()
self.rhs[n:] = self.b
self.Proj.solve(self.rhs)
self.x_feasible = self.Proj.x[:n].copy()
self.t_feasible = cputime() - self.t_feasible
self.CheckAccurate()
if self.debug:
self._write(' done (%-5.2fs)\n' % self.t_feasible)
return
def pass_to_funnel(nlp, **kwargs):
verbose = (kwargs['print_level'] == 2)
qn = kwargs.pop('quasi_newton')
t = cputime()
if qn:
funn = StructuredLDFPFunnel(nlp, logger_name='funnel.solver', **kwargs)
# funn = LDFPFunnel(nlp, logger_name='funnel.solver', **kwargs)
else:
funn = Funnel(nlp, logger_name='funnel.solver', **kwargs)
t_setup = cputime() - t # Setup time.
funn.solve() #reg=1.0e-8)
return (t_setup, funn)
def pass_to_trunk(nlp, showbanner=True):
if nlp.nbounds > 0 or nlp.m > 0: # Check for unconstrained problem
sys.stderr.write('%s has %d bounds and %d general constraints\n' %
(ProblemName, nlp.nbounds, nlp.m))
return None
t = cputime()
tr = TR(Delta=1.0, eta1=0.05, eta2=0.9, gamma1=0.25, gamma2=2.5)
# When instantiating TrunkFramework of TrunkLbfgsFramework,
# we select a trust-region subproblem solver of our choice.
TRNK = solver(nlp, tr, TRSolver, silent=False, ny=True, inexact=True)
t_setup = cputime() - t # Setup time
if showbanner:
print
print '------------------------------------------'
print 'Trunk: Solving problem %-s with parameters' % ProblemName
hdr = 'eta1 = %-g eta2 = %-g gamma1 = %-g gamma2 = %-g Delta0 = %-g'
print hdr % (tr.eta1, tr.eta2, tr.gamma1, tr.gamma2, tr.Delta)
print '------------------------------------------'
print
TRNK.Solve()
# Output final statistics
print
print 'Final variables:', TRNK.x
print
print '-------------------------------'
print 'Trunk: End of Execution'
print ' Problem : %-s' % ProblemName
print ' Dimension : %-d' % nlp.n
print ' Initial/Final Objective : %-g/%-g' % (TRNK.f0, TRNK.f)
print ' Initial/Final Gradient Norm : %-g/%-g' % (TRNK.g0, TRNK.gnorm)
print ' Number of iterations : %-d' % TRNK.iter
print ' Number of function evals : %-d' % TRNK.nlp.feval
print ' Number of gradient evals : %-d' % TRNK.nlp.geval
print ' Number of Hessian evals : %-d' % TRNK.nlp.Heval
print ' Number of matvec products : %-d' % TRNK.nlp.Hprod
print ' Total/Average Lanczos iter : %-d/%-g' % (TRNK.total_cgiter, (
float(TRNK.total_cgiter) / TRNK.iter))
print ' Setup/Solve time : %-gs/%-gs' % (t_setup, TRNK.tsolve)
print ' Total time : %-gs' % (t_setup + TRNK.tsolve)
print '-------------------------------'
return TRNK
def pass_to_elastic(nlp, **kwargs):
verbose = (kwargs.get('print_level') >= 2)
kwargs.pop('print_level')
maxiter = kwargs.get('maxit', 200)
t = cputime()
eif = EIF(nlp, maxiter=maxiter, **kwargs)
t_setup = cputime() - t # Setup time.
if verbose:
nlp.display_basic_info()
try:
eif.solve()
except:
pass
return (t_setup, eif)
def pass_to_elastic(nlp, **kwargs):
verbose = (kwargs.get('print_level') >= 2)
kwargs.pop('print_level')
maxiter = kwargs.get('maxit',200)
t = cputime()
eif = EIF(nlp, maxiter=maxiter, **kwargs)
t_setup = cputime() - t # Setup time.
if verbose:
nlp.display_basic_info()
try:
eif.solve()
except:
pass
return (t_setup, eif)
def Factorize(self):
"""
Assemble projection matrix and factorize it
P = [ G A^T ]
[ A 0 ],
where G is the preconditioner, or the identity matrix if no
preconditioner was given.
"""
if self.A is None:
raise ValueError, 'No linear equality constraints were specified'
# Form projection matrix
P = spmatrix.ll_mat_sym(self.n + self.m, self.nnzA + self.n)
if self.precon is not None:
P[:self.n,:self.n] = self.precon
else:
r = range(self.n)
P.put(1, r, r)
#for i in range(self.n):
# P[i,i] = 1
P[self.n:,:self.n] = self.A
# Add regularization if requested.
if self.dreg > 0.0:
r = range(self.n, self.n + self.m)
P.put(-self.dreg, r, r)
if self.debug:
msg = 'Factorizing projection matrix '
msg += '(size %-d, nnz = %-d)...\n' % (P.shape[0],P.nnz)
self._write(msg)
self.t_fact = cputime()
self.Proj = LBLContext(P)
self.t_fact = cputime() - self.t_fact
if self.debug:
msg = ' done (%-5.2fs)\n' % self.t_fact
self._write(msg)
self.factorized = True
return
def Factorize(self):
"""
Assemble projection matrix and factorize it
P = [ G A^T ]
[ A 0 ],
where G is the preconditioner, or the identity matrix if no
preconditioner was given.
"""
if self.A is None:
raise ValueError, 'No linear equality constraints were specified'
# Form projection matrix
P = spmatrix.ll_mat_sym(self.n + self.m, self.nnzA + self.n)
if self.precon is not None:
P[:self.n, :self.n] = self.precon
else:
r = range(self.n)
P.put(1, r, r)
#for i in range(self.n):
# P[i,i] = 1
P[self.n:, :self.n] = self.A
# Add regularization if requested.
if self.dreg > 0.0:
r = range(self.n, self.n + self.m)
P.put(-self.dreg, r, r)
if self.debug:
msg = 'Factorizing projection matrix '
msg += '(size %-d, nnz = %-d)...\n' % (P.shape[0], P.nnz)
self._write(msg)
self.t_fact = cputime()
self.Proj = LBLContext(P)
self.t_fact = cputime() - self.t_fact
if self.debug:
msg = ' done (%-5.2fs)\n' % self.t_fact
self._write(msg)
self.factorized = True
return
def SolveSystem(A, rhs, itref_threshold=1.0e-6, nitrefmax=5, **kwargs):
# Obtain Sils context object
t = cputime()
LBL = LBLContext(A, **kwargs)
t_analyze = cputime() - t
# Solve system and compute residual
t = cputime()
LBL.solve(rhs)
t_solve = cputime() - t_analyze
# Compute residual norm
nrhsp1 = norms.norm_infty(rhs) + 1
nr = norms.norm2(LBL.residual)/nrhsp1
# If residual not small, perform iterative refinement
LBL.refine(rhs, tol = itref_threshold, nitref=nitrefmax)
nr1 = norms.norm_infty(LBL.residual)/nrhsp1
return (LBL.x, LBL.residual, nr, nr1, t_analyze, t_solve, LBL.neig)
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations (default: 10n)
:tolerance: Stopping tolerance (default: 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default: `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
Upon exit, the following members of the class instance are set:
x..............final iterate
y..............final value of the Lagrange multipliers associated
to A1 x + A2 s = b
z..............final value of the Lagrange multipliers associated
to s>=0
obj_value......final cost
iter...........total number of iterations
kktResid.......final relative residual
solve_time.....time to solve the LP
status.........string describing the exit status
short_status...short version of status, used for printing.
"""
lp = self.lp
itermax = kwargs.get('itermax', max(100,10*lp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape ; on = lp.original_n
A = self.A ; b = self.b ; c = self.c ; H = self.H
regpr = self.regpr ; regdu = self.regdu
regpr_min = self.regpr_min ; regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
# set_initial_guess() initializes self.LBL which is reused below.
(x,y,z) = self.set_initial_guess(self.lp, **kwargs)
# Slack variables are the trailing variables in x.
s = x[on:] ; ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
col_scale = np.empty(n)
# Allocate room for right-hand side of linear systems.
rhs = np.zeros(n+m)
finished = False
iter = 0
# Acceptance thresholds for primal and dual reg parameters.
#t1 = t2 = 0.99
solve_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if self.verbose and iter % 20 == 0:
sys.stdout.write('\n' + self.header + '\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Compute residuals.
pFeas = A*x - b
comp = s*z ; sz = sum(comp) # comp = S z
dFeas = y*A ; dFeas[:on] -= self.c # dFeas1 = A1'y - c
dFeas[on:] += z # dFeas2 = A2'y + z
mu = sz/ns
# Compute residual norms and scaled residual norms.
# We don't need to keep both the scaled and unscaled residuals
# store.
#pResid = norm_infty(pFeas + regdu * r)/(1+self.normc)
#dResid = norm_infty(dFeas - regpr * q)/(1+self.normb)
pResid = norm2(pFeas) ; spResid = pResid/(1+self.normc)
cResid = norm2(comp) ; scResid = cResid/self.normbc
dResid = norm2(dFeas) ; sdResid = dResid/(1+self.normb)
# Compute relative duality gap.
cx = np.dot(c,x[:on])
by = np.dot(b,y)
rgap = cx - by
#rgap += regdu * (rNorm**2 + np.dot(r,y))
rgap = abs(rgap) / (1 + abs(cx))
rgap2 = mu /(1 + abs(cx))
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
#kktResid = max(pResid, cResid, dResid)
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status= 'iter'
finished = True
continue
# Adjust regularization parameters
#mu = sum(comp)/ns
#if mu < 1:
# regpr = sqrt(mu)
# regdu = sqrt(mu)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
if iter == 0:
regpr = self.regpr ; regdu = self.regdu
if regpr > 0:
q = dFeas/regpr ; qNorm = norm2(q) ; rho_q = regpr * qNorm
else:
q = dFeas ; qNorm = norm2(q) ; rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas/regdu ; rNorm = norm2(r) ; del_r = regdu * rNorm
else:
r = -pFeas ; rNorm = norm2(r) ; del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
# Adjust regularization parameters.
#regpr = max(min(regpr/10, regpr**(1.1)), regpr_min)
#regdu = max(min(regdu/10, regdu**(1.1)), regdu_min)
# 1) rho+ |dx| <= const * s'z
# 2) del+ |dy| <= const * s'z
if regdu > 0:
regdu = min(regdu/10, sz/normdy/10, (sz/normdy)**(1.1))
regdu = max(regdu, regdu_min)
if regpr > 0:
regpr = min(regpr/10, sz/normdx/10, (sz/normdx)**(1.1))
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
#if mu < 1.0e-8 * mu0 and rho_q > 1.0e+3 * kktResid * self.normbc: #* mu * self.normbc:
if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter-1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
#if mu < 1.0e-8 * mu0 and del_r > 1.0e+3 * kktResid * self.normbc: # * mu * self.normbc:
if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter-1:
if du_infeas_count > 6:
status='Problem seems to be (locally) primal infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
# Display objective and residual data.
if self.verbose:
sys.stdout.write(self.format1 % (iter, cx, pResid, dResid,
cResid, rgap, qNorm, rNorm))
# Record some quantities for display
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
# Repeatedly assemble system and compute step until primal and
# dual regularization parameters have appropriate values.
# Reset primal and dual regularization parameters to best guess
#if iter > 0:
# regpr = max(regpr_min, 0.5*sigma*dResid/normds)
# regdu = max(regdu_min, 0.5*sigma*pResid/normdy)
step_acceptable = False
while not step_acceptable:
# Solve the linear system
#
# [-pI 0 A1'] [∆x] [c - A1' y ]
# [ 0 -(S^{-1} Z + pI) A2'] [∆s] = [ - A2' y - µ S^{-1} e]
# [ A1 A2 dI ] [∆y] [b - A1 x - A2 s ]
#
# where s are the slack variables, p is the primal
# regularization parameter, d is the dual regularization
# parameter, and A = [ A1 A2 ] where the columns of A1
# correspond to the original problem variables and those of A2
# correspond to slack variables.
#
# We recover ∆z = -z - S^{-1} (Z ∆s + µ e).
# Compute augmented matrix and factorize it.
factorized = False
nb_bump = 0
while not factorized and nb_bump < 5:
if self.stabilize:
col_scale[:on] = sqrt(regpr)
col_scale[on:] = np.sqrt(z/s + regpr)
H.put(-sqrt(regdu), range(n))
H.put( sqrt(regdu), range(n,n+m))
AA = self.A.copy()
AA.col_scale(1/col_scale)
H[n:,:n] = AA
else:
if regpr > 0: H.put(-regpr, range(on))
H.put(-z/s - regpr, range(on,n))
if regdu > 0: H.put(regdu, range(n,n+m))
#if iter == 5:
# # Export current matrix to file for futher inspection.
# import os
# name = os.path.basename(self.lp.name)
# fname = '.'.join(name.split('.')[:-1]) + '.mtx'
# H.exportMmf(fname)
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
sys.stderr.write('Primal-Dual Matrix ')
sys.stderr.write('Rank Deficient')
if regdu == 0.0:
sys.stderr.write('... No regularization in effect')
sys.stderr.write('... bailing out\n')
factorized = False
nb_bump = 5
continue
else:
sys.stderr.write('... bumping up reg parameters\n')
regpr *= 10 ; regdu *= 10
nb_bump += 1
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and nb_bump >= 5:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue # Does this get us out of the outer while?
# Compute duality measure.
mu = sz/ns
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
rhs[:n] = -dFeas
rhs[on:n] += z
rhs[n:] = -pFeas
# if 'stabilize' is on, must scale right-hand side.
if self.stabilize:
rhs[:n] /= col_scale
rhs[n:] /= sqrt(regdu)
(step, nres, neig) = self.solveSystem(rhs)
# Unscale step if 'stabilize' is on.
if self.stabilize:
step[:n] *= sqrt(regdu) / col_scale
# Recover dx and dz.
dx = step[:n]
ds = dx[on:]
dz = -z * (1 + ds/s)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
sigma = (muAff/mu)**3
# Incorporate predictor information for corrector step.
comp += ds*dz
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100*mu)
# Assemble right-hand side with centering information.
comp -= sigma * mu
if PredictorCorrector:
# Only update rhs[on:n]; the rest of rhs did not change.
if self.stabilize:
rhs[on:n] += (comp/s - z)/col_scale[on:n]
else:
rhs[on:n] += comp/s - z
else:
rhs[:n] = -dFeas
rhs[on:n] += comp/s
rhs[n:] = -pFeas
# If 'stabilize' is on, must scale right-hand side.
# In the predictor-corrector method, this has already been
# done.
if self.stabilize:
rhs[:n] /= col_scale
rhs[n:] /= sqrt(regdu)
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Unscale step if 'stabilize' is on.
if self.stabilize:
step[:n] *= sqrt(regdu) / col_scale
# Recover step.
dx = step[:n]
ds = dx[on:]
dy = step[n:]
normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx)
step_acceptable = True # Must get rid of this
# End while not step_acceptable
# Recover step in z.
dz = -(comp + z*ds)/s
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0-mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip)
alpha_p *= max(1-mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid)
alpha_d *= max(1-mult, gamma_d)
if ip==id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
if self.verbose:
sys.stdout.write(self.format2 % (mu, alpha_p, alpha_d,
nres, regpr, regdu, rho_q,
del_r, mins, minz, maxs))
# Update primal variables and slacks.
x += alpha_p * dx
# Update dual variables.
y += alpha_d * dy
z += alpha_d * dz
# Update perturbation vectors.
q *= (1-alpha_p) ; q += alpha_p * dx
r *= (1-alpha_d) ; r += alpha_d * dy
qNorm = norm2(q) ; rNorm = norm2(r)
rho_q = regpr * qNorm/(1+self.normc) ; rho_q_min = min(rho_q_min, rho_q)
del_r = regdu * rNorm/(1+self.normb) ; del_r_min = min(del_r_min, del_r)
iter += 1
solve_time = cputime() - solve_time
if self.verbose:
sys.stdout.write('\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = np.dot(self.c, x[:on]) + self.c0
return
def Solve(self):
# Find feasible solution
if self.A is not None:
if self.factorize and not self.factorized: self.Factorize()
if self.b is not None: self.FindFeasible()
n = self.n
m = self.m
nMatvec = 0
alpha = beta = omega = 0.0
self.t_solve = cputime()
# Obtain fixed vector r0 = projected initial residual
# (initial x = 0 in homogeneous problem.)
if self.A is not None:
self.rhs[:n] = self.r
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
r0 = self.Proj.x[:n].copy()
Btv = self.r - r0
else:
r0 = self.c
# Initialize search direction
self.p = self.r
# Further initializations
rr0 = rr00 = numpy.dot(self.r, r0)
residNorm = self.residNorm0 = sqrt(rr0)
stopTol = self.abstol + self.reltol * self.residNorm0
finished = False
if self.debug:
self._write(self.header)
self._write('-' * len(self.header) + '\n')
if self.debug:
self._write(self.fmt % (nMatvec, residNorm, rr0, alpha, omega))
while not finished:
# Project p
self.rhs[:n] = self.p
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
self.Pp = self.Proj.x[:n]
# Compute alpha and s
if self._matvec_found:
# Here we must copy Ap to prevent it from being overwritten
# in the next matvec. We still need Ap when we update p below.
self.Ap = self.matvec(self.Pp).copy()
else:
self.H.matvec(self.Pp, self.Ap)
nMatvec += 1
alpha = rr0 / numpy.dot(r0, self.Ap)
self.s = self.r - alpha * self.Ap
# Project s
self.rhs[:n] = self.s - Btv # Iterative semi-refinement
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
self.Ps = self.Proj.x[:n].copy()
Btv = self.s - self.Ps
residNorm = sqrt(numpy.dot(self.s, self.Ps))
# Test for termination in the CGS process
if residNorm <= stopTol or nMatvec > self.nMatvecMax:
self.x += alpha * self.Pp
if nMatvec > self.nMatvecMax:
reason = 'matvec'
else:
reason = 's small'
finished = True
else:
# Project A*Ps
if self._matvec_found:
self.As = self.matvec(self.Ps)
else:
self.H.matvec(self.Ps, self.As)
nMatvec += 1
self.rhs[:n] = self.As
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
# Compute omega and update x
sAs = numpy.dot(self.Ps, self.As)
AsPAs = numpy.dot(self.As, self.Proj.x[:n])
omega = sAs / AsPAs
self.x += alpha * self.Pp + omega * self.Ps
# Check for termination
if nMatvec > self.nMatvecMax:
finished = True
reason = 'matvec'
else:
# Update residual
self.r = self.s - omega * self.As
rr0_next = numpy.dot(self.r, r0)
beta = alpha / omega * rr0_next / rr0
rr0 = rr0_next
self.p -= omega * self.Ap
self.p *= beta
self.p += self.r
# Check for termination in the Bi-CGSTAB process
if abs(rr0) < 1.0e-12 * rr00:
self.rhs[:n] = self.r
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
rPr = numpy.dot(self.r, self.Proj.x[:n])
if sqrt(rPr) <= stopTol:
finished = True
reason = 'r small'
# Display current iteration info
if self.debug:
self._write(self.fmt % (nMatvec, residNorm, rr0, alpha, omega))
# End while
# Obtain final solution x
if self.x_feasible is not None:
self.x += self.x_feasible
if self.A is not None:
# Find (weighted) least-squares Lagrange multipliers from
# [ G B^T ] [w] [c - Hx]
# [ B 0 ] [v] = [ 0 ]
if self._matvec_found:
self.rhs[:n] = -self.matvec(self.x)
else:
self.H.matvec(-self.x, self.rhs[:n])
self.rhs[:n] += self.c
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
self.v = self.Proj.x[n:].copy()
self.t_solve = cputime() - self.t_solve
self.converged = (nMatvec < self.nMatvecMax)
self.nMatvec = nMatvec
self.residNorm = residNorm
self.status = reason
return
def solve(self, **kwargs):
"""
Solve current problem with trust-funnel framework.
:keywords:
:ny: Enable Nocedal-Yuan backtracking linesearch.
:returns:
This method sets the following members of the instance:
:f: Final objective value
:optimal: Flag indicating whether normal stopping conditions were
attained
:pResid: Final primal residual
:dResid: Final dual residual
:niter: Total number of iterations
:tsolve: Solve time.
"""
ny = kwargs.get('ny', True)
reg = kwargs.get('reg', 0.0)
#ny = False
tsolve = cputime()
# Set some shortcuts.
nlp = self.nlp
n = nlp.n
m = nlp.m
x = self.x
f = self.f
c = self.cons(x)
y = nlp.pi0.copy()
self.it = 0
# Initialize some constants.
kappa_n = 1.0e+2 # Factor of pNorm in normal step TR radius.
kappa_b = 0.99 # Fraction of TR to compute tangential step.
kappa_delta = 0.1 # Progress factor to compute tangential step.
# Trust-region parameters.
eta_1 = 1.0e-5
eta_2 = 0.95
eta_3 = 0.5
gamma_1 = 0.25
gamma_3 = 2.5
kappa_tx1 = 0.9 # Factor of theta_max in max acceptable infeasibility.
kappa_tx2 = 0.5 # Convex combination factor of theta and thetaTrial.
# Compute constraint violation.
theta = 0.5 * np.dot(c,c)
# Set initial funnel radius.
kappa_ca = 1.0e+3 # Max initial funnel radius.
kappa_cr = 2.0 # Infeasibility tolerance factor.
theta_max = max(kappa_ca, kappa_cr * theta)
# Evaluate first-order derivatives.
g = nlp.grad(x)
J = self.jac(x)
Jop = PysparseLinearOperator(J)
# Initial radius for f- and c-iterations.
Delta_f = max(self.Delta_f, .1 * np.linalg.norm(g))
Delta_c = max(self.Delta_c, .1 * sqrt(2*theta))
# Reset initial multipliers to least-squares estimates by
# approximately solving:
# [ I J' ] [ w ] [ -g ]
# [ J 0 ] [ y ] = [ 0 ].
# This is equivalent to solving
# minimize |g + J'y|.
if m > 0:
y, _, _ = self.lsq(Jop.T, -g, reg=reg)
pNorm = cNorm = 0
if m > 0:
pNorm = np.linalg.norm(c);
cNorm = np.linalg.norm(c, np.inf)
grad_lag = g + Jop.T * y
else:
grad_lag = g.copy()
dNorm = np.linalg.norm(grad_lag)/(1 + np.linalg.norm(y))
# Display current info if requested.
self.log.info(self.hdr)
self.log.info(self.linefmt1 % (0, ' ', ' ', ' ', ' ', f, pNorm,
dNorm, Delta_f, Delta_c, theta_max, 0))
# Compute primal stopping tolerance.
stop_p = max(self.atol, self.stop_p * pNorm)
self.log.debug('pNorm = %8.2e, cNorm = %8.2e, dNorm = %8.e2' % (pNorm,cNorm,dNorm))
optimal = (pNorm <= stop_p) and (dNorm <= self.stop_d)
self.log.debug('optimal: %s' % repr(optimal))
# Start of main iteration.
while not optimal and (self.it < self.maxit):
self.it += 1
Delta = min(Delta_f, Delta_c)
cgiter = 0
# 1. Compute normal step as an (approximate) solution to
# minimize |c + J n| subject to |n| <= min(Delta_c, kN |c|).
if self.it > 1 and \
pNorm <= stop_p and \
dNorm >= 1.0e+4 * self.stop_d:
self.log.debug('Setting nStep=0 b/c need to work on optimality')
nStep = np.zeros(n)
nStepNorm = 0.0
n_end = '0'
m_xpn = 0 # Model value at x+n.
else:
nStep_max = min(Delta_c, kappa_n * pNorm)
nStep, nStepNorm, lsq_status = self.lsq(Jop, -c,
radius=nStep_max,
reg=reg)
if lsq_status == 'residual small':
n_end = 'r'
elif lsq_status == 'trust-region boundary active':
n_end = 'b'
else:
n_end = '?'
# Evaluate the model of the obective after the normal step.
_Hv = self.hprod(x, y, nStep) # H*nStep
m_xpn = np.dot(g, nStep) + 0.5 * np.dot(nStep, _Hv)
self.log.debug('Normal step norm = %8.2e' % nStepNorm)
self.log.debug('Model value: %9.2e' % m_xpn)
# 2. Compute tangential step if normal step is not too long.
if nStepNorm <= kappa_b * Delta:
# 2.1. Compute Lagrange multiplier estimates and dual residuals
# by minimizing |(g + H n) + J'y|
if nStepNorm == 0.0:
gN = g # Note: this is just a pointer ; g will not be modified below.
else:
gN = g + _Hv
y_new, y_norm, _ = self.lsq(Jop.T, -gN, reg=reg)
r = gN + Jop.T * y_new
# Here Nick does iterative refinement to improve r and y_new...
# Compute dual optimality measure.
residNorm = np.linalg.norm(r)
norm_gN = np.linalg.norm(gN)
pi = 0.0
if residNorm > 0:
pi = abs(np.dot(gN, r))/residNorm
# 2.2. If the dual residuals are large, compute a suitable
# tangential step as a solution to:
# minimize g't + 1/2 t' H t
# subject to Jt = 0, |n+t| <= Delta.
if pi > self.forcing(3, theta):
self.log.debug('Computing tStep...')
Delta_within = Delta - nStepNorm
Hop = SimpleLinearOperator(n, n,
lambda v: self.hprod(x,y_new,v),
symmetric=True)
PPCG = ProjectedCG(gN, Hop,
A=J.matrix if m > 0 else None,
radius=Delta_within, dreg=reg)
PPCG.Solve()
tStep = PPCG.step
tStepNorm = PPCG.stepNorm
cgiter = PPCG.iter
self.log.debug('|t| = %8.2e' % tStepNorm)
if PPCG.status == 'residual small':
t_end = 'r'
elif PPCG.onBoundary and not PPCG.infDescent:
t_end = 'b'
elif PPCG.infDescent:
t_end = '-'
elif PPCG.status == 'max iter':
t_end = '>'
else:
t_end = '?'
# Compute total step and model decrease.
step = nStep + tStep
stepNorm = np.linalg.norm(step)
_Hv = self.hprod(x,y,step) # y or y_new?
m_xps = np.dot(g, step) + 0.5 * np.dot(step, _Hv)
else:
self.log.debug('Setting tStep=0 b/c pi is sufficiently small')
tStepNorm = 0
t_end = '0'
step = nStep
stepNorm = nStepNorm
m_xps = m_xpn
y = y_new
else:
# No need to compute a tangential step.
self.log.debug('Setting tStep=0 b/c the normal step is too large')
t_end = '0'
y = np.zeros(m)
tStepNorm = 0.0
step = nStep
stepNorm = nStepNorm
m_xps = m_xpn
self.log.debug('Model decrease = %9.2e' % m_xps)
# Compute trial point and evaluate local data.
xTrial = x + step
fTrial = nlp.obj(xTrial)
cTrial = self.cons(xTrial)
thetaTrial = 0.5 * np.dot(cTrial, cTrial)
delta_f = -m_xps # Overall improvement in the model.
delta_ft = m_xpn - m_xps # Improvement due to tangential step.
Jspc = c + Jop * step
# Compute improvement in linearized feasibility.
delta_feas = theta - 0.5 * np.dot(Jspc, Jspc)
# Decide whether to consider the current iteration
# an f- or a c-iteration.
if tStepNorm > 0 and (delta_f >= self.forcing(2, theta)) and \
delta_f >= kappa_delta * delta_ft and \
thetaTrial <= theta_max:
# Step 3. Consider that this is an f-iteration.
it_type = 'f'
# Decide whether trial point is accepted.
ratio = (f - fTrial)/delta_f
self.log.debug('f-iter ratio = %9.2e' % ratio)
if ratio >= eta_1: # Successful step.
suc = 's'
x = xTrial
f = fTrial
c = cTrial
self.step = step.copy()
theta = thetaTrial
# Decide whether to update f-trust-region radius.
if ratio >= eta_2:
suc = 'v'
Delta_f = min(max(Delta_f, gamma_3 * stepNorm),
1.0e+10)
# Decide whether to update c-trust-region radius.
if thetaTrial < eta_3 * theta_max:
ns = nStepNorm if nStepNorm > 0 else Delta_c
Delta_c = min(max(Delta_c, gamma_3 * ns),
1.0e+10)
self.log.debug('New Delta_f = %8.2e' % Delta_f)
self.log.debug('New Delta_c = %8.2e' % Delta_c)
else: # Unsuccessful step (ratio < eta_1).
attempt_SOC = True
suc = 'u'
if attempt_SOC:
self.log.debug(' Attempting second-order correction')
# Attempt a second-order correction by solving
# minimize |cTrial + J n| subject to |n| <= Delta_c.
socStep, socStepNorm, socStatus = self.lsq(Jop,
-cTrial,
radius=Delta_c,
reg=reg)
if socStatus != 'trust-region boundary active':
# Consider SOC step as candidate step.
xSoc = xTrial + socStep
fSoc = nlp.obj(xSoc) ; cSoc = self.cons(xSoc)
thetaSoc = 0.5 * np.dot(cSoc, cSoc)
ratio = (f - fSoc)/delta_f
# Decide whether to accept SOC step.
if ratio >= eta_1 and thetaSoc <= theta_max:
suc = '2'
x = xSoc
f = fSoc
c = cSoc
theta = thetaSoc
self.step = step + socStep
else:
# Backtracking linesearch a la Nocedal & Yuan.
# Abandon SOC step. Backtrack from x+step.
if ny:
(x, f, alpha) = self.nyf(x, f, fTrial,
g, step)
#g = nlp.grad(x)
c = self.cons(x)
theta = 0.5 * np.dot(c,c)
self.step = step + alpha * socStep
Delta_f = min(alpha, .8) * stepNorm
suc = 'y'
else:
Delta_f = gamma_1 * Delta_f
else: # SOC step lies on boundary of trust region.
Delta_f = gamma_1 * Delta_f
else: # SOC step not attempted.
# Backtracking linesearch a la Nocedal & Yuan.
if ny:
(x, f, alpha) = self.nyf(x, f, fTrial, g, step)
#g = nlp.grad(x)
c = self.cons(x)
theta = 0.5 * np.dot(c,c)
self.step = alpha * step
Delta_f = min(alpha, .8) * stepNorm
suc = 'y'
else:
Delta_f = gamma_1 * Delta_f
else:
# Step 4. Consider that this is a c-iteration.
it_type = 'c'
# Display information.
self.log.debug('c-iteration because ')
if tStepNorm == 0.0:
self.log.debug('|t|=0')
if delta_f < self.forcing(2, theta):
self.log.debug('delta_f=%8.2e < forcing=%8.2e'\
% (delta_f, self.forcing(2, theta)))
if delta_f < kappa_delta * delta_ft:
self.log.debug('delta_f=%8.2e < frac * delta_ft=%8.2e' \
% (delta_f, delta_ft))
if thetaTrial > theta_max:
self.log.debug('thetaTrial=%8.2e > theta_max=%8.2e' \
% (thetaTrial, theta_max))
# Step 4.1. Check trial point for acceptability.
if delta_feas < 0:
self.log.debug(' !!! Warning: delta_feas is negative !!!')
ratio = (theta - thetaTrial + 1.0e-16)/(delta_feas + 1.0e-16)
self.log.debug('c-iter ratio = %9.2e' % ratio)
if ratio >= eta_1: # Successful step.
x = xTrial
f = fTrial
c = cTrial
self.step = step.copy()
suc = 's'
# Step 4.2. Update Delta_c.
if ratio >= eta_2: # Very successful step.
ns = nStepNorm if nStepNorm > 0 else Delta_c
Delta_c = min(max(Delta_c, gamma_3 * ns),
1.0e+10)
suc = 'v'
# Step 4.3. Update maximum acceptable infeasibility.
theta_max = max(kappa_tx1 * theta_max,
kappa_tx2*theta + (1-kappa_tx2)*thetaTrial)
theta = thetaTrial
else: # Unsuccessful step.
# Backtracking linesearch a la Nocedal & Yuan.
ns = nStepNorm if nStepNorm > 0 else Delta_c
if ny:
(x, c, theta, alpha) = self.nyc(x, theta, thetaTrial,
c, Jop.T*c, step)
f = nlp.obj(x)
#g = nlp.grad(x)
self.step = alpha * step
Delta_c = min(alpha, .8) * ns
suc = 'y'
else:
Delta_c = gamma_1 * ns #Delta_c
suc = 'u'
self.log.debug('New Delta_c = %8.2e' % Delta_c)
self.log.debug('New theta_max = %8.2e' % theta_max)
# Step 5. Book keeping.
if ratio >= eta_1 or ny:
g = nlp.grad(x)
J = self.jac(x)
Jop = PysparseLinearOperator(J)
self.post_iteration()
pNorm = cNorm = 0
if m > 0:
pNorm = np.linalg.norm(c)
cNorm = np.linalg.norm(c, np.inf)
grad_lag = g + Jop.T * y
else:
grad_lag = g.copy()
dNorm = np.linalg.norm(grad_lag)/(1 + np.linalg.norm(y))
if self.it % 20 == 0:
self.log.info(self.hdr)
self.log.info(self.linefmt % (self.it, it_type, suc, n_end,
t_end, f, pNorm, dNorm, Delta_f,
Delta_c, theta_max, cgiter))
optimal = (pNorm <= stop_p) and (dNorm <= self.stop_d)
# End while.
self.tsolve = cputime() - tsolve
if optimal:
self.status = 0 # Successful solve.
self.log.info('Found an optimal solution! Yeah!')
else:
self.status = 1 # Refine this in the future.
self.x = x
self.f = f
self.optimal = optimal
self.pResid = pNorm
self.dResid = dNorm
self.niter = self.it
return
nA = A.shape[0]
nC = C.shape[0]
K = spmatrix.ll_mat_sym(nA + nC, A.nnz + C.nnz + min(nA,nC))
K[:nA,:nA] = A
K[nA:,nA:] = C
K[nA:,nA:].scale(-1.0)
idx = np.arange(min(nA,nC), dtype=np.int)
K.put(1, nA+idx, idx)
# Create right-hand side rhs=K*e
e = np.ones(nA+nC)
rhs = np.empty(nA+nC)
K.matvec(e,rhs)
# Factorize and solve Kx = rhs, knowing K is sqd
t = cputime()
P = LBLContext(K, sqd=True)
t = cputime() - t
sys.stderr.write('Factorization time with sqd=True : %5.2fs ' % t )
P.solve(rhs, get_resid=False)
sys.stderr.write('Error: %7.1e\n' % np.linalg.norm(P.x - e, ord=np.Inf))
# Do it all over again, pretending we don't know K is sqd
t = cputime()
P = LBLContext(K)
t = cputime() - t
sys.stderr.write('Factorization time with sqd=False: %5.2fs ' % t )
P.solve(rhs, get_resid=False)
sys.stderr.write('Error: %7.1e\n' % np.linalg.norm(P.x - e, ord=np.Inf))
try:
opts_solve = {}
if options.maxiter is not None:
opts_solve['itermax'] = options.maxiter
if options.tol is not None:
opts_solve['tolerance'] = options.tol
# Set printing standards for arrays.
numpy.set_printoptions(precision=3, linewidth=80, threshold=10, edgeitems=3)
if not options.verbose:
sys.stderr.write(hdr + '\n' + '-'*len(hdr) + '\n')
for probname in args:
t_setup = cputime()
qp = SlackFramework(probname)
t_setup = cputime() - t_setup
# isqp() should be implemented in the near future.
#if not qp.isqp():
# sys.stderr.write('Problem %s is not a linear program\n' % probname)
# qp.close()
# continue
# Pass problem to RegQP.
regqp = RegQPInteriorPointSolver(qp,
scale=not options.no_scale,
verbose=options.verbose,
**opts_init)
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations
(default 10n)
:tolerance: Stopping tolerance (default 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
:return:
:x: final iterate
:y: final value of the Lagrange multipliers associated
to `A1 x + A2 s = b`
:z: final value of the Lagrange multipliers associated
to `s >= 0`
:obj_value: final cost
:iter: total number of iterations
:kktResid: final relative residual
:solve_time: time to solve the LP
:status: string describing the exit status
:short_status: short version of status, used for printing.
"""
lp = self.lp
itermax = kwargs.get('itermax', max(100,10*lp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape ; on = lp.original_n
A = self.A ; b = self.b ; c = self.c ; H = self.H
regpr = self.regpr ; regdu = self.regdu
regpr_min = self.regpr_min ; regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
# set_initial_guess() initializes self.LBL which is reused below.
(x,y,z) = self.set_initial_guess(self.lp, **kwargs)
# Slack variables are the trailing variables in x.
s = x[on:] ; ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
col_scale = np.empty(n)
# Allocate room for right-hand side of linear systems.
rhs = np.zeros(n+m)
finished = False
iter = 0
# Acceptance thresholds for primal and dual reg parameters.
#t1 = t2 = 0.99
solve_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if self.verbose and iter % 20 == 0:
sys.stdout.write('\n' + self.header + '\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Compute residuals.
pFeas = A*x - b
comp = s*z ; sz = sum(comp) # comp = S z
dFeas = y*A ; dFeas[:on] -= self.c # dFeas1 = A1'y - c
dFeas[on:] += z # dFeas2 = A2'y + z
mu = sz/ns
# Compute residual norms and scaled residual norms.
pResid = norm2(pFeas) ; spResid = pResid/(1+self.normb+self.normA)
cResid = norm2(comp) ; scResid = cResid/self.normbc
dResid = norm2(dFeas) ; sdResid = dResid/(1+self.normc+self.normA)
# Compute relative duality gap.
cx = np.dot(c,x[:on])
by = np.dot(b,y)
rgap = cx - by
rgap = abs(rgap) / (1 + abs(cx))
rgap2 = mu /(1 + abs(cx))
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
#kktResid = max(pResid, cResid, dResid)
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status= 'iter'
finished = True
continue
# Adjust regularization parameters
#mu = sum(comp)/ns
#if mu < 1:
# regpr = sqrt(mu)
# regdu = sqrt(mu)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
if iter == 0:
regpr = self.regpr ; regdu = self.regdu
if regpr > 0:
q = dFeas/regpr ; qNorm = norm2(q) ; rho_q = regpr * qNorm
else:
q = dFeas ; qNorm = norm2(q) ; rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas/regdu ; rNorm = norm2(r) ; del_r = regdu * rNorm
else:
r = -pFeas ; rNorm = norm2(r) ; del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
# Adjust regularization parameters.
#regpr = max(min(regpr/10, regpr**(1.1)), regpr_min)
#regdu = max(min(regdu/10, regdu**(1.1)), regdu_min)
# 1) rho+ |dx| <= const * s'z
# 2) del+ |dy| <= const * s'z
if regdu > 0:
regdu = min(regdu/10, sz/normdy/10, (sz/normdy)**(1.1))
regdu = max(regdu, regdu_min)
if regpr > 0:
regpr = min(regpr/10, sz/normdx/10, (sz/normdx)**(1.1))
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
#if mu < 1.0e-8 * mu0 and rho_q > 1.0e+3 * kktResid * self.normbc: #* mu * self.normbc:
if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter-1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
#if mu < 1.0e-8 * mu0 and del_r > 1.0e+3 * kktResid * self.normbc: # * mu * self.normbc:
if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter-1:
if du_infeas_count > 6:
status='Problem seems to be (locally) primal infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
# Display objective and residual data.
if self.verbose:
sys.stdout.write(self.format1 % (iter, cx, pResid, dResid,
cResid, rgap, qNorm, rNorm))
# Record some quantities for display
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
# Repeatedly assemble system and compute step until primal and
# dual regularization parameters have appropriate values.
# Reset primal and dual regularization parameters to best guess
#if iter > 0:
# regpr = max(regpr_min, 0.5*sigma*dResid/normds)
# regdu = max(regdu_min, 0.5*sigma*pResid/normdy)
step_acceptable = False
while not step_acceptable:
# Solve the linear system
#
# [-pI 0 A1'] [∆x] [c - A1' y ]
# [ 0 -(S^{-1} Z + pI) A2'] [∆s] = [ - A2' y - µ S^{-1} e]
# [ A1 A2 dI ] [∆y] [b - A1 x - A2 s ]
#
# where s are the slack variables, p is the primal
# regularization parameter, d is the dual regularization
# parameter, and A = [ A1 A2 ] where the columns of A1
# correspond to the original problem variables and those of A2
# correspond to slack variables.
#
# We recover ∆z = -z - S^{-1} (Z ∆s + µ e).
# Compute augmented matrix and factorize it.
factorized = False
nb_bump = 0
while not factorized and nb_bump < 5:
if self.stabilize:
col_scale[:on] = sqrt(regpr)
col_scale[on:] = np.sqrt(z/s + regpr)
H.put(-sqrt(regdu), range(n))
H.put( sqrt(regdu), range(n,n+m))
AA = self.A.copy()
AA.col_scale(1/col_scale)
H[n:,:n] = AA
else:
if regpr > 0: H.put(-regpr, range(on))
H.put(-z/s - regpr, range(on,n))
if regdu > 0: H.put(regdu, range(n,n+m))
#if iter == 5:
# # Export current matrix to file for futher inspection.
# import os
# name = os.path.basename(self.lp.name)
# fname = '.'.join(name.split('.')[:-1]) + '.mtx'
# H.exportMmf(fname)
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
sys.stderr.write('Primal-Dual Matrix ')
sys.stderr.write('Rank Deficient')
if regdu == 0.0:
sys.stderr.write('... No regularization in effect')
sys.stderr.write('... bailing out\n')
factorized = False
nb_bump = 5
continue
else:
sys.stderr.write('... bumping up reg parameters\n')
regpr *= 10 ; regdu *= 10
nb_bump += 1
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and nb_bump >= 5:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue # Does this get us out of the outer while?
# Compute duality measure.
mu = sz/ns
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
rhs[:n] = -dFeas
rhs[on:n] += z
rhs[n:] = -pFeas
# if 'stabilize' is on, must scale right-hand side.
if self.stabilize:
rhs[:n] /= col_scale
rhs[n:] /= sqrt(regdu)
(step, nres, neig) = self.solveSystem(rhs)
# Unscale step if 'stabilize' is on.
if self.stabilize:
step[:n] *= sqrt(regdu) / col_scale
# Recover dx and dz.
dx = step[:n]
ds = dx[on:]
dz = -z * (1 + ds/s)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
sigma = (muAff/mu)**3
# Incorporate predictor information for corrector step.
comp += ds*dz
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100*mu)
# Assemble right-hand side with centering information.
comp -= sigma * mu
if PredictorCorrector:
# Only update rhs[on:n]; the rest of rhs did not change.
if self.stabilize:
rhs[on:n] += (comp/s - z)/col_scale[on:n]
else:
rhs[on:n] += comp/s - z
else:
rhs[:n] = -dFeas
rhs[on:n] += comp/s
rhs[n:] = -pFeas
# If 'stabilize' is on, must scale right-hand side.
# In the predictor-corrector method, this has already been
# done.
if self.stabilize:
rhs[:n] /= col_scale
rhs[n:] /= sqrt(regdu)
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Unscale step if 'stabilize' is on.
if self.stabilize:
step[:n] *= sqrt(regdu) / col_scale
# Recover step.
dx = step[:n]
ds = dx[on:]
dy = step[n:]
normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx)
step_acceptable = True # Must get rid of this
# End while not step_acceptable
# Recover step in z.
dz = -(comp + z*ds)/s
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0-mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip)
alpha_p *= max(1-mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid)
alpha_d *= max(1-mult, gamma_d)
if ip==id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
if self.verbose:
sys.stdout.write(self.format2 % (mu, alpha_p, alpha_d,
nres, regpr, regdu, rho_q,
del_r, mins, minz, maxs))
# Update primal variables and slacks.
x += alpha_p * dx
# Update dual variables.
y += alpha_d * dy
z += alpha_d * dz
# Update perturbation vectors.
q *= (1-alpha_p) ; q += alpha_p * dx
r *= (1-alpha_d) ; r += alpha_d * dy
qNorm = norm2(q) ; rNorm = norm2(r)
rho_q = regpr * qNorm/(1+self.normc) ; rho_q_min = min(rho_q_min, rho_q)
del_r = regdu * rNorm/(1+self.normb) ; del_r_min = min(del_r_min, del_r)
iter += 1
solve_time = cputime() - solve_time
if self.verbose:
sys.stdout.write('\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = np.dot(self.c, x[:on]) + self.c0
return
def __init__(self, qp, **kwargs):
"""
Solve a convex quadratic program of the form::
minimize c' x + 1/2 x' Q x
subject to A1 x + A2 s = b, (QP)
s >= 0,
where Q is a symmetric positive semi-definite matrix, the variables
x are the original problem variables and s are slack variables. Any
quadratic program may be converted to the above form by instantiation
of the `SlackFramework` class. The conversion to the slack formulation
is mandatory in this implementation.
The method is a variant of Mehrotra's predictor-corrector method where
steps are computed by solving the primal-dual system in augmented form.
Primal and dual regularization parameters may be specified by the user
via the opional keyword arguments `regpr` and `regdu`. Both should be
positive real numbers and should not be "too large". By default they are
set to 1.0 and updated at each iteration.
If `scale` is set to `True`, (QP) is scaled automatically prior to
solution so as to equilibrate the rows and columns of the constraint
matrix [A1 A2].
Advantages of this method are that it is not sensitive to dense columns
in A, no special treatment of the unbounded variables x is required, and
a sparse symmetric quasi-definite system of equations is solved at each
iteration. The latter, although indefinite, possesses a Cholesky-like
factorization. Those properties makes the method typically more robust
that a standard predictor-corrector implementation and the linear system
solves are often much faster than in a traditional interior-point method
in augmented form.
:keywords:
:scale: Perform row and column equilibration of the constraint
matrix [A1 A2] prior to solution (default: `True`).
:regpr: Initial value of primal regularization parameter
(default: `1.0`).
:regdu: Initial value of dual regularization parameter
(default: `1.0`).
:bump_max: Max number of times regularization parameters are
increased when a factorization fails (default 5).
:logger_name: Name of a logger to control output.
:verbose: Turn on verbose mode (default `False`).
"""
if not isinstance(qp, SlackFramework):
msg = 'Input problem must be an instance of SlackFramework'
raise ValueError, msg
# Grab logger if one was configured.
logger_name = kwargs.get('logger_name', 'cqp.solver')
self.log = logging.getLogger(logger_name)
self.verbose = kwargs.get('verbose', True)
scale = kwargs.get('scale', True)
self.qp = qp
self.A = qp.A() # Constraint matrix
if not isinstance(self.A, PysparseMatrix):
self.A = PysparseMatrix(matrix=self.A)
m, n = self.A.shape ; on = qp.original_n
# Record number of slack variables in QP
self.nSlacks = qp.n - on
# Collect basic info about the problem.
zero = np.zeros(n)
self.b = -qp.cons(zero) # Right-hand side
self.c0 = qp.obj(zero) # Constant term in objective
self.c = qp.grad(zero[:on]) # Cost vector
self.Q = PysparseMatrix(matrix=qp.hess(zero[:on],
np.zeros(qp.original_m)))
# Apply in-place problem scaling if requested.
self.prob_scaled = False
if scale:
self.t_scale = cputime()
self.scale()
self.t_scale = cputime() - self.t_scale
else:
# self.scale() sets self.normQ to the Frobenius norm of Q
# and self.normA to the Frobenius norm of A as a by-product.
# If we're not scaling, set normQ and normA manually.
self.normQ = self.Q.matrix.norm('fro')
self.normA = self.A.matrix.norm('fro')
self.normb = norm_infty(self.b)
self.normc = norm_infty(self.c)
self.normbc = 1 + max(self.normb, self.normc)
# Initialize augmented matrix.
self.H = self.initialize_kkt_matrix()
# It will be more efficient to keep the diagonal of Q around.
self.diagQ = self.Q.take(range(qp.original_n))
# We perform the analyze phase on the augmented system only once.
# self.LBL will be initialized in solve().
self.LBL = None
# Set regularization parameters.
self.regpr = kwargs.get('regpr', 1.0) ; self.regpr_min = 1.0e-8
self.regdu = kwargs.get('regdu', 1.0) ; self.regdu_min = 1.0e-8
# Max number of times regularization parameters are increased.
self.bump_max = kwargs.get('bump_max', 5)
# Check input parameters.
if self.regpr < 0.0: self.regpr = 0.0
if self.regdu < 0.0: self.regdu = 0.0
# Initialize format strings for display
fmt_hdr = '%-4s %9s' + ' %-8s'*6 + ' %-7s %-4s %-4s' + ' %-8s'*8
self.header = fmt_hdr % ('Iter', 'Cost', 'pResid', 'dResid', 'cResid',
'rGap', 'qNorm', 'rNorm', 'Mu', 'AlPr', 'AlDu',
'LS Resid', 'RegPr', 'RegDu', 'Rho q', 'Del r',
'Min(s)', 'Min(z)', 'Max(s)')
self.format1 = '%-4d %9.2e'
self.format1 += ' %-8.2e' * 6
self.format2 = ' %-7.1e %-4.2f %-4.2f'
self.format2 += ' %-8.2e' * 8
self.cond_history = []
self.berr_history = []
self.derr_history = []
self.nrms_history = []
self.lres_history = []
if self.verbose: self.display_stats()
return
def __init__(self, qp, **kwargs):
"""
Solve a convex quadratic program of the form::
minimize c' x + 1/2 x' Q x
subject to A1 x + A2 s = b, (QP)
s >= 0,
where Q is a symmetric positive semi-definite matrix, the variables
x are the original problem variables and s are slack variables. Any
quadratic program may be converted to the above form by instantiation
of the `SlackFramework` class. The conversion to the slack formulation
is mandatory in this implementation.
The method is a variant of Mehrotra's predictor-corrector method where
steps are computed by solving the primal-dual system in augmented form.
Primal and dual regularization parameters may be specified by the user
via the opional keyword arguments `regpr` and `regdu`. Both should be
positive real numbers and should not be "too large". By default they are
set to 1.0 and updated at each iteration.
If `scale` is set to `True`, (QP) is scaled automatically prior to
solution so as to equilibrate the rows and columns of the constraint
matrix [A1 A2].
Advantages of this method are that it is not sensitive to dense columns
in A, no special treatment of the unbounded variables x is required, and
a sparse symmetric quasi-definite system of equations is solved at each
iteration. The latter, although indefinite, possesses a Cholesky-like
factorization. Those properties makes the method typically more robust
that a standard predictor-corrector implementation and the linear system
solves are often much faster than in a traditional interior-point method
in augmented form.
:keywords:
:scale: Perform row and column equilibration of the constraint
matrix [A1 A2] prior to solution (default: `True`).
:regpr: Initial value of primal regularization parameter
(default: `1.0`).
:regdu: Initial value of dual regularization parameter
(default: `1.0`).
:verbose: Turn on verbose mode (default `False`).
"""
if not isinstance(qp, SlackFramework):
msg = 'Input problem must be an instance of SlackFramework'
raise ValueError, msg
self.verbose = kwargs.get('verbose', True)
scale = kwargs.get('scale', True)
self.qp = qp
self.A = qp.A() # Constraint matrix
if not isinstance(self.A, PysparseMatrix):
self.A = PysparseMatrix(matrix=self.A)
m, n = self.A.shape
on = qp.original_n
# Record number of slack variables in QP
self.nSlacks = qp.n - on
# Collect basic info about the problem.
zero = np.zeros(n)
self.b = -qp.cons(zero) # Right-hand side
self.c0 = qp.obj(zero) # Constant term in objective
self.c = qp.grad(zero[:on]) # Cost vector
self.Q = PysparseMatrix(
matrix=qp.hess(zero[:on], np.zeros(qp.original_m)))
# Apply in-place problem scaling if requested.
self.prob_scaled = False
if scale:
self.t_scale = cputime()
self.scale()
self.t_scale = cputime() - self.t_scale
else:
# self.scale() sets self.normQ to the Frobenius norm of Q
# as a by-product. If we're not scaling, set normQ manually.
self.normQ = self.Q.matrix.norm('fro')
self.normb = norm_infty(self.b)
self.normc = norm_infty(self.c)
self.normbc = 1 + max(self.normb, self.normc)
# Initialize augmented matrix
self.H = PysparseMatrix(size=n + m,
sizeHint=n + m + self.A.nnz + self.Q.nnz,
symmetric=True)
# The (1,1) block will always be Q (save for its diagonal).
self.H[:on, :on] = -self.Q
# The (2,1) block will always be A. We store it now once and for all.
self.H[n:, :n] = self.A
# It will be more efficient to keep the diagonal of Q around.
self.diagQ = self.Q.take(range(qp.original_n))
# We perform the analyze phase on the augmented system only once.
# self.LBL will be initialized in solve().
self.LBL = None
# Set regularization parameters.
self.regpr = kwargs.get('regpr', 1.0)
self.regpr_min = 1.0e-8
self.regdu = kwargs.get('regdu', 1.0)
self.regdu_min = 1.0e-8
# Check input parameters.
if self.regpr < 0.0: self.regpr = 0.0
if self.regdu < 0.0: self.regdu = 0.0
# Initialize format strings for display
fmt_hdr = '%-4s %9s' + ' %-8s' * 6 + ' %-7s %-4s %-4s' + ' %-8s' * 8
self.header = fmt_hdr % ('Iter', 'Cost', 'pResid', 'dResid', 'cResid',
'rGap', 'qNorm', 'rNorm', 'Mu', 'AlPr',
'AlDu', 'LS Resid', 'RegPr', 'RegDu', 'Rho q',
'Del r', 'Min(s)', 'Min(z)', 'Max(s)')
self.format1 = '%-4d %9.2e'
self.format1 += ' %-8.2e' * 6
self.format2 = ' %-7.1e %-4.2f %-4.2f'
self.format2 += ' %-8.2e' * 8 + '\n'
if self.verbose: self.display_stats()
return
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations (default: 10n)
:tolerance: Stopping tolerance (default: 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default: `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
Upon exit, the following members of the class instance are set:
* x..............final iterate
* y..............final value of the Lagrange multipliers associated
to A1 x + A2 s = b
* z..............final value of the Lagrange multipliers associated
to s>=0
* obj_value......final cost
* iter...........total number of iterations
* kktResid.......final relative residual
* solve_time.....time to solve the QP
* status.........string describing the exit status.
* short_status...short version of status, used for printing.
"""
qp = self.qp
itermax = kwargs.get('itermax', max(100, 10 * qp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape
on = qp.original_n
A = self.A
b = self.b
c = self.c
Q = self.Q
diagQ = self.diagQ
H = self.H
regpr = self.regpr
regdu = self.regdu
regpr_min = self.regpr_min
regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
(x, y, z) = self.set_initial_guess(self.qp, **kwargs)
# Slack variables are the trailing variables in x.
s = x[on:]
ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
col_scale = np.empty(n)
# Allocate room for right-hand side of linear systems.
rhs = np.zeros(n + m)
finished = False
iter = 0
setup_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if self.verbose and iter % 20 == 0:
sys.stdout.write('\n' + self.header + '\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Compute residuals.
pFeas = A * x - b
comp = s * z
sz = sum(comp) # comp = S z
Qx = Q * x[:on]
dFeas = y * A
dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx
dFeas[on:] += z # dFeas2 = A2'y + z
# Compute duality measure.
if ns > 0:
mu = sz / ns
else:
mu = 0.0
# Compute residual norms and scaled residual norms.
#pResid = norm_infty(pFeas + regdu * r)/(1+self.normc+self.normQ)
#dResid = norm_infty(dFeas - regpr * q)/(1+self.normb+self.normQ)
pResid = norm2(pFeas)
spResid = pResid / (1 + self.normc + self.normQ)
dResid = norm2(dFeas)
sdResid = dResid / (1 + self.normb + self.normQ)
if ns > 0:
cResid = norm_infty(comp) / (self.normbc + self.normQ)
else:
cResid = 0.0
# Compute relative duality gap.
cx = np.dot(c, x[:on])
xQx = np.dot(x[:on], Qx)
by = np.dot(b, y)
rgap = cx + xQx - by
#rgap += regdu * (rNorm**2 + np.dot(r,y))
rgap = abs(rgap) / (1 + abs(cx) + self.normQ)
rgap2 = mu / (1 + abs(cx) + self.normQ)
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
#kktResid = max(pResid, cResid, dResid)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
# Should probably get rid of q when regpr=0 and of r when regdu=0.
if iter == 0:
if regpr > 0:
q = dFeas / regpr
qNorm = dResid / regpr
rho_q = dResid
else:
q = dFeas
qNorm = dResid
rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas / regdu
rNorm = pResid / regdu
del_r = pResid
else:
r = -pFeas
rNorm = pResid
del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
regdu = min(regdu / 10, sz / normdy / 10, (sz / normdy)**(1.1))
regdu = max(regdu, regdu_min)
regpr = min(regpr / 10, sz / normdx / 10, (sz / normdx)**(1.1))
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter - 1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter - 1:
if du_infeas_count > 6:
status = 'Problem seems to be (locally) primal infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
# Display objective and residual data.
if self.verbose:
sys.stdout.write(self.format1 %
(iter, cx + 0.5 * xQx, pResid, dResid, cResid,
rgap, qNorm, rNorm))
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status = 'iter'
finished = True
continue
# Record some quantities for display
if ns > 0:
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
else:
mins = minz = maxs = 0
# Solve the linear system
#
# [ -(Q+pI) 0 A1' ] [∆x] = [c + Q x - A1' y ]
# [ 0 -(S^{-1} Z + pI) A2' ] [∆s] [ - A2' y - µ S^{-1} e]
# [ A1 A2 dI ] [∆y] [ b - A1 x - A2 s ]
#
# where s are the slack variables, p is the primal regularization
# parameter, d is the dual regularization parameter, and
# A = [ A1 A2 ] where the columns of A1 correspond to the original
# problem variables and those of A2 correspond to slack variables.
#
# We recover ∆z = -z - S^{-1} (Z ∆s + µ e).
# Compute augmented matrix and factorize it.
factorized = False
nb_bump = 0
while not factorized and nb_bump < 5:
H.put(-diagQ - regpr, range(on))
H.put(-z / s - regpr, range(on, n))
H.put(regdu, range(n, n + m))
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up the
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
sys.stderr.write('Primal-Dual Matrix Rank Deficient')
sys.stderr.write('... bumping up reg parameters\n')
regpr *= 100
regdu *= 100
nb_bump += 1
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and nb_bump == 5:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
rhs[:n] = -dFeas
rhs[on:n] += z
rhs[n:] = -pFeas
(step, nres, neig) = self.solveSystem(rhs)
# Recover dx and dz.
dx = step[:n]
ds = dx[on:]
dz = -z * (1 + ds / s)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns
sigma = (muAff / mu)**3
# Incorporate predictor information for corrector step.
comp += ds * dz
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100 * mu)
# Assemble right-hand side with centering information.
comp -= sigma * mu
if PredictorCorrector:
# Only update rhs[on:n]; the rest of the vector did not change.
rhs[on:n] += comp / s - z
else:
rhs[:n] = -dFeas
rhs[on:n] += comp / s
rhs[n:] = -pFeas
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Recover step.
dx = step[:n]
ds = dx[on:]
dy = step[n:]
dz = -(comp + z * ds) / s
normds = norm2(ds)
normdy = norm2(dy)
normdx = norm2(dx)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0 - mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult * mu_tmp - s[ip] * zip) / (alpha_p *
ds[ip] * zip)
alpha_p *= max(1 - mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult * mu_tmp - z[id] * sid) / (alpha_d *
dz[id] * sid)
alpha_d *= max(1 - mult, gamma_d)
if ip == id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
if self.verbose:
sys.stdout.write(self.format2 %
(mu, alpha_p, alpha_d, nres, regpr, regdu,
rho_q, del_r, mins, minz, maxs))
# Update iterates and perturbation vectors.
x += alpha_p * dx # This also updates slack variables.
y += alpha_d * dy
z += alpha_d * dz
q *= (1 - alpha_p)
q += alpha_p * dx
r *= (1 - alpha_d)
r += alpha_d * dy
qNorm = norm2(q)
rNorm = norm2(r)
rho_q = regpr * qNorm / (1 + self.normc)
rho_q_min = min(rho_q_min, rho_q)
del_r = regdu * rNorm / (1 + self.normb)
del_r_min = min(del_r_min, del_r)
iter += 1
solve_time = cputime() - setup_time
if self.verbose:
sys.stdout.write('\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid
self.cResid = cResid
self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = self.c0 + cx + 0.5 * xQx
return
def test_pycfs(ProblemList, plist=[0, 2, 5, 10], latex=False):
if len(ProblemList) == 0:
usage()
sys.exit(1)
if latex:
# For LaTeX output
fmt = '%-12s & %-5s & %-6s & %-2s & %-4s & %-8s & %-4s & %-8s & %-6s & %-6s\\\\\n'
fmt1 = '%-12s & %-5d & %-6d & '
fmt2 = '%-2d & %-4d & %-8.1e & %-4d & %-8.1e & %-6.2f & %-6.2f\\\\\n'
hline = '\\hline\n'
skip = '&&&'
else:
# For ASCII output
fmt = '%-12s %-5s %-6s %-2s %-4s %-8s %-4s %-8s %-6s %-6s\n'
fmt1 = '%-12s %-5d %-6d '
fmt2 = '%-2d %-4d %-8.1e %-4d %-8.1e %-6.2f %-6.2f\n'
skip = ' ' * 26
header = fmt % ('Name','Size','nnz','p','info','shift','iter','relResid','fact','solve')
lhead = len(header)
if not latex: hline = '-' * lhead + '\n'
sys.stderr.write(hline + header + hline)
time_list = {} # Record timings
iter_list = {} # Record number of iterations
for problem in ProblemList:
A = spmatrix.ll_mat_from_mtx(problem)
(m, n) = A.shape
if m != n: break
prob = os.path.basename(problem)
if prob[-4:] == '.mtx': prob = prob[:-4]
# Right-hand side is Ae, as in Icfs.
e = numpy.ones(n, 'd');
b = numpy.ones(n, 'd');
A.matvec(e, b)
sys.stdout.write(fmt1 % (prob, n, A.nnz))
advance = False
# Call icfs and pcg
tlist = []
ilist = []
for pval in plist:
t0 = cputime()
P = pycfs.PycfsContext(A, mem=pval)
t_fact = cputime() - t0
P.solve(b)
t_solve = P.tsolve
tlist.append(t_fact + t_solve)
ilist.append(P.iter)
if advance: sys.stdout.write(skip)
sys.stdout.write(fmt2 % (pval, P.info, P.shift, P.iter, P.relres, t_fact, t_solve))
advance = True
time_list[prob] = tlist
iter_list[prob] = ilist
sys.stderr.write(hline)
return (time_list,iter_list)
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations (default: 10n)
:tolerance: Stopping tolerance (default: 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default: `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
Upon exit, the following members of the class instance are set:
* x..............final iterate
* y..............final value of the Lagrange multipliers associated
to A1 x + A2 s = b
* z..............final value of the Lagrange multipliers associated
to s>=0
* obj_value......final cost
* iter...........total number of iterations
* kktResid.......final relative residual
* solve_time.....time to solve the QP
* status.........string describing the exit status.
* short_status...short version of status, used for printing.
"""
qp = self.qp
itermax = kwargs.get('itermax', max(100,10*qp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape ; on = qp.original_n
A = self.A ; b = self.b ; c = self.c ; Q = self.Q ; diagQ = self.diagQ
H = self.H
regpr = self.regpr ; regdu = self.regdu
regpr_min = self.regpr_min ; regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
(x,y,z) = self.set_initial_guess(self.qp, **kwargs)
# Slack variables are the trailing variables in x.
s = x[on:] ; ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
col_scale = np.empty(n)
# Allocate room for right-hand side of linear systems.
rhs = np.zeros(n+m)
finished = False
iter = 0
setup_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if self.verbose and iter % 20 == 0:
sys.stdout.write('\n' + self.header + '\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Compute residuals.
pFeas = A*x - b
comp = s*z ; sz = sum(comp) # comp = S z
Qx = Q*x[:on]
dFeas = y*A ; dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx
dFeas[on:] += z # dFeas2 = A2'y + z
# Compute duality measure.
if ns > 0:
mu = sz/ns
else:
mu = 0.0
# Compute residual norms and scaled residual norms.
#pResid = norm_infty(pFeas + regdu * r)/(1+self.normc+self.normQ)
#dResid = norm_infty(dFeas - regpr * q)/(1+self.normb+self.normQ)
pResid = norm2(pFeas) ; spResid = pResid/(1+self.normc+self.normQ)
dResid = norm2(dFeas) ; sdResid = dResid/(1+self.normb+self.normQ)
if ns > 0:
cResid = norm_infty(comp)/(self.normbc+self.normQ)
else:
cResid = 0.0
# Compute relative duality gap.
cx = np.dot(c,x[:on])
xQx = np.dot(x[:on],Qx)
by = np.dot(b,y)
rgap = cx + xQx - by
#rgap += regdu * (rNorm**2 + np.dot(r,y))
rgap = abs(rgap) / (1 + abs(cx) + self.normQ)
rgap2 = mu / (1 + abs(cx) + self.normQ)
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
#kktResid = max(pResid, cResid, dResid)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
# Should probably get rid of q when regpr=0 and of r when regdu=0.
if iter == 0:
if regpr > 0:
q = dFeas/regpr ; qNorm = dResid/regpr ; rho_q = dResid
else:
q = dFeas ; qNorm = dResid ; rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas/regdu ; rNorm = pResid/regdu ; del_r = pResid
else:
r = -pFeas ; rNorm = pResid ; del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
regdu = min(regdu/10, sz/normdy/10, (sz/normdy)**(1.1))
regdu = max(regdu, regdu_min)
regpr = min(regpr/10, sz/normdx/10, (sz/normdx)**(1.1))
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter-1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter-1:
if du_infeas_count > 6:
status='Problem seems to be (locally) primal infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
# Display objective and residual data.
if self.verbose:
sys.stdout.write(self.format1 % (iter, cx + 0.5 * xQx, pResid,
dResid, cResid, rgap, qNorm,
rNorm))
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status = 'iter'
finished = True
continue
# Record some quantities for display
if ns > 0:
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
else:
mins = minz = maxs = 0
# Solve the linear system
#
# [ -(Q+pI) 0 A1' ] [∆x] = [c + Q x - A1' y ]
# [ 0 -(S^{-1} Z + pI) A2' ] [∆s] [ - A2' y - µ S^{-1} e]
# [ A1 A2 dI ] [∆y] [ b - A1 x - A2 s ]
#
# where s are the slack variables, p is the primal regularization
# parameter, d is the dual regularization parameter, and
# A = [ A1 A2 ] where the columns of A1 correspond to the original
# problem variables and those of A2 correspond to slack variables.
#
# We recover ∆z = -z - S^{-1} (Z ∆s + µ e).
# Compute augmented matrix and factorize it.
factorized = False
nb_bump = 0
while not factorized and nb_bump < 5:
H.put(-diagQ - regpr, range(on))
H.put(-z/s - regpr, range(on,n))
H.put(regdu, range(n,n+m))
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up the
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
sys.stderr.write('Primal-Dual Matrix Rank Deficient')
sys.stderr.write('... bumping up reg parameters\n')
regpr *= 100 ; regdu *= 100
nb_bump += 1
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and nb_bump == 5:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
rhs[:n] = -dFeas
rhs[on:n] += z
rhs[n:] = -pFeas
(step, nres, neig) = self.solveSystem(rhs)
# Recover dx and dz.
dx = step[:n]
ds = dx[on:]
dz = -z * (1 + ds/s)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
sigma = (muAff/mu)**3
# Incorporate predictor information for corrector step.
comp += ds*dz
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100*mu)
# Assemble right-hand side with centering information.
comp -= sigma * mu
if PredictorCorrector:
# Only update rhs[on:n]; the rest of the vector did not change.
rhs[on:n] += comp/s - z
else:
rhs[:n] = -dFeas
rhs[on:n] += comp/s
rhs[n:] = -pFeas
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Recover step.
dx = step[:n]
ds = dx[on:]
dy = step[n:]
dz = -(comp + z*ds)/s
normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0-mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip)
alpha_p *= max(1-mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid)
alpha_d *= max(1-mult, gamma_d)
if ip==id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
if self.verbose:
sys.stdout.write(self.format2 % (mu, alpha_p, alpha_d,
nres, regpr, regdu, rho_q,
del_r, mins, minz, maxs))
# Update iterates and perturbation vectors.
x += alpha_p * dx # This also updates slack variables.
y += alpha_d * dy
z += alpha_d * dz
q *= (1-alpha_p) ; q += alpha_p * dx
r *= (1-alpha_d) ; r += alpha_d * dy
qNorm = norm2(q) ; rNorm = norm2(r)
rho_q = regpr * qNorm/(1+self.normc) ; rho_q_min = min(rho_q_min, rho_q)
del_r = regdu * rNorm/(1+self.normb) ; del_r_min = min(del_r_min, del_r)
iter += 1
solve_time = cputime() - setup_time
if self.verbose:
sys.stdout.write('\n')
sys.stdout.write('-' * len(self.header) + '\n')
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = self.c0 + cx + 0.5 * xQx
return
def Solve(self):
if self.A is not None:
if self.factorize and not self.factorized: self.Factorize()
if self.b is not None: self.FindFeasible()
n = self.n
m = self.m
xNorm2 = 0.0 # Squared norm of current iterate x, not counting x_feas
# Obtain initial projected residual
self.t_solve = cputime()
if self.A is not None:
if self.b is not None:
self.rhs[:n] = self.c + self.H * self.x_feasible
self.rhs[n:] = 0.0
else:
self.rhs[:n] = self.c
self.Proj.solve( self.rhs )
r = g = self.Proj.x[:n]
self.v = self.Proj.x[n:]
#self.CheckAccurate()
else:
g = self.c
r = g.copy()
# Initialize search direction
p = -g
pHp = None
self.residNorm0 = numpy.dot(r,g)
rg = self.residNorm0
threshold = max( self.abstol, self.reltol * sqrt(self.residNorm0) )
iter = 0
onBoundary = False
if self.debug:
self._write( self.header )
self._write( '-' * len(self.header) + '\n' )
self._write( self.fmt1 % (iter, rg) )
while sqrt(rg) > threshold and iter < self.maxiter and not onBoundary:
Hp = self.H * p
pHp = numpy.dot(p,Hp)
# Display current iteration info
if self.debug: self._write( self.fmt % (iter, rg, pHp) )
if self.radius is not None:
# Compute steplength to the boundary
sigma = self.to_boundary(self.x, p, self.radius, ss=xNorm2)
elif pHp <= 0.0:
self._write('Problem is not second-order sufficient\n')
status = 'problem not SOS'
self.infDescent = True
self.dir = p
continue
alpha = rg/pHp
if self.radius is not None and (pHp <= 0.0 or alpha > sigma):
# p is a direction of singularity or negative curvature or
# next iterate will lie past the boundary of the trust region
# Move to boundary of trust-region
self.x += sigma * p
xNorm2 = self.radius * self.radius
status = 'on boundary (sigma = %g)' % sigma
self.infDescent = True
onBoundary = True
continue
# Make sure nonnegativity bounds remain enforced, if requested
if (self.btol is not None) and (self.cur_iter is not None):
stepBnd = self.ftb(self.x, p)
if stepBnd < alpha:
self.x += stepBnd * p
status = 'on boundary'
onBoundary = True
continue
# Move on
self.x += alpha * p
r += alpha * Hp
if self.A is not None:
# Project current residual
self.rhs[:n] = r
self.Proj.solve( self.rhs )
# Perform actual iterative refinement, if necessary
#self.Proj.refine( self.rhs, nitref=self.max_itref,
# tol=self.itref_tol )
# Obtain new projected gradient
g = self.Proj.x[:n]
if self.precon is not None:
# Prepare for iterative semi-refinement
self.A.matvec_transp( self.Proj.x[n:], self.v )
else:
g = r
rg_next = numpy.dot(r,g)
beta = rg_next/rg
p = -g + beta * p
if self.precon is not None:
# Perform iterative semi-refinement
r = r - self.v
else:
r = g
rg = rg_next
if self.radius is not None:
xNorm2 = numpy.dot( self.x, self.x )
iter += 1
# Output info about the last iteration
if self.debug and iter > 0:
self._write( self.fmt % (iter, rg, pHp) )
# Obtain final solution x
self.xNorm2 = xNorm2
self.stepNorm = sqrt(xNorm2)
if self.x_feasible is not None:
self.x += self.x_feasible
if self.A is not None:
# Find (weighted) least-squares Lagrange multipliers
self.rhs[:n] = - self.c - self.H * self.x
self.rhs[n:] = 0.0
self.Proj.solve( self.rhs )
self.v = self.Proj.x[n:].copy()
self.t_solve = cputime() - self.t_solve
self.step = self.x # Alias for consistency with TruncatedCG.
self.onBoundary = onBoundary
self.converged = (iter < self.maxiter)
if iter < self.maxiter and not onBoundary:
status = 'residual small'
elif iter >= self.maxiter:
status = 'max iter'
self.iter = iter
self.nMatvec = iter
self.residNorm = sqrt(rg)
self.status = status
return
def test_pycfs(ProblemList, plist=[0, 2, 5, 10], latex=False):
if len(ProblemList) == 0:
usage()
sys.exit(1)
if latex:
# For LaTeX output
fmt = '%-12s & %-5s & %-6s & %-2s & %-4s & %-8s & %-4s & %-8s & %-6s & %-6s\\\\\n'
fmt1 = '%-12s & %-5d & %-6d & '
fmt2 = '%-2d & %-4d & %-8.1e & %-4d & %-8.1e & %-6.2f & %-6.2f\\\\\n'
hline = '\\hline\n'
skip = '&&&'
else:
# For ASCII output
fmt = '%-12s %-5s %-6s %-2s %-4s %-8s %-4s %-8s %-6s %-6s\n'
fmt1 = '%-12s %-5d %-6d '
fmt2 = '%-2d %-4d %-8.1e %-4d %-8.1e %-6.2f %-6.2f\n'
skip = ' ' * 26
header = fmt % ('Name', 'Size', 'nnz', 'p', 'info', 'shift', 'iter',
'relResid', 'fact', 'solve')
lhead = len(header)
if not latex: hline = '-' * lhead + '\n'
sys.stderr.write(hline + header + hline)
time_list = {} # Record timings
iter_list = {} # Record number of iterations
for problem in ProblemList:
A = spmatrix.ll_mat_from_mtx(problem)
(m, n) = A.shape
if m != n: break
prob = os.path.basename(problem)
if prob[-4:] == '.mtx': prob = prob[:-4]
# Right-hand side is Ae, as in Icfs.
e = numpy.ones(n, 'd')
b = numpy.ones(n, 'd')
A.matvec(e, b)
sys.stdout.write(fmt1 % (prob, n, A.nnz))
advance = False
# Call icfs and pcg
tlist = []
ilist = []
for pval in plist:
t0 = cputime()
P = pycfs.PycfsContext(A, mem=pval)
t_fact = cputime() - t0
P.solve(b)
t_solve = P.tsolve
tlist.append(t_fact + t_solve)
ilist.append(P.iter)
if advance: sys.stdout.write(skip)
sys.stdout.write(
fmt2 %
(pval, P.info, P.shift, P.iter, P.relres, t_fact, t_solve))
advance = True
time_list[prob] = tlist
iter_list[prob] = ilist
sys.stderr.write(hline)
return (time_list, iter_list)
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations (default: 10n)
:tolerance: Stopping tolerance (default: 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default: `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
:returns:
:x: final iterate
:y: final value of the Lagrange multipliers associated
to `A1 x + A2 s = b`
:z: final value of the Lagrange multipliers associated
to `s >= 0`
:obj_value: final cost
:iter: total number of iterations
:kktResid: final relative residual
:solve_time: time to solve the QP
:status: string describing the exit status.
:short_status: short version of status, used for printing.
"""
qp = self.qp
itermax = kwargs.get('itermax', max(100, 10 * qp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape
on = qp.original_n
A = self.A
b = self.b
c = self.c
Q = self.Q
diagQ = self.diagQ
H = self.H
regpr = self.regpr
regdu = self.regdu
regpr_min = self.regpr_min
regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
(x, y, z) = self.set_initial_guess(**kwargs)
# Slack variables are the trailing variables in x.
s = x[on:]
ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
# Allocate room for right-hand side of linear systems.
rhs = self.initialize_rhs()
finished = False
iter = 0
setup_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if iter % 50 == 0:
self.log.info(self.header)
self.log.info('-' * len(self.header))
# Compute residuals.
pFeas = A * x - b
comp = s * z
sz = sum(comp) # comp = Sz
Qx = Q * x[:on]
dFeas = y * A
dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx
dFeas[on:] += z # dFeas2 = A2'y + z
# Compute duality measure.
if ns > 0:
mu = sz / ns
else:
mu = 0.0
# Compute residual norms and scaled residual norms.
pResid = norm2(pFeas)
spResid = pResid / (1 + self.normb + self.normA + self.normQ)
dResid = norm2(dFeas)
sdResid = dResid / (1 + self.normc + self.normA + self.normQ)
if ns > 0:
cResid = norm_infty(comp) / (self.normbc + self.normA +
self.normQ)
else:
cResid = 0.0
# Compute relative duality gap.
cx = np.dot(c, x[:on])
xQx = np.dot(x[:on], Qx)
by = np.dot(b, y)
rgap = cx + xQx - by
rgap = abs(rgap) / (1 + abs(cx) + self.normA + self.normQ)
rgap2 = mu / (1 + abs(cx) + self.normA + self.normQ)
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
# Should probably get rid of q when regpr=0 and of r when regdu=0.
if iter == 0:
if regpr > 0:
q = dFeas / regpr
qNorm = dResid / regpr
rho_q = dResid
else:
q = dFeas
qNorm = dResid
rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas / regdu
rNorm = pResid / regdu
del_r = pResid
else:
r = -pFeas
rNorm = pResid
del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
if regdu > 0:
regdu = regdu / 10
regdu = max(regdu, regdu_min)
if regpr > 0:
regpr = regpr / 10
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
if mu < tolerance/100 * mu0 and \
rho_q > 1./tolerance/1.0e+6 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter - 1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual'
status += ' infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
else:
pr_infeas_count = 0
if mu < tolerance/100 * mu0 and \
del_r > 1./tolerance/1.0e+6 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter - 1:
if du_infeas_count > 6:
status = 'Problem seems to be (locally) primal'
status += ' infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
else:
du_infeas_count = 0
# Display objective and residual data.
output_line = self.format1 % (iter, cx + 0.5 * xQx, pResid, dResid,
cResid, rgap, qNorm, rNorm)
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status = 'iter'
finished = True
continue
# Record some quantities for display
if ns > 0:
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
else:
mins = minz = maxs = 0
# Compute augmented matrix and factorize it.
factorized = False
degenerate = False
nb_bump = 0
while not factorized and not degenerate:
self.update_linear_system(s, z, regpr, regdu)
self.log.debug('Factorizing')
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up the
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
self.log.info('Primal-Dual Matrix Rank Deficient' + \
'... bumping up reg parameters')
if regpr == 0. and regdu == 0.:
degenerate = True
else:
if regpr > 0:
regpr *= 100
if regdu > 0:
regdu *= 100
nb_bump += 1
degenerate = nb_bump > self.bump_max
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and degenerate:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
self.set_affine_scaling_rhs(rhs, pFeas, dFeas, s, z)
(step, nres, neig) = self.solveSystem(rhs)
# Recover dx and dz.
dx, ds, dy, dz = self.get_affine_scaling_dxsyz(
step, x, s, y, z)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns
sigma = (muAff / mu)**3
# Incorporate predictor information for corrector step.
# Only update rhs[on:n]; the rest of the vector did not change.
comp += ds * dz
comp -= sigma * mu
self.update_corrector_rhs(rhs, s, z, comp)
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100 * mu)
comp -= sigma * mu
# Assemble rhs.
self.update_long_step_rhs(rhs, pFeas, dFeas, comp, s)
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Recover step.
dx, ds, dy, dz = self.get_dxsyz(step, x, s, y, z, comp)
normds = norm2(ds)
normdy = norm2(dy)
normdx = norm2(dx)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0 - mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic.
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult * mu_tmp - s[ip] * zip) / (alpha_p *
ds[ip] * zip)
alpha_p *= max(1 - mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult * mu_tmp - z[id] * sid) / (alpha_d *
dz[id] * sid)
alpha_d *= max(1 - mult, gamma_d)
if ip == id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
output_line += self.format2 % (mu, alpha_p, alpha_d, nres, regpr,
regdu, rho_q, del_r, mins, minz,
maxs)
self.log.info(output_line)
# Update iterates and perturbation vectors.
x += alpha_p * dx # This also updates slack variables.
y += alpha_d * dy
z += alpha_d * dz
q *= (1 - alpha_p)
q += alpha_p * dx
r *= (1 - alpha_d)
r += alpha_d * dy
qNorm = norm2(q)
rNorm = norm2(r)
if regpr > 0:
rho_q = regpr * qNorm / (1 + self.normc)
rho_q_min = min(rho_q_min, rho_q)
else:
rho_q = 0.0
if regdu > 0:
del_r = regdu * rNorm / (1 + self.normb)
del_r_min = min(del_r_min, del_r)
else:
del_r = 0.0
iter += 1
solve_time = cputime() - setup_time
self.log.info('-' * len(self.header))
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid
self.cResid = cResid
self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = self.c0 + cx + 0.5 * xQx
return
def __init__(self, lp, **kwargs):
"""
Solve a linear program of the form::
minimize c' x subject to A1 x + A2 s = b and s >= 0, (LP)
where the variables x are the original problem variables and s are
slack variables. Any linear program may be converted to the above form
by instantiation of the `SlackFramework` class. The conversion to the
slack formulation is mandatory in this implementation.
The method is a variant of Mehrotra's predictor-corrector method where
steps are computed by solving the primal-dual system in augmented form.
Primal and dual regularization parameters may be specified by the user
via the opional keyword arguments `regpr` and `regdu`. Both should be
positive real numbers and should not be "too large". By default they are
set to 1.0 and updated at each iteration.
If `scale` is set to `True`, (LP) is scaled automatically prior to
solution so as to equilibrate the rows and columns of the constraint
matrix [A1 A2].
Advantages of this method are that it is not sensitive to dense columns
in A, no special treatment of the unbounded variables x is required, and
a sparse symmetric quasi-definite system of equations is solved at each
iteration. The latter, although indefinite, possesses a Cholesky-like
factorization. Those properties makes the method typically more robust
that a standard predictor-corrector implementation and the linear system
solves are often much faster than in a traditional interior-point method
in augmented form.
:keywords:
:scale: Perform row and column equilibration of the constraint
matrix [A1 A2] prior to solution (default: `True`).
:stabilize: Scale the linear system to be solved at each iteration
(default: `True`).
:regpr: Initial value of primal regularization parameter
(default: `1.0`).
:regdu: Initial value of dual regularization parameter
(default: `1.0`).
:verbose: Turn on verbose mode (default `False`).
"""
if not isinstance(lp, SlackFramework):
msg = 'Input problem must be an instance of SlackFramework'
raise ValueError, msg
scale = kwargs.get('scale', True)
self.verbose = kwargs.get('verbose', True)
self.stabilize = kwargs.get('stabilize', True)
self.lp = lp
self.A = lp.A() # Constraint matrix
if not isinstance(self.A, PysparseMatrix):
self.A = PysparseMatrix(matrix=self.A)
m, n = self.A.shape
# Record number of slack variables in LP
self.nSlacks = lp.n - lp.original_n
# Constant vectors
zero = np.zeros(n)
self.b = -lp.cons(zero) # Right-hand side
self.c0 = lp.obj(zero) # Constant term in objective
self.c = lp.grad(zero[:lp.original_n]) #lp.cost() # Cost vector
# Apply in-place problem scaling if requested.
self.prob_scaled = False
if scale:
self.t_scale = cputime()
self.scale()
self.t_scale = cputime() - self.t_scale
self.normb = norm2(self.b)
self.normc = norm2(self.c)
self.normbc = 1 + max(self.normb, self.normc)
# Initialize augmented matrix
self.H = PysparseMatrix(size=n+m,
sizeHint=n+m+self.A.nnz,
symmetric=True)
# We perform the analyze phase on the augmented system only once.
# self.LBL will be initialized in set_initial_guess().
self.LBL = None
self.regpr = kwargs.get('regpr', 1.0) ; self.regpr_min = 1.0e-8
self.regdu = kwargs.get('regdu', 1.0) ; self.regdu_min = 1.0e-8
# Check input parameters.
if self.regpr < 0.0: self.regpr = 0.0
if self.regdu < 0.0: self.regdu = 0.0
# Dual regularization is necessary for stabilization.
if self.regdu == 0.0:
sys.stderr.write('Warning: No dual regularization in effect\n')
sys.stderr.write(' Stabilization has been turned off\n')
self.stabilize = False
# Initialize format strings for display
fmt_hdr = '%-4s %9s' + ' %-8s'*6 + ' %-7s %-4s %-4s' + ' %-8s'*8
self.header = fmt_hdr % ('Iter', 'Cost', 'pResid', 'dResid', 'cResid',
'rGap', 'qNorm', 'rNorm', 'Mu', 'AlPr', 'AlDu',
'LS Resid', 'RegPr', 'RegDu', 'Rho q', 'Del r',
'Min(s)', 'Min(z)', 'Max(s)')
self.format1 = '%-4d %9.2e'
self.format1 += ' %-8.2e' * 6
self.format2 = ' %-7.1e %-4.2f %-4.2f'
self.format2 += ' %-8.2e' * 8 + '\n'
if self.verbose: self.display_stats()
return
def gltr_explicit(H, g, **kwargs):
G = pygltr.PyGltrContext(g, **kwargs)
t = cputime()
G.explicit_solve(H.to_csr())
t = cputime() - t
return (G.m, G.mult, G.snorm, G.niter, G.nc, G.ierr, t)
def gltr_explicit(H, g, **kwargs):
G = pygltr.PyGltrContext(g, **kwargs)
t = cputime()
G.explicit_solve(H.to_csr())
t = cputime() - t
return (G.m, G.mult, G.snorm, G.niter, G.nc, G.ierr, t)
def Solve(self):
if self.A is not None:
if self.factorize and not self.factorized: self.Factorize()
if self.b is not None: self.FindFeasible()
n = self.n
m = self.m
xNorm2 = 0.0 # Squared norm of current iterate x, not counting x_feas
# Obtain initial projected residual
self.t_solve = cputime()
if self.A is not None:
if self.b is not None:
self.rhs[:n] = self.c + self.H * self.x_feasible
self.rhs[n:] = 0.0
else:
self.rhs[:n] = self.c
self.Proj.solve(self.rhs)
r = g = self.Proj.x[:n]
self.v = self.Proj.x[n:]
#self.CheckAccurate()
else:
g = self.c
r = g.copy()
# Initialize search direction
p = -g
pHp = None
self.residNorm0 = numpy.dot(r, g)
rg = self.residNorm0
threshold = max(self.abstol, self.reltol * sqrt(self.residNorm0))
iter = 0
onBoundary = False
if self.debug:
self._write(self.header)
self._write('-' * len(self.header) + '\n')
self._write(self.fmt1 % (iter, rg))
while sqrt(rg) > threshold and iter < self.maxiter and not onBoundary:
Hp = self.H * p
pHp = numpy.dot(p, Hp)
# Display current iteration info
if self.debug: self._write(self.fmt % (iter, rg, pHp))
if self.radius is not None:
# Compute steplength to the boundary
sigma = self.to_boundary(self.x, p, self.radius, ss=xNorm2)
elif pHp <= 0.0:
self._write('Problem is not second-order sufficient\n')
status = 'problem not SOS'
self.infDescent = True
self.dir = p
continue
alpha = rg / pHp
if self.radius is not None and (pHp <= 0.0 or alpha > sigma):
# p is a direction of singularity or negative curvature or
# next iterate will lie past the boundary of the trust region
# Move to boundary of trust-region
self.x += sigma * p
xNorm2 = self.radius * self.radius
status = 'on boundary (sigma = %g)' % sigma
self.infDescent = True
onBoundary = True
continue
# Make sure nonnegativity bounds remain enforced, if requested
if (self.btol is not None) and (self.cur_iter is not None):
stepBnd = self.ftb(self.x, p)
if stepBnd < alpha:
self.x += stepBnd * p
status = 'on boundary'
onBoundary = True
continue
# Move on
self.x += alpha * p
r += alpha * Hp
if self.A is not None:
# Project current residual
self.rhs[:n] = r
self.Proj.solve(self.rhs)
# Perform actual iterative refinement, if necessary
#self.Proj.refine( self.rhs, nitref=self.max_itref,
# tol=self.itref_tol )
# Obtain new projected gradient
g = self.Proj.x[:n]
if self.precon is not None:
# Prepare for iterative semi-refinement
self.A.matvec_transp(self.Proj.x[n:], self.v)
else:
g = r
rg_next = numpy.dot(r, g)
beta = rg_next / rg
p = -g + beta * p
if self.precon is not None:
# Perform iterative semi-refinement
r = r - self.v
else:
r = g
rg = rg_next
if self.radius is not None:
xNorm2 = numpy.dot(self.x, self.x)
iter += 1
# Output info about the last iteration
if self.debug and iter > 0:
self._write(self.fmt % (iter, rg, pHp))
# Obtain final solution x
self.xNorm2 = xNorm2
self.stepNorm = sqrt(xNorm2)
if self.x_feasible is not None:
self.x += self.x_feasible
if self.A is not None:
# Find (weighted) least-squares Lagrange multipliers
self.rhs[:n] = -self.c - self.H * self.x
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
self.v = self.Proj.x[n:].copy()
self.t_solve = cputime() - self.t_solve
self.step = self.x # Alias for consistency with TruncatedCG.
self.onBoundary = onBoundary
self.converged = (iter < self.maxiter)
if iter < self.maxiter and not onBoundary:
status = 'residual small'
elif iter >= self.maxiter:
status = 'max iter'
self.iter = iter
self.nMatvec = iter
self.residNorm = sqrt(rg)
self.status = status
return
def __init__(self, lp, **kwargs):
"""
Solve a linear program of the form::
minimize c' x subject to A1 x + A2 s = b and s >= 0, (LP)
where the variables x are the original problem variables and s are
slack variables. Any linear program may be converted to the above form
by instantiation of the `SlackFramework` class. The conversion to the
slack formulation is mandatory in this implementation.
The method is a variant of Mehrotra's predictor-corrector method where
steps are computed by solving the primal-dual system in augmented form.
Primal and dual regularization parameters may be specified by the user
via the opional keyword arguments `regpr` and `regdu`. Both should be
positive real numbers and should not be "too large". By default they are
set to 1.0 and updated at each iteration.
If `scale` is set to `True`, (LP) is scaled automatically prior to
solution so as to equilibrate the rows and columns of the constraint
matrix [A1 A2].
Advantages of this method are that it is not sensitive to dense columns
in A, no special treatment of the unbounded variables x is required, and
a sparse symmetric quasi-definite system of equations is solved at each
iteration. The latter, although indefinite, possesses a Cholesky-like
factorization. Those properties makes the method typically more robust
that a standard predictor-corrector implementation and the linear system
solves are often much faster than in a traditional interior-point method
in augmented form.
:keywords:
:scale: Perform row and column equilibration of the constraint
matrix [A1 A2] prior to solution (default: `True`).
:stabilize: Scale the linear system to be solved at each iteration
(default: `True`).
:regpr: Initial value of primal regularization parameter
(default: `1.0`).
:regdu: Initial value of dual regularization parameter
(default: `1.0`).
:verbose: Turn on verbose mode (default `False`).
"""
if not isinstance(lp, SlackFramework):
msg = 'Input problem must be an instance of SlackFramework'
raise ValueError, msg
scale = kwargs.get('scale', True)
self.verbose = kwargs.get('verbose', True)
self.stabilize = kwargs.get('stabilize', True)
self.lp = lp
self.A = lp.A() # Constraint matrix
if not isinstance(self.A, PysparseMatrix):
self.A = PysparseMatrix(matrix=self.A)
m, n = self.A.shape
# Record number of slack variables in LP
self.nSlacks = lp.n - lp.original_n
# Constant vectors
zero = np.zeros(n)
self.b = -lp.cons(zero) # Right-hand side
self.c0 = lp.obj(zero) # Constant term in objective
self.c = lp.grad(zero[:lp.original_n]) #lp.cost() # Cost vector
# Apply in-place problem scaling if requested.
self.prob_scaled = False
if scale:
self.t_scale = cputime()
self.scale()
self.t_scale = cputime() - self.t_scale
else:
# scale() sets self.normA to the Frobenius norm of A as a
# by-product. Set it manually here if scaling is not enabled.
self.normA = self.A.matrix.norm('fro')
self.normb = norm_infty(self.b) #norm2(self.b)
self.normc = norm_infty(self.c) #norm2(self.c)
self.normbc = 1 + max(self.normb, self.normc)
# Initialize augmented matrix
self.H = PysparseMatrix(size=n+m,
sizeHint=n+m+self.A.nnz,
symmetric=True)
# We perform the analyze phase on the augmented system only once.
# self.LBL will be initialized in set_initial_guess().
self.LBL = None
self.regpr = kwargs.get('regpr', 1.0) ; self.regpr_min = 1.0e-8
self.regdu = kwargs.get('regdu', 1.0) ; self.regdu_min = 1.0e-8
# Check input parameters.
if self.regpr < 0.0: self.regpr = 0.0
if self.regdu < 0.0: self.regdu = 0.0
# Dual regularization is necessary for stabilization.
if self.regdu == 0.0:
sys.stderr.write('Warning: No dual regularization in effect\n')
sys.stderr.write(' Stabilization has been turned off\n')
self.stabilize = False
# Initialize format strings for display
fmt_hdr = '%-4s %9s' + ' %-8s'*6 + ' %-7s %-4s %-4s' + ' %-8s'*8
self.header = fmt_hdr % ('Iter', 'Cost', 'pResid', 'dResid', 'cResid',
'rGap', 'qNorm', 'rNorm', 'Mu', 'AlPr', 'AlDu',
'LS Resid', 'RegPr', 'RegDu', 'Rho q', 'Del r',
'Min(s)', 'Min(z)', 'Max(s)')
self.format1 = '%-4d %9.2e'
self.format1 += ' %-8.2e' * 6
self.format2 = ' %-7.1e %-4.2f %-4.2f'
self.format2 += ' %-8.2e' * 8 + '\n'
if self.verbose: self.display_stats()
return
def Solve( self ):
# Find feasible solution
if self.A is not None:
if self.factorize and not self.factorized: self.Factorize()
if self.b is not None: self.FindFeasible()
n = self.n
m = self.m
nMatvec = 0
alpha = beta = omega = 0.0
self.t_solve = cputime()
# Obtain fixed vector r0 = projected initial residual
# (initial x = 0 in homogeneous problem.)
if self.A is not None:
self.rhs[:n] = self.r
self.rhs[n:] = 0.0
self.Proj.solve( self.rhs )
r0 = self.Proj.x[:n].copy()
Btv = self.r - r0
else:
r0 = self.c
# Initialize search direction
self.p = self.r
# Further initializations
rr0 = rr00 = numpy.dot(self.r, r0)
residNorm = self.residNorm0 = sqrt(rr0)
stopTol = self.abstol + self.reltol * self.residNorm0
finished = False
if self.debug:
self._write( self.header )
self._write( '-' * len(self.header) + '\n' )
if self.debug:
self._write(self.fmt % (nMatvec, residNorm, rr0, alpha, omega))
while not finished:
# Project p
self.rhs[:n] = self.p
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
self.Pp = self.Proj.x[:n]
# Compute alpha and s
if self._matvec_found:
# Here we must copy Ap to prevent it from being overwritten
# in the next matvec. We still need Ap when we update p below.
self.Ap = self.matvec( self.Pp ).copy()
else:
self.H.matvec( self.Pp, self.Ap )
nMatvec += 1
alpha = rr0/numpy.dot(r0, self.Ap)
self.s = self.r - alpha * self.Ap
# Project s
self.rhs[:n] = self.s - Btv # Iterative semi-refinement
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
self.Ps = self.Proj.x[:n].copy()
Btv = self.s - self.Ps
residNorm = sqrt(numpy.dot(self.s, self.Ps))
# Test for termination in the CGS process
if residNorm <= stopTol or nMatvec > self.nMatvecMax:
self.x += alpha * self.Pp
if nMatvec > self.nMatvecMax:
reason = 'matvec'
else:
reason = 's small'
finished = True
else:
# Project A*Ps
if self._matvec_found:
self.As = self.matvec( self.Ps )
else:
self.H.matvec( self.Ps, self.As )
nMatvec += 1
self.rhs[:n] = self.As
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
# Compute omega and update x
sAs = numpy.dot(self.Ps, self.As)
AsPAs = numpy.dot(self.As, self.Proj.x[:n])
omega = sAs/AsPAs
self.x += alpha * self.Pp + omega * self.Ps
# Check for termination
if nMatvec > self.nMatvecMax:
finished = True
reason = 'matvec'
else:
# Update residual
self.r = self.s - omega * self.As
rr0_next = numpy.dot(self.r, r0)
beta = alpha/omega * rr0_next/rr0
rr0 = rr0_next
self.p -= omega * self.Ap
self.p *= beta
self.p += self.r
# Check for termination in the Bi-CGSTAB process
if abs(rr0) < 1.0e-12 * rr00:
self.rhs[:n] = self.r
self.rhs[n:] = 0.0
self.Proj.solve(self.rhs)
rPr = numpy.dot(self.r, self.Proj.x[:n])
if sqrt(rPr) <= stopTol:
finished = True
reason = 'r small'
# Display current iteration info
if self.debug:
self._write(self.fmt % (nMatvec, residNorm, rr0, alpha, omega))
# End while
# Obtain final solution x
if self.x_feasible is not None:
self.x += self.x_feasible
if self.A is not None:
# Find (weighted) least-squares Lagrange multipliers from
# [ G B^T ] [w] [c - Hx]
# [ B 0 ] [v] = [ 0 ]
if self._matvec_found:
self.rhs[:n] = -self.matvec( self.x )
else:
self.H.matvec( -self.x, self.rhs[:n] )
self.rhs[:n] += self.c
self.rhs[n:] = 0.0
self.Proj.solve( self.rhs )
self.v = self.Proj.x[n:].copy()
self.t_solve = cputime() - self.t_solve
self.converged = (nMatvec < self.nMatvecMax)
self.nMatvec = nMatvec
self.residNorm = residNorm
self.status = reason
return
def solve(self, **kwargs):
"""
Solve the input problem with the primal-dual-regularized
interior-point method. Accepted input keyword arguments are
:keywords:
:itermax: The maximum allowed number of iterations (default: 10n)
:tolerance: Stopping tolerance (default: 1.0e-6)
:PredictorCorrector: Use the predictor-corrector method
(default: `True`). If set to `False`, a variant
of the long-step method is used. The long-step
method is generally slower and less robust.
:returns:
:x: final iterate
:y: final value of the Lagrange multipliers associated
to `A1 x + A2 s = b`
:z: final value of the Lagrange multipliers associated
to `s >= 0`
:obj_value: final cost
:iter: total number of iterations
:kktResid: final relative residual
:solve_time: time to solve the QP
:status: string describing the exit status.
:short_status: short version of status, used for printing.
"""
qp = self.qp
itermax = kwargs.get('itermax', max(100,10*qp.n))
tolerance = kwargs.get('tolerance', 1.0e-6)
PredictorCorrector = kwargs.get('PredictorCorrector', True)
check_infeasible = kwargs.get('check_infeasible', True)
# Transfer pointers for convenience.
m, n = self.A.shape ; on = qp.original_n
A = self.A ; b = self.b ; c = self.c ; Q = self.Q ; diagQ = self.diagQ
H = self.H
regpr = self.regpr ; regdu = self.regdu
regpr_min = self.regpr_min ; regdu_min = self.regdu_min
# Obtain initial point from Mehrotra's heuristic.
(x,y,z) = self.set_initial_guess(**kwargs)
# Slack variables are the trailing variables in x.
s = x[on:] ; ns = self.nSlacks
# Initialize steps in dual variables.
dz = np.zeros(ns)
# Allocate room for right-hand side of linear systems.
rhs = self.initialize_rhs()
finished = False
iter = 0
setup_time = cputime()
# Main loop.
while not finished:
# Display initial header every so often.
if iter % 50 == 0:
self.log.info(self.header)
self.log.info('-' * len(self.header))
# Compute residuals.
pFeas = A*x - b
comp = s*z ; sz = sum(comp) # comp = Sz
Qx = Q*x[:on]
dFeas = y*A ; dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx
dFeas[on:] += z # dFeas2 = A2'y + z
# Compute duality measure.
if ns > 0:
mu = sz/ns
else:
mu = 0.0
# Compute residual norms and scaled residual norms.
pResid = norm2(pFeas)
spResid = pResid/(1+self.normb+self.normA+self.normQ)
dResid = norm2(dFeas)
sdResid = dResid/(1+self.normc+self.normA+self.normQ)
if ns > 0:
cResid = norm_infty(comp)/(self.normbc+self.normA+self.normQ)
else:
cResid = 0.0
# Compute relative duality gap.
cx = np.dot(c,x[:on])
xQx = np.dot(x[:on],Qx)
by = np.dot(b,y)
rgap = cx + xQx - by
rgap = abs(rgap) / (1 + abs(cx) + self.normA + self.normQ)
rgap2 = mu / (1 + abs(cx) + self.normA + self.normQ)
# Compute overall residual for stopping condition.
kktResid = max(spResid, sdResid, rgap2)
# At the first iteration, initialize perturbation vectors
# (q=primal, r=dual).
# Should probably get rid of q when regpr=0 and of r when regdu=0.
if iter == 0:
if regpr > 0:
q = dFeas/regpr ; qNorm = dResid/regpr ; rho_q = dResid
else:
q = dFeas ; qNorm = dResid ; rho_q = 0.0
rho_q_min = rho_q
if regdu > 0:
r = -pFeas/regdu ; rNorm = pResid/regdu ; del_r = pResid
else:
r = -pFeas ; rNorm = pResid ; del_r = 0.0
del_r_min = del_r
pr_infeas_count = 0 # Used to detect primal infeasibility.
du_infeas_count = 0 # Used to detect dual infeasibility.
pr_last_iter = 0
du_last_iter = 0
mu0 = mu
else:
if regdu > 0:
regdu = regdu/10
regdu = max(regdu, regdu_min)
if regpr > 0:
regpr = regpr/10
regpr = max(regpr, regpr_min)
# Check for infeasible problem.
if check_infeasible:
if mu < tolerance/100 * mu0 and \
rho_q > 1./tolerance/1.0e+6 * rho_q_min:
pr_infeas_count += 1
if pr_infeas_count > 1 and pr_last_iter == iter-1:
if pr_infeas_count > 6:
status = 'Problem seems to be (locally) dual'
status += ' infeasible'
short_status = 'dInf'
finished = True
continue
pr_last_iter = iter
else:
pr_infeas_count = 0
if mu < tolerance/100 * mu0 and \
del_r > 1./tolerance/1.0e+6 * del_r_min:
du_infeas_count += 1
if du_infeas_count > 1 and du_last_iter == iter-1:
if du_infeas_count > 6:
status = 'Problem seems to be (locally) primal'
status += ' infeasible'
short_status = 'pInf'
finished = True
continue
du_last_iter = iter
else:
du_infeas_count = 0
# Display objective and residual data.
output_line = self.format1 % (iter, cx + 0.5 * xQx, pResid,
dResid, cResid, rgap, qNorm,
rNorm)
if kktResid <= tolerance:
status = 'Optimal solution found'
short_status = 'opt'
finished = True
continue
if iter >= itermax:
status = 'Maximum number of iterations reached'
short_status = 'iter'
finished = True
continue
# Record some quantities for display
if ns > 0:
mins = np.min(s)
minz = np.min(z)
maxs = np.max(s)
else:
mins = minz = maxs = 0
# Compute augmented matrix and factorize it.
factorized = False
degenerate = False
nb_bump = 0
while not factorized and not degenerate:
self.update_linear_system(s, z, regpr, regdu)
self.log.debug('Factorizing')
self.LBL.factorize(H)
factorized = True
# If the augmented matrix does not have full rank, bump up the
# regularization parameters.
if not self.LBL.isFullRank:
if self.verbose:
self.log.info('Primal-Dual Matrix Rank Deficient' + \
'... bumping up reg parameters')
if regpr == 0. and regdu == 0.:
degenerate = True
else:
if regpr > 0:
regpr *= 100
if regdu > 0:
regdu *= 100
nb_bump += 1
degenerate = nb_bump > self.bump_max
factorized = False
# Abandon if regularization is unsuccessful.
if not self.LBL.isFullRank and degenerate:
status = 'Unable to regularize sufficiently.'
short_status = 'degn'
finished = True
continue
if PredictorCorrector:
# Use Mehrotra predictor-corrector method.
# Compute affine-scaling step, i.e. with centering = 0.
self.set_affine_scaling_rhs(rhs, pFeas, dFeas, s, z)
(step, nres, neig) = self.solveSystem(rhs)
# Recover dx and dz.
dx, ds, dy, dz = self.get_affine_scaling_dxsyz(step, x, s, y, z)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, ip) = self.maxStepLength(z, dz)
# Estimate duality gap after affine-scaling step.
muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
sigma = (muAff/mu)**3
# Incorporate predictor information for corrector step.
# Only update rhs[on:n]; the rest of the vector did not change.
comp += ds*dz
comp -= sigma * mu
self.update_corrector_rhs(rhs, s, z, comp)
else:
# Use long-step method: Compute centering parameter.
sigma = min(0.1, 100*mu)
comp -= sigma * mu
# Assemble rhs.
self.update_long_step_rhs(rhs, pFeas, dFeas, comp, s)
# Solve augmented system.
(step, nres, neig) = self.solveSystem(rhs)
# Recover step.
dx, ds, dy, dz = self.get_dxsyz(step, x, s, y, z, comp)
normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx)
# Compute largest allowed primal and dual stepsizes.
(alpha_p, ip) = self.maxStepLength(s, ds)
(alpha_d, id) = self.maxStepLength(z, dz)
# Compute fraction-to-the-boundary factor.
tau = max(.9995, 1.0-mu)
if PredictorCorrector:
# Compute actual stepsize using Mehrotra's heuristic.
mult = 0.1
# ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0
if (ip != -1 or id != -1) and ip != id:
mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns
if ip != -1 and ip != id:
zip = z[ip] + alpha_d * dz[ip]
gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip)
alpha_p *= max(1-mult, gamma_p)
if id != -1 and ip != id:
sid = s[id] + alpha_p * ds[id]
gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid)
alpha_d *= max(1-mult, gamma_d)
if ip==id and ip != -1:
# There is a division by zero in Mehrotra's heuristic
# Fall back on classical rule.
alpha_p *= tau
alpha_d *= tau
else:
alpha_p *= tau
alpha_d *= tau
# Display data.
output_line += self.format2 % (mu, alpha_p, alpha_d,
nres, regpr, regdu, rho_q,
del_r, mins, minz, maxs)
self.log.info(output_line)
# Update iterates and perturbation vectors.
x += alpha_p * dx # This also updates slack variables.
y += alpha_d * dy
z += alpha_d * dz
q *= (1-alpha_p) ; q += alpha_p * dx
r *= (1-alpha_d) ; r += alpha_d * dy
qNorm = norm2(q) ; rNorm = norm2(r)
if regpr > 0:
rho_q = regpr * qNorm/(1+self.normc)
rho_q_min = min(rho_q_min, rho_q)
else:
rho_q = 0.0
if regdu > 0:
del_r = regdu * rNorm/(1+self.normb)
del_r_min = min(del_r_min, del_r)
else:
del_r = 0.0
iter += 1
solve_time = cputime() - setup_time
self.log.info('-' * len(self.header))
# Transfer final values to class members.
self.x = x
self.y = y
self.z = z
self.iter = iter
self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid
self.rgap = rgap
self.kktResid = kktResid
self.solve_time = solve_time
self.status = status
self.short_status = short_status
# Unscale problem if applicable.
if self.prob_scaled: self.unscale()
# Recompute final objective value.
self.obj_value = self.c0 + cx + 0.5 * xQx
return
def solve(self, **kwargs):
"""
Solve current problem with trust-funnel framework.
:keywords:
:ny: Enable Nocedal-Yuan backtracking linesearch.
:returns:
This method sets the following members of the instance:
:f: Final objective value
:optimal: Flag indicating whether normal stopping conditions were
attained
:pResid: Final primal residual
:dResid: Final dual residual
:niter: Total number of iterations
:tsolve: Solve time.
"""
ny = kwargs.get("ny", True)
reg = kwargs.get("reg", 0.0)
# ny = False
tsolve = cputime()
# Set some shortcuts.
nlp = self.nlp
n = nlp.n
m = nlp.m
x = self.x
f = self.f
c = self.cons(x)
y = nlp.pi0.copy()
self.it = 0
# Initialize some constants.
kappa_n = 1.0e2 # Factor of pNorm in normal step TR radius.
kappa_b = 0.99 # Fraction of TR to compute tangential step.
kappa_delta = 0.1 # Progress factor to compute tangential step.
# Trust-region parameters.
eta_1 = 1.0e-5
eta_2 = 0.95
eta_3 = 0.5
gamma_1 = 0.25
gamma_3 = 2.5
kappa_tx1 = 0.9 # Factor of theta_max in max acceptable infeasibility.
kappa_tx2 = 0.5 # Convex combination factor of theta and thetaTrial.
# Compute constraint violation.
theta = 0.5 * np.dot(c, c)
# Set initial funnel radius.
kappa_ca = 1.0e3 # Max initial funnel radius.
kappa_cr = 2.0 # Infeasibility tolerance factor.
theta_max = max(kappa_ca, kappa_cr * theta)
# Evaluate first-order derivatives.
g = nlp.grad(x)
J = self.jac(x)
Jop = PysparseLinearOperator(J)
# Initial radius for f- and c-iterations.
Delta_f = max(self.Delta_f, 0.1 * np.linalg.norm(g))
Delta_c = max(self.Delta_c, 0.1 * sqrt(2 * theta))
# Reset initial multipliers to least-squares estimates by
# approximately solving:
# [ I J' ] [ w ] [ -g ]
# [ J 0 ] [ y ] = [ 0 ].
# This is equivalent to solving
# minimize |g + J'y|.
if m > 0:
y, _, _ = self.lsq(Jop.T, -g, reg=reg)
pNorm = cNorm = 0
if m > 0:
pNorm = np.linalg.norm(c)
cNorm = np.linalg.norm(c, np.inf)
grad_lag = g + Jop.T * y
else:
grad_lag = g.copy()
dNorm = np.linalg.norm(grad_lag) / (1 + np.linalg.norm(y))
# Display current info if requested.
self.log.info(self.hdr)
self.log.info(self.linefmt1 % (0, " ", " ", " ", " ", f, pNorm, dNorm, Delta_f, Delta_c, theta_max, 0))
# Compute primal stopping tolerance.
stop_p = max(self.atol, self.stop_p * pNorm)
self.log.debug("pNorm = %8.2e, cNorm = %8.2e, dNorm = %8.e2" % (pNorm, cNorm, dNorm))
optimal = (pNorm <= stop_p) and (dNorm <= self.stop_d)
self.log.debug("optimal: %s" % repr(optimal))
# Start of main iteration.
while not optimal and (self.it < self.maxit):
self.it += 1
Delta = min(Delta_f, Delta_c)
cgiter = 0
# 1. Compute normal step as an (approximate) solution to
# minimize |c + J n| subject to |n| <= min(Delta_c, kN |c|).
if self.it > 1 and pNorm <= stop_p and dNorm >= 1.0e4 * self.stop_d:
self.log.debug("Setting nStep=0 b/c need to work on optimality")
nStep = np.zeros(n)
nStepNorm = 0.0
n_end = "0"
m_xpn = 0 # Model value at x+n.
else:
nStep_max = min(Delta_c, kappa_n * pNorm)
nStep, nStepNorm, lsq_status = self.lsq(Jop, -c, radius=nStep_max, reg=reg)
if lsq_status == "residual small":
n_end = "r"
elif lsq_status == "trust-region boundary active":
n_end = "b"
else:
n_end = "?"
# Evaluate the model of the obective after the normal step.
_Hv = self.hprod(x, y, nStep) # H*nStep
m_xpn = np.dot(g, nStep) + 0.5 * np.dot(nStep, _Hv)
self.log.debug("Normal step norm = %8.2e" % nStepNorm)
self.log.debug("Model value: %9.2e" % m_xpn)
# 2. Compute tangential step if normal step is not too long.
if nStepNorm <= kappa_b * Delta:
# 2.1. Compute Lagrange multiplier estimates and dual residuals
# by minimizing |(g + H n) + J'y|
if nStepNorm == 0.0:
gN = g # Note: this is just a pointer ; g will not be modified below.
else:
gN = g + _Hv
y_new, y_norm, _ = self.lsq(Jop.T, -gN, reg=reg)
r = gN + Jop.T * y_new
# Here Nick does iterative refinement to improve r and y_new...
# Compute dual optimality measure.
residNorm = np.linalg.norm(r)
norm_gN = np.linalg.norm(gN)
pi = 0.0
if residNorm > 0:
pi = abs(np.dot(gN, r)) / residNorm
# 2.2. If the dual residuals are large, compute a suitable
# tangential step as a solution to:
# minimize g't + 1/2 t' H t
# subject to Jt = 0, |n+t| <= Delta.
if pi > self.forcing(3, theta):
self.log.debug("Computing tStep...")
Delta_within = Delta - nStepNorm
Hop = SimpleLinearOperator(n, n, lambda v: self.hprod(x, y_new, v), symmetric=True)
PPCG = ProjectedCG(gN, Hop, A=J.matrix if m > 0 else None, radius=Delta_within, dreg=reg)
PPCG.Solve()
tStep = PPCG.step
tStepNorm = PPCG.stepNorm
cgiter = PPCG.iter
self.log.debug("|t| = %8.2e" % tStepNorm)
if PPCG.status == "residual small":
t_end = "r"
elif PPCG.onBoundary and not PPCG.infDescent:
t_end = "b"
elif PPCG.infDescent:
t_end = "-"
elif PPCG.status == "max iter":
t_end = ">"
else:
t_end = "?"
# Compute total step and model decrease.
step = nStep + tStep
stepNorm = np.linalg.norm(step)
_Hv = self.hprod(x, y, step) # y or y_new?
m_xps = np.dot(g, step) + 0.5 * np.dot(step, _Hv)
else:
self.log.debug("Setting tStep=0 b/c pi is sufficiently small")
tStepNorm = 0
t_end = "0"
step = nStep
stepNorm = nStepNorm
m_xps = m_xpn
y = y_new
else:
# No need to compute a tangential step.
self.log.debug("Setting tStep=0 b/c the normal step is too large")
t_end = "0"
y = np.zeros(m)
tStepNorm = 0.0
step = nStep
stepNorm = nStepNorm
m_xps = m_xpn
self.log.debug("Model decrease = %9.2e" % m_xps)
# Compute trial point and evaluate local data.
xTrial = x + step
fTrial = nlp.obj(xTrial)
cTrial = self.cons(xTrial)
thetaTrial = 0.5 * np.dot(cTrial, cTrial)
delta_f = -m_xps # Overall improvement in the model.
delta_ft = m_xpn - m_xps # Improvement due to tangential step.
Jspc = c + Jop * step
# Compute improvement in linearized feasibility.
delta_feas = theta - 0.5 * np.dot(Jspc, Jspc)
# Decide whether to consider the current iteration
# an f- or a c-iteration.
if (
tStepNorm > 0
and (delta_f >= self.forcing(2, theta))
and delta_f >= kappa_delta * delta_ft
and thetaTrial <= theta_max
):
# Step 3. Consider that this is an f-iteration.
it_type = "f"
# Decide whether trial point is accepted.
ratio = (f - fTrial) / delta_f
self.log.debug("f-iter ratio = %9.2e" % ratio)
if ratio >= eta_1: # Successful step.
suc = "s"
x = xTrial
f = fTrial
c = cTrial
self.step = step.copy()
theta = thetaTrial
# Decide whether to update f-trust-region radius.
if ratio >= eta_2:
suc = "v"
Delta_f = min(max(Delta_f, gamma_3 * stepNorm), 1.0e10)
# Decide whether to update c-trust-region radius.
if thetaTrial < eta_3 * theta_max:
ns = nStepNorm if nStepNorm > 0 else Delta_c
Delta_c = min(max(Delta_c, gamma_3 * ns), 1.0e10)
self.log.debug("New Delta_f = %8.2e" % Delta_f)
self.log.debug("New Delta_c = %8.2e" % Delta_c)
else: # Unsuccessful step (ratio < eta_1).
attempt_SOC = True
suc = "u"
if attempt_SOC:
self.log.debug(" Attempting second-order correction")
# Attempt a second-order correction by solving
# minimize |cTrial + J n| subject to |n| <= Delta_c.
socStep, socStepNorm, socStatus = self.lsq(Jop, -cTrial, radius=Delta_c, reg=reg)
if socStatus != "trust-region boundary active":
# Consider SOC step as candidate step.
xSoc = xTrial + socStep
fSoc = nlp.obj(xSoc)
cSoc = self.cons(xSoc)
thetaSoc = 0.5 * np.dot(cSoc, cSoc)
ratio = (f - fSoc) / delta_f
# Decide whether to accept SOC step.
if ratio >= eta_1 and thetaSoc <= theta_max:
suc = "2"
x = xSoc
f = fSoc
c = cSoc
theta = thetaSoc
self.step = step + socStep
else:
# Backtracking linesearch a la Nocedal & Yuan.
# Abandon SOC step. Backtrack from x+step.
if ny:
(x, f, alpha) = self.nyf(x, f, fTrial, g, step)
# g = nlp.grad(x)
c = self.cons(x)
theta = 0.5 * np.dot(c, c)
self.step = step + alpha * socStep
Delta_f = min(alpha, 0.8) * stepNorm
suc = "y"
else:
Delta_f = gamma_1 * Delta_f
else: # SOC step lies on boundary of trust region.
Delta_f = gamma_1 * Delta_f
else: # SOC step not attempted.
# Backtracking linesearch a la Nocedal & Yuan.
if ny:
(x, f, alpha) = self.nyf(x, f, fTrial, g, step)
# g = nlp.grad(x)
c = self.cons(x)
theta = 0.5 * np.dot(c, c)
self.step = alpha * step
Delta_f = min(alpha, 0.8) * stepNorm
suc = "y"
else:
Delta_f = gamma_1 * Delta_f
else:
# Step 4. Consider that this is a c-iteration.
it_type = "c"
# Display information.
self.log.debug("c-iteration because ")
if tStepNorm == 0.0:
self.log.debug("|t|=0")
if delta_f < self.forcing(2, theta):
self.log.debug("delta_f=%8.2e < forcing=%8.2e" % (delta_f, self.forcing(2, theta)))
if delta_f < kappa_delta * delta_ft:
self.log.debug("delta_f=%8.2e < frac * delta_ft=%8.2e" % (delta_f, delta_ft))
if thetaTrial > theta_max:
self.log.debug("thetaTrial=%8.2e > theta_max=%8.2e" % (thetaTrial, theta_max))
# Step 4.1. Check trial point for acceptability.
if delta_feas < 0:
self.log.debug(" !!! Warning: delta_feas is negative !!!")
ratio = (theta - thetaTrial + 1.0e-16) / (delta_feas + 1.0e-16)
self.log.debug("c-iter ratio = %9.2e" % ratio)
if ratio >= eta_1: # Successful step.
x = xTrial
f = fTrial
c = cTrial
self.step = step.copy()
suc = "s"
# Step 4.2. Update Delta_c.
if ratio >= eta_2: # Very successful step.
ns = nStepNorm if nStepNorm > 0 else Delta_c
Delta_c = min(max(Delta_c, gamma_3 * ns), 1.0e10)
suc = "v"
# Step 4.3. Update maximum acceptable infeasibility.
theta_max = max(kappa_tx1 * theta_max, kappa_tx2 * theta + (1 - kappa_tx2) * thetaTrial)
theta = thetaTrial
else: # Unsuccessful step.
# Backtracking linesearch a la Nocedal & Yuan.
ns = nStepNorm if nStepNorm > 0 else Delta_c
if ny:
(x, c, theta, alpha) = self.nyc(x, theta, thetaTrial, c, Jop.T * c, step)
f = nlp.obj(x)
# g = nlp.grad(x)
self.step = alpha * step
Delta_c = min(alpha, 0.8) * ns
suc = "y"
else:
Delta_c = gamma_1 * ns # Delta_c
suc = "u"
self.log.debug("New Delta_c = %8.2e" % Delta_c)
self.log.debug("New theta_max = %8.2e" % theta_max)
# Step 5. Book keeping.
if ratio >= eta_1 or ny:
g = nlp.grad(x)
J = self.jac(x)
Jop = PysparseLinearOperator(J)
self.post_iteration()
pNorm = cNorm = 0
if m > 0:
pNorm = np.linalg.norm(c)
cNorm = np.linalg.norm(c, np.inf)
grad_lag = g + Jop.T * y
else:
grad_lag = g.copy()
dNorm = np.linalg.norm(grad_lag) / (1 + np.linalg.norm(y))
if self.it % 20 == 0:
self.log.info(self.hdr)
self.log.info(
self.linefmt
% (self.it, it_type, suc, n_end, t_end, f, pNorm, dNorm, Delta_f, Delta_c, theta_max, cgiter)
)
optimal = (pNorm <= stop_p) and (dNorm <= self.stop_d)
# End while.
self.tsolve = cputime() - tsolve
if optimal:
self.status = 0 # Successful solve.
self.log.info("Found an optimal solution! Yeah!")
else:
self.status = 1 # Refine this in the future.
self.x = x
self.f = f
self.optimal = optimal
self.pResid = pNorm
self.dResid = dNorm
self.niter = self.it
return
def Solve(self, **kwargs):
nlp = self.nlp
# Gather initial information.
self.f = self.nlp.obj(self.x)
self.f0 = self.f
self.g = self.nlp.grad(self.x) # Current gradient
self.g_old = self.g # Previous gradient
self.gnorm = norms.norm2(self.g)
self.g0 = self.gnorm
# Reset initial trust-region radius.
# self.TR.Delta = 0.1 * self.g0
if self.inexact:
cgtol = 1.0
else:
cgtol = -1.0
stoptol = max(self.abstol, self.reltol * self.g0)
step_status = None
exitOptimal = exitIter = exitUser = False
# Initialize non-monotonicity parameters.
if not self.monotone:
fMin = fRef = fCan = self.f0
l = 0
sigRef = sigCan = 0
t = cputime()
# Print out header and initial log.
if self.iter % 20 == 0 and self.verbose:
self.log.info(self.hline)
self.log.info(self.header)
self.log.info(self.hline)
self.log.info(self.format0 % (self.iter, self.f,
self.gnorm, '', '',
self.TR.Delta, ''))
while not (exitUser or exitOptimal or exitIter):
self.iter += 1
self.alpha = 1.0
# Save current gradient
if self.save_g:
self.g_old = self.g.copy()
# Iteratively minimize the quadratic model in the trust region
# m(s) = <g, s> + 1/2 <s, Hs>
# Note that m(s) does not include f(x): m(0) = 0.
if self.inexact:
cgtol = max(1.0e-6, min(0.5 * cgtol, sqrt(self.gnorm)))
H = SimpleLinearOperator(nlp.n, nlp.n,
lambda v: self.hprod(v),
symmetric=True)
self.solver = self.TrSolver(self.g, H)
self.solver.Solve(prec=self.precon,
radius=self.TR.Delta,
reltol=cgtol,
#debug=True
)
step = self.solver.step
snorm = self.solver.stepNorm
cgiter = self.solver.niter
# Obtain model value at next candidate
m = self.solver.m
if m is None:
m = numpy.dot(self.g, step) + 0.5*numpy.dot(step, H * step)
self.total_cgiter += cgiter
x_trial = self.x + step
f_trial = nlp.obj(x_trial)
rho = self.TR.Rho(self.f, f_trial, m)
if not self.monotone:
rhoHis = (fRef - f_trial)/(sigRef - m)
rho = max(rho, rhoHis)
step_status = 'Rej'
if rho >= self.TR.eta1:
# Trust-region step is accepted.
self.TR.UpdateRadius(rho, snorm)
self.x = x_trial
self.f = f_trial
self.g = nlp.grad(self.x)
self.gnorm = norms.norm2(self.g)
step_status = 'Acc'
# Update non-monotonicity parameters.
if not self.monotone:
sigRef = sigRef - m
sigCan = sigCan - m
if f_trial < fMin:
fCan = f_trial
fMin = f_trial
sigCan = 0
l = 0
else:
l = l + 1
if f_trial > fCan:
fCan = f_trial
sigCan = 0
if l == self.nIterNonMono:
fRef = fCan
sigRef = sigCan
else:
# Trust-region step is rejected.
if self.ny: # Backtracking linesearch following "Nocedal & Yuan"
slope = numpy.dot(self.g, step)
bk = 0
while bk < self.nbk and \
f_trial >= self.f + 1.0e-4 * self.alpha * slope:
bk = bk + 1
self.alpha /= 1.2
x_trial = self.x + self.alpha * step
f_trial = nlp.obj(x_trial)
self.x = x_trial
self.f = f_trial
self.g = nlp.grad(self.x)
self.gnorm = norms.norm2(self.g)
self.TR.Delta = self.alpha * snorm
step_status = 'N-Y'
else:
self.TR.UpdateRadius(rho, snorm)
self.step_status = step_status
self.radii.append(self.TR.Delta)
status = ''
try:
self.PostIteration()
except UserExitRequest:
status = 'usr'
# Print out header, say, every 20 iterations
if self.iter % 20 == 0 and self.verbose:
self.log.info(self.hline)
self.log.info(self.header)
self.log.info(self.hline)
if self.verbose:
pstatus = step_status if step_status != 'Acc' else ''
self.log.info(self.format % (self.iter, self.f,
self.gnorm, cgiter, rho,
self.TR.Delta, pstatus))
exitOptimal = self.gnorm < stoptol
exitIter = self.iter > self.maxiter
exitUser = status == 'usr'
self.tsolve = cputime() - t # Solve time
# Set final solver status.
if status == 'usr':
pass
elif self.gnorm <= stoptol:
status = 'opt'
else: # self.iter > self.maxiter:
status = 'itr'
self.status = status
nA = A.shape[0]
nC = C.shape[0]
K = spmatrix.ll_mat_sym(nA + nC, A.nnz + C.nnz + min(nA, nC))
K[:nA, :nA] = A
K[nA:, nA:] = C
K[nA:, nA:].scale(-1.0)
idx = np.arange(min(nA, nC), dtype=np.int)
K.put(1, nA + idx, idx)
# Create right-hand side rhs=K*e
e = np.ones(nA + nC)
rhs = np.empty(nA + nC)
K.matvec(e, rhs)
# Factorize and solve Kx = rhs, knowing K is sqd
t = cputime()
P = LBLContext(K, sqd=True)
t = cputime() - t
sys.stderr.write('Factorization time with sqd=True : %5.2fs ' % t)
P.solve(rhs, get_resid=False)
sys.stderr.write('Error: %7.1e\n' % np.linalg.norm(P.x - e, ord=np.Inf))
# Do it all over again, pretending we don't know K is sqd
t = cputime()
P = LBLContext(K)
t = cputime() - t
sys.stderr.write('Factorization time with sqd=False: %5.2fs ' % t)
P.solve(rhs, get_resid=False)
sys.stderr.write('Error: %7.1e\n' % np.linalg.norm(P.x - e, ord=np.Inf))
try:
if options.n_dim is not None:
n = options.n_dim
if options.delta_size is not None:
delta = options.delta_size
# Set printing standards for arrays.
numpy.set_printoptions(precision=3, linewidth=70, threshold=10, edgeitems=2)
multiple_problems = len(args) > 1
#if not options.verbose:
#log.info(hdr)
#log.info('-'*len(hdr))
args = ['example']
for probname in args:
t_setup = cputime()
#lsqp = DCT(n,m,delta)#partial_
#n=2;m= 2;
#lsqp= exampleliop(n,m)
lsqp = partial_DCT(n,m,delta)
t_setup = cputime() - t_setup
# Pass problem to RegQP.
regqp = Solver(lsqp,
verbose=options.verbose,
**opts_init)
regqp.solve(PredictorCorrector=not options.longstep,
check_infeasible=not options.assume_feasible,
**opts_solve)
def Solve(self, **kwargs):
nlp = self.nlp
# Gather initial information.
self.f = self.nlp.obj(self.x)
self.f0 = self.f
self.g = self.nlp.grad(self.x) # Current gradient
self.g_old = self.g # Previous gradient
self.gnorm = norms.norm2(self.g)
self.g0 = self.gnorm
# Reset initial trust-region radius.
# self.TR.Delta = 0.1 * self.g0
if self.inexact:
cgtol = 1.0
else:
cgtol = -1.0
stoptol = max(self.abstol, self.reltol * self.g0)
step_status = None
exitOptimal = exitIter = exitUser = False
# Initialize non-monotonicity parameters.
if not self.monotone:
fMin = fRef = fCan = self.f0
l = 0
sigRef = sigCan = 0
t = cputime()
# Print out header and initial log.
if self.iter % 20 == 0 and self.verbose:
self.log.info(self.hline)
self.log.info(self.header)
self.log.info(self.hline)
self.log.info(
self.format0 %
(self.iter, self.f, self.gnorm, '', '', self.TR.Delta, ''))
while not (exitUser or exitOptimal or exitIter):
self.iter += 1
self.alpha = 1.0
# Save current gradient
if self.save_g:
self.g_old = self.g.copy()
# Iteratively minimize the quadratic model in the trust region
# m(s) = <g, s> + 1/2 <s, Hs>
# Note that m(s) does not include f(x): m(0) = 0.
if self.inexact:
cgtol = max(1.0e-6, min(0.5 * cgtol, sqrt(self.gnorm)))
H = SimpleLinearOperator(nlp.n,
nlp.n,
lambda v: self.hprod(v),
symmetric=True)
self.solver = self.TrSolver(self.g, H)
self.solver.Solve(
prec=self.precon,
radius=self.TR.Delta,
reltol=cgtol,
#debug=True
)
step = self.solver.step
snorm = self.solver.stepNorm
cgiter = self.solver.niter
# Obtain model value at next candidate
m = self.solver.m
if m is None:
m = numpy.dot(self.g, step) + 0.5 * numpy.dot(step, H * step)
self.total_cgiter += cgiter
x_trial = self.x + step
f_trial = nlp.obj(x_trial)
rho = self.TR.Rho(self.f, f_trial, m)
if not self.monotone:
rhoHis = (fRef - f_trial) / (sigRef - m)
rho = max(rho, rhoHis)
step_status = 'Rej'
if rho >= self.TR.eta1:
# Trust-region step is accepted.
self.TR.UpdateRadius(rho, snorm)
self.x = x_trial
self.f = f_trial
self.g = nlp.grad(self.x)
self.gnorm = norms.norm2(self.g)
step_status = 'Acc'
# Update non-monotonicity parameters.
if not self.monotone:
sigRef = sigRef - m
sigCan = sigCan - m
if f_trial < fMin:
fCan = f_trial
fMin = f_trial
sigCan = 0
l = 0
else:
l = l + 1
if f_trial > fCan:
fCan = f_trial
sigCan = 0
if l == self.nIterNonMono:
fRef = fCan
sigRef = sigCan
else:
# Trust-region step is rejected.
if self.ny: # Backtracking linesearch following "Nocedal & Yuan"
slope = numpy.dot(self.g, step)
bk = 0
while bk < self.nbk and \
f_trial >= self.f + 1.0e-4 * self.alpha * slope:
bk = bk + 1
self.alpha /= 1.2
x_trial = self.x + self.alpha * step
f_trial = nlp.obj(x_trial)
self.x = x_trial
self.f = f_trial
self.g = nlp.grad(self.x)
self.gnorm = norms.norm2(self.g)
self.TR.Delta = self.alpha * snorm
step_status = 'N-Y'
else:
self.TR.UpdateRadius(rho, snorm)
self.step_status = step_status
self.radii.append(self.TR.Delta)
status = ''
try:
self.PostIteration()
except UserExitRequest:
status = 'usr'
# Print out header, say, every 20 iterations
if self.iter % 20 == 0 and self.verbose:
self.log.info(self.hline)
self.log.info(self.header)
self.log.info(self.hline)
if self.verbose:
pstatus = step_status if step_status != 'Acc' else ''
self.log.info(self.format %
(self.iter, self.f, self.gnorm, cgiter, rho,
self.TR.Delta, pstatus))
exitOptimal = self.gnorm < stoptol
exitIter = self.iter > self.maxiter
exitUser = status == 'usr'
self.tsolve = cputime() - t # Solve time
# Set final solver status.
if status == 'usr':
pass
elif self.gnorm <= stoptol:
status = 'opt'
else: # self.iter > self.maxiter:
status = 'itr'
self.status = status