def __init__(self, nlp, **kwargs): self.nlp = nlp self.npairs = kwargs.get('npairs', 5) self.silent = kwargs.get('silent', False) self.abstol = kwargs.get('abstol', 1.0e-6) self.reltol = kwargs.get('reltol', self.nlp.stop_d) self.iter = 0 self.nresets = 0 self.converged = False self.lbfgs = InverseLBFGS(self.nlp.n, **kwargs) # Code for testingLBFGS self.alt_lbfgs = LBFGS(self.nlp.n, **kwargs) self.x = kwargs.get('x0', self.nlp.x0) self.f = self.nlp.obj(self.x) self.g = self.nlp.grad(self.x) self.gnorm = norms.norm2(self.g) self.f0 = self.f self.g0 = self.gnorm # Optional arguments self.maxiter = kwargs.get('maxiter', max(10*self.nlp.n, 1000)) self.tsolve = 0.0
def solve(self): tstart = cputime() # Initial LBFGS matrix is the identity. In other words, # the initial search direction is the steepest descent direction. # This is the original L-BFGS stopping condition. #stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x)) stoptol = max(self.abstol, self.reltol * self.g0) while self.gnorm > stoptol and self.iter < self.maxiter: if not self.silent: print '%-5d %-12g %-12g' % (self.iter, self.f, self.gnorm) # Obtain search direction d = self.lbfgs.matvec(-self.g) # Prepare for modified More-Thuente linesearch if self.iter == 0: stp0 = 1.0/self.gnorm else: stp0 = 1.0 SWLS = StrongWolfeLineSearch(self.f, self.x, self.g, d, lambda z: self.nlp.obj(z), lambda z: self.nlp.grad(z), stp = stp0) # Perform linesearch SWLS.search() # SWLS.x contains the new iterate # SWLS.g contains the objective gradient at the new iterate # SWLS.f contains the objective value at the new iterate s = SWLS.x - self.x self.x = SWLS.x y = SWLS.g - self.g self.g = SWLS.g self.gnorm = norms.norm2(self.g) self.f = SWLS.f #stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x)) # Update inverse Hessian approximation using the most recent pair self.lbfgs.store(s, y) self.iter += 1 # Code for testing the LBFGS implementation self.alt_lbfgs.store(s, y) self.tsolve = cputime() - tstart self.converged = (self.iter < self.maxiter)
def solve(self): tstart = cputime() # Initial LBFGS matrix is the identity. In other words, # the initial search direction is the steepest descent direction. # This is the original L-BFGS stopping condition. #stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x)) stoptol = max(self.abstol, self.reltol * self.g0) while self.gnorm > stoptol and self.iter < self.maxiter: if not self.silent: print '%-5d %-12g %-12g' % (self.iter, self.f, self.gnorm) # Obtain search direction d = self.lbfgs.matvec(-self.g) # Prepare for modified More-Thuente linesearch if self.iter == 0: stp0 = 1.0 / self.gnorm else: stp0 = 1.0 SWLS = StrongWolfeLineSearch(self.f, self.x, self.g, d, lambda z: self.nlp.obj(z), lambda z: self.nlp.grad(z), stp=stp0) # Perform linesearch SWLS.search() # SWLS.x contains the new iterate # SWLS.g contains the objective gradient at the new iterate # SWLS.f contains the objective value at the new iterate s = SWLS.x - self.x self.x = SWLS.x y = SWLS.g - self.g self.g = SWLS.g self.gnorm = norms.norm2(self.g) self.f = SWLS.f #stoptol = self.nlp.stop_d * max(1.0, norms.norm2(self.x)) # Update inverse Hessian approximation using the most recent pair self.lbfgs.store(s, y) self.iter += 1 self.tsolve = cputime() - tstart self.converged = (self.iter < self.maxiter)
def solveSystem(self, rhs, itref_threshold=1.0e-5, nitrefmax=5): """ Solve the augmented system with right-hand side `rhs` and optionally perform iterative refinement. Return the solution vector (as a reference), the 2-norm of the residual and the number of negative eigenvalues of the coefficient matrix. """ self.log.debug('Solving linear system') self.LBL.solve(rhs) self.LBL.refine(rhs, tol=itref_threshold, nitref=nitrefmax) # Collect statistics on the linear system solve. self.cond_history.append((self.LBL.cond, self.LBL.cond2)) self.berr_history.append((self.LBL.berr, self.LBL.berr2)) self.derr_history.append(self.LBL.dirError) self.nrms_history.append((self.LBL.matNorm, self.LBL.xNorm)) self.lres_history.append(self.LBL.relRes) nr = norm2(self.LBL.residual) return (self.LBL.x, nr, self.LBL.neig)
def SolveSystem(A, rhs, itref_threshold=1.0e-6, nitrefmax=5, **kwargs): # Obtain Sils context object t = cputime() LBL = LBLContext(A, **kwargs) t_analyze = cputime() - t # Solve system and compute residual t = cputime() LBL.solve(rhs) t_solve = cputime() - t_analyze # Compute residual norm nrhsp1 = norms.norm_infty(rhs) + 1 nr = norms.norm2(LBL.residual)/nrhsp1 # If residual not small, perform iterative refinement LBL.refine(rhs, tol = itref_threshold, nitref=nitrefmax) nr1 = norms.norm_infty(LBL.residual)/nrhsp1 return (LBL.x, LBL.residual, nr, nr1, t_analyze, t_solve, LBL.neig)
def __init__(self, nlp, **kwargs): self.nlp = nlp self.npairs = kwargs.get('npairs', 5) self.silent = kwargs.get('silent', False) self.abstol = kwargs.get('abstol', 1.0e-6) self.reltol = kwargs.get('reltol', self.nlp.stop_d) self.iter = 0 self.nresets = 0 self.converged = False self.lbfgs = InverseLBFGS(self.nlp.n, **kwargs) self.x = kwargs.get('x0', self.nlp.x0) self.f = self.nlp.obj(self.x) self.g = self.nlp.grad(self.x) self.gnorm = norms.norm2(self.g) self.f0 = self.f self.g0 = self.gnorm # Optional arguments self.maxiter = kwargs.get('maxiter', max(10 * self.nlp.n, 1000)) self.tsolve = 0.0
def __init__(self, lp, **kwargs): """ Solve a linear program of the form:: minimize c' x subject to A1 x + A2 s = b and s >= 0, (LP) where the variables x are the original problem variables and s are slack variables. Any linear program may be converted to the above form by instantiation of the `SlackFramework` class. The conversion to the slack formulation is mandatory in this implementation. The method is a variant of Mehrotra's predictor-corrector method where steps are computed by solving the primal-dual system in augmented form. Primal and dual regularization parameters may be specified by the user via the opional keyword arguments `regpr` and `regdu`. Both should be positive real numbers and should not be "too large". By default they are set to 1.0 and updated at each iteration. If `scale` is set to `True`, (LP) is scaled automatically prior to solution so as to equilibrate the rows and columns of the constraint matrix [A1 A2]. Advantages of this method are that it is not sensitive to dense columns in A, no special treatment of the unbounded variables x is required, and a sparse symmetric quasi-definite system of equations is solved at each iteration. The latter, although indefinite, possesses a Cholesky-like factorization. Those properties makes the method typically more robust that a standard predictor-corrector implementation and the linear system solves are often much faster than in a traditional interior-point method in augmented form. :keywords: :scale: Perform row and column equilibration of the constraint matrix [A1 A2] prior to solution (default: `True`). :stabilize: Scale the linear system to be solved at each iteration (default: `True`). :regpr: Initial value of primal regularization parameter (default: `1.0`). :regdu: Initial value of dual regularization parameter (default: `1.0`). :verbose: Turn on verbose mode (default `False`). """ if not isinstance(lp, SlackFramework): msg = 'Input problem must be an instance of SlackFramework' raise ValueError, msg scale = kwargs.get('scale', True) self.verbose = kwargs.get('verbose', True) self.stabilize = kwargs.get('stabilize', True) self.lp = lp self.A = lp.A() # Constraint matrix if not isinstance(self.A, PysparseMatrix): self.A = PysparseMatrix(matrix=self.A) m, n = self.A.shape # Record number of slack variables in LP self.nSlacks = lp.n - lp.original_n # Constant vectors zero = np.zeros(n) self.b = -lp.cons(zero) # Right-hand side self.c0 = lp.obj(zero) # Constant term in objective self.c = lp.grad(zero[:lp.original_n]) #lp.cost() # Cost vector # Apply in-place problem scaling if requested. self.prob_scaled = False if scale: self.t_scale = cputime() self.scale() self.t_scale = cputime() - self.t_scale self.normb = norm2(self.b) self.normc = norm2(self.c) self.normbc = 1 + max(self.normb, self.normc) # Initialize augmented matrix self.H = PysparseMatrix(size=n+m, sizeHint=n+m+self.A.nnz, symmetric=True) # We perform the analyze phase on the augmented system only once. # self.LBL will be initialized in set_initial_guess(). self.LBL = None self.regpr = kwargs.get('regpr', 1.0) ; self.regpr_min = 1.0e-8 self.regdu = kwargs.get('regdu', 1.0) ; self.regdu_min = 1.0e-8 # Check input parameters. if self.regpr < 0.0: self.regpr = 0.0 if self.regdu < 0.0: self.regdu = 0.0 # Dual regularization is necessary for stabilization. if self.regdu == 0.0: sys.stderr.write('Warning: No dual regularization in effect\n') sys.stderr.write(' Stabilization has been turned off\n') self.stabilize = False # Initialize format strings for display fmt_hdr = '%-4s %9s' + ' %-8s'*6 + ' %-7s %-4s %-4s' + ' %-8s'*8 self.header = fmt_hdr % ('Iter', 'Cost', 'pResid', 'dResid', 'cResid', 'rGap', 'qNorm', 'rNorm', 'Mu', 'AlPr', 'AlDu', 'LS Resid', 'RegPr', 'RegDu', 'Rho q', 'Del r', 'Min(s)', 'Min(z)', 'Max(s)') self.format1 = '%-4d %9.2e' self.format1 += ' %-8.2e' * 6 self.format2 = ' %-7.1e %-4.2f %-4.2f' self.format2 += ' %-8.2e' * 8 + '\n' if self.verbose: self.display_stats() return
# Call projected CG to solve problem # min <g,d> + 1/2 <d, Hd> # s.t Jd = 0, |d| <= delta CG = Ppcg(g, SimpleLinearOperator(nlp.n, nlp.n, lambda p: nlp.hprod(nlp.x0, nlp.pi0, p), symmetric=True), A=J, radius=delta, debug=True) #CG = Ppcg(g, A=J, rhs=-c, matvec=hprod, debug=True) CG.Solve() Jd = numpy.empty(nlp.m, 'd') J.matvec(CG.x, Jd) feas = norms.norm_infty(Jd) #feas = norms.norm_infty( c + Jd ) # for nonzero rhs constraint snorm = norms.norm2(CG.x) # The trust-region is in the l2 norm sys.stdout.write('Number of variables : %-d\n' % nlp.n) sys.stdout.write('Number of constraints : %-d\n' % nlp.m) sys.stdout.write('Converged : %-s\n' % repr(CG.converged)) sys.stdout.write('Trust-region radius : %8.1e\n' % delta) sys.stdout.write('Solution norm : %8.1e\n' % snorm) sys.stdout.write('Final/Initial Residual : ') sys.stdout.write('%8.1e / %8.1e\n' % (CG.residNorm, CG.residNorm0)) sys.stdout.write('Feasibility error : %8.1e\n' % feas) sys.stdout.write('Number of iterations : %-d\n' % CG.iter)
def solve(self, **kwargs): """ Solve the input problem with the primal-dual-regularized interior-point method. Accepted input keyword arguments are :keywords: :itermax: The maximum allowed number of iterations (default: 10n) :tolerance: Stopping tolerance (default: 1.0e-6) :PredictorCorrector: Use the predictor-corrector method (default: `True`). If set to `False`, a variant of the long-step method is used. The long-step method is generally slower and less robust. Upon exit, the following members of the class instance are set: x..............final iterate y..............final value of the Lagrange multipliers associated to A1 x + A2 s = b z..............final value of the Lagrange multipliers associated to s>=0 obj_value......final cost iter...........total number of iterations kktResid.......final relative residual solve_time.....time to solve the LP status.........string describing the exit status short_status...short version of status, used for printing. """ lp = self.lp itermax = kwargs.get('itermax', max(100,10*lp.n)) tolerance = kwargs.get('tolerance', 1.0e-6) PredictorCorrector = kwargs.get('PredictorCorrector', True) check_infeasible = kwargs.get('check_infeasible', True) # Transfer pointers for convenience. m, n = self.A.shape ; on = lp.original_n A = self.A ; b = self.b ; c = self.c ; H = self.H regpr = self.regpr ; regdu = self.regdu regpr_min = self.regpr_min ; regdu_min = self.regdu_min # Obtain initial point from Mehrotra's heuristic. # set_initial_guess() initializes self.LBL which is reused below. (x,y,z) = self.set_initial_guess(self.lp, **kwargs) # Slack variables are the trailing variables in x. s = x[on:] ; ns = self.nSlacks # Initialize steps in dual variables. dz = np.zeros(ns) col_scale = np.empty(n) # Allocate room for right-hand side of linear systems. rhs = np.zeros(n+m) finished = False iter = 0 # Acceptance thresholds for primal and dual reg parameters. #t1 = t2 = 0.99 solve_time = cputime() # Main loop. while not finished: # Display initial header every so often. if self.verbose and iter % 20 == 0: sys.stdout.write('\n' + self.header + '\n') sys.stdout.write('-' * len(self.header) + '\n') # Compute residuals. pFeas = A*x - b comp = s*z ; sz = sum(comp) # comp = S z dFeas = y*A ; dFeas[:on] -= self.c # dFeas1 = A1'y - c dFeas[on:] += z # dFeas2 = A2'y + z mu = sz/ns # Compute residual norms and scaled residual norms. # We don't need to keep both the scaled and unscaled residuals # store. #pResid = norm_infty(pFeas + regdu * r)/(1+self.normc) #dResid = norm_infty(dFeas - regpr * q)/(1+self.normb) pResid = norm2(pFeas) ; spResid = pResid/(1+self.normc) cResid = norm2(comp) ; scResid = cResid/self.normbc dResid = norm2(dFeas) ; sdResid = dResid/(1+self.normb) # Compute relative duality gap. cx = np.dot(c,x[:on]) by = np.dot(b,y) rgap = cx - by #rgap += regdu * (rNorm**2 + np.dot(r,y)) rgap = abs(rgap) / (1 + abs(cx)) rgap2 = mu /(1 + abs(cx)) # Compute overall residual for stopping condition. kktResid = max(spResid, sdResid, rgap2) #kktResid = max(pResid, cResid, dResid) if kktResid <= tolerance: status = 'Optimal solution found' short_status = 'opt' finished = True continue if iter >= itermax: status = 'Maximum number of iterations reached' short_status= 'iter' finished = True continue # Adjust regularization parameters #mu = sum(comp)/ns #if mu < 1: # regpr = sqrt(mu) # regdu = sqrt(mu) # At the first iteration, initialize perturbation vectors # (q=primal, r=dual). if iter == 0: regpr = self.regpr ; regdu = self.regdu if regpr > 0: q = dFeas/regpr ; qNorm = norm2(q) ; rho_q = regpr * qNorm else: q = dFeas ; qNorm = norm2(q) ; rho_q = 0.0 rho_q_min = rho_q if regdu > 0: r = -pFeas/regdu ; rNorm = norm2(r) ; del_r = regdu * rNorm else: r = -pFeas ; rNorm = norm2(r) ; del_r = 0.0 del_r_min = del_r pr_infeas_count = 0 # Used to detect primal infeasibility. du_infeas_count = 0 # Used to detect dual infeasibility. pr_last_iter = 0 du_last_iter = 0 mu0 = mu else: # Adjust regularization parameters. #regpr = max(min(regpr/10, regpr**(1.1)), regpr_min) #regdu = max(min(regdu/10, regdu**(1.1)), regdu_min) # 1) rho+ |dx| <= const * s'z # 2) del+ |dy| <= const * s'z if regdu > 0: regdu = min(regdu/10, sz/normdy/10, (sz/normdy)**(1.1)) regdu = max(regdu, regdu_min) if regpr > 0: regpr = min(regpr/10, sz/normdx/10, (sz/normdx)**(1.1)) regpr = max(regpr, regpr_min) # Check for infeasible problem. if check_infeasible: #if mu < 1.0e-8 * mu0 and rho_q > 1.0e+3 * kktResid * self.normbc: #* mu * self.normbc: if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min: pr_infeas_count += 1 if pr_infeas_count > 1 and pr_last_iter == iter-1: if pr_infeas_count > 6: status = 'Problem seems to be (locally) dual infeasible' short_status = 'dInf' finished = True continue pr_last_iter = iter #if mu < 1.0e-8 * mu0 and del_r > 1.0e+3 * kktResid * self.normbc: # * mu * self.normbc: if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min: du_infeas_count += 1 if du_infeas_count > 1 and du_last_iter == iter-1: if du_infeas_count > 6: status='Problem seems to be (locally) primal infeasible' short_status = 'pInf' finished = True continue du_last_iter = iter # Display objective and residual data. if self.verbose: sys.stdout.write(self.format1 % (iter, cx, pResid, dResid, cResid, rgap, qNorm, rNorm)) # Record some quantities for display mins = np.min(s) minz = np.min(z) maxs = np.max(s) # Repeatedly assemble system and compute step until primal and # dual regularization parameters have appropriate values. # Reset primal and dual regularization parameters to best guess #if iter > 0: # regpr = max(regpr_min, 0.5*sigma*dResid/normds) # regdu = max(regdu_min, 0.5*sigma*pResid/normdy) step_acceptable = False while not step_acceptable: # Solve the linear system # # [-pI 0 A1'] [∆x] [c - A1' y ] # [ 0 -(S^{-1} Z + pI) A2'] [∆s] = [ - A2' y - µ S^{-1} e] # [ A1 A2 dI ] [∆y] [b - A1 x - A2 s ] # # where s are the slack variables, p is the primal # regularization parameter, d is the dual regularization # parameter, and A = [ A1 A2 ] where the columns of A1 # correspond to the original problem variables and those of A2 # correspond to slack variables. # # We recover ∆z = -z - S^{-1} (Z ∆s + µ e). # Compute augmented matrix and factorize it. factorized = False nb_bump = 0 while not factorized and nb_bump < 5: if self.stabilize: col_scale[:on] = sqrt(regpr) col_scale[on:] = np.sqrt(z/s + regpr) H.put(-sqrt(regdu), range(n)) H.put( sqrt(regdu), range(n,n+m)) AA = self.A.copy() AA.col_scale(1/col_scale) H[n:,:n] = AA else: if regpr > 0: H.put(-regpr, range(on)) H.put(-z/s - regpr, range(on,n)) if regdu > 0: H.put(regdu, range(n,n+m)) #if iter == 5: # # Export current matrix to file for futher inspection. # import os # name = os.path.basename(self.lp.name) # fname = '.'.join(name.split('.')[:-1]) + '.mtx' # H.exportMmf(fname) self.LBL.factorize(H) factorized = True # If the augmented matrix does not have full rank, bump up # regularization parameters. if not self.LBL.isFullRank: if self.verbose: sys.stderr.write('Primal-Dual Matrix ') sys.stderr.write('Rank Deficient') if regdu == 0.0: sys.stderr.write('... No regularization in effect') sys.stderr.write('... bailing out\n') factorized = False nb_bump = 5 continue else: sys.stderr.write('... bumping up reg parameters\n') regpr *= 10 ; regdu *= 10 nb_bump += 1 factorized = False # Abandon if regularization is unsuccessful. if not self.LBL.isFullRank and nb_bump >= 5: status = 'Unable to regularize sufficiently.' short_status = 'degn' finished = True continue # Does this get us out of the outer while? # Compute duality measure. mu = sz/ns if PredictorCorrector: # Use Mehrotra predictor-corrector method. # Compute affine-scaling step, i.e. with centering = 0. rhs[:n] = -dFeas rhs[on:n] += z rhs[n:] = -pFeas # if 'stabilize' is on, must scale right-hand side. if self.stabilize: rhs[:n] /= col_scale rhs[n:] /= sqrt(regdu) (step, nres, neig) = self.solveSystem(rhs) # Unscale step if 'stabilize' is on. if self.stabilize: step[:n] *= sqrt(regdu) / col_scale # Recover dx and dz. dx = step[:n] ds = dx[on:] dz = -z * (1 + ds/s) # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, ip) = self.maxStepLength(z, dz) # Estimate duality gap after affine-scaling step. muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns sigma = (muAff/mu)**3 # Incorporate predictor information for corrector step. comp += ds*dz else: # Use long-step method: Compute centering parameter. sigma = min(0.1, 100*mu) # Assemble right-hand side with centering information. comp -= sigma * mu if PredictorCorrector: # Only update rhs[on:n]; the rest of rhs did not change. if self.stabilize: rhs[on:n] += (comp/s - z)/col_scale[on:n] else: rhs[on:n] += comp/s - z else: rhs[:n] = -dFeas rhs[on:n] += comp/s rhs[n:] = -pFeas # If 'stabilize' is on, must scale right-hand side. # In the predictor-corrector method, this has already been # done. if self.stabilize: rhs[:n] /= col_scale rhs[n:] /= sqrt(regdu) # Solve augmented system. (step, nres, neig) = self.solveSystem(rhs) # Unscale step if 'stabilize' is on. if self.stabilize: step[:n] *= sqrt(regdu) / col_scale # Recover step. dx = step[:n] ds = dx[on:] dy = step[n:] normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx) step_acceptable = True # Must get rid of this # End while not step_acceptable # Recover step in z. dz = -(comp + z*ds)/s # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, id) = self.maxStepLength(z, dz) # Compute fraction-to-the-boundary factor. tau = max(.9995, 1.0-mu) if PredictorCorrector: # Compute actual stepsize using Mehrotra's heuristic mult = 0.1 # ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0 if (ip != -1 or id != -1) and ip != id: mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns if ip != -1 and ip != id: zip = z[ip] + alpha_d * dz[ip] gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip) alpha_p *= max(1-mult, gamma_p) if id != -1 and ip != id: sid = s[id] + alpha_p * ds[id] gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid) alpha_d *= max(1-mult, gamma_d) if ip==id and ip != -1: # There is a division by zero in Mehrotra's heuristic # Fall back on classical rule. alpha_p *= tau alpha_d *= tau else: alpha_p *= tau alpha_d *= tau # Display data. if self.verbose: sys.stdout.write(self.format2 % (mu, alpha_p, alpha_d, nres, regpr, regdu, rho_q, del_r, mins, minz, maxs)) # Update primal variables and slacks. x += alpha_p * dx # Update dual variables. y += alpha_d * dy z += alpha_d * dz # Update perturbation vectors. q *= (1-alpha_p) ; q += alpha_p * dx r *= (1-alpha_d) ; r += alpha_d * dy qNorm = norm2(q) ; rNorm = norm2(r) rho_q = regpr * qNorm/(1+self.normc) ; rho_q_min = min(rho_q_min, rho_q) del_r = regdu * rNorm/(1+self.normb) ; del_r_min = min(del_r_min, del_r) iter += 1 solve_time = cputime() - solve_time if self.verbose: sys.stdout.write('\n') sys.stdout.write('-' * len(self.header) + '\n') # Transfer final values to class members. self.x = x self.y = y self.z = z self.iter = iter self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid self.rgap = rgap self.kktResid = kktResid self.solve_time = solve_time self.status = status self.short_status = short_status # Unscale problem if applicable. if self.prob_scaled: self.unscale() # Recompute final objective value. self.obj_value = np.dot(self.c, x[:on]) + self.c0 return
#output_line += format2 % (regqp.mu, regqp.alpha_p, regqp.alpha_d, #regqp.nres, regqp.regpr, regqp.regdu, regqp.rho_q, #regqp.del_w, regqp.mins, regqp.minz, regqp.maxs) #log.info(output_line) log.info('-' * len(regqp.header)) #sys.stdout.write(fmt % (probname, regqp.iter, regqp.obj_value, #regqp.pResid, regqp.dResid, regqp.rgap, #t_setup, regqp.solve_time, regqp.short_status)) #if regqp.short_status == 'degn': #sys.stdout.write(' F') # Could not regularize sufficiently. #sys.stdout.write('\n') x = regqp.x[0:n] x0 = 0.1*np.ones([1,n]) x0[0] = -0.1 norm_x0_x = norm2(x0-x) nnz = nnz_elements(x,1e-3) Probname = str(m)+'--'+str(n) numpy.set_printoptions(threshold=5) numpy.set_printoptions(threshold='nan') print x[0:10] log.info('difference between initialize point and minimizer %6.f'%norm_x0_x) x = regqp.x[0:n] log.info('#Iterations: %-d' % regqp.iter) log.info('RelResidual: %7.1e' % regqp.kktResid) log.info('Final cost : %21.15e' % regqp.obj_value) log.info('Setup time : %6.2fs' % t_setup)
# J.matvec(e, c) # Call projected CG to solve problem # min <g,d> + 1/2 <d, Hd> # s.t Jd = 0, |d| <= delta CG = Ppcg( g, SimpleLinearOperator(nlp.n, nlp.n, lambda p: nlp.hprod(nlp.x0, nlp.pi0, p), symmetric=True), A=J, radius=delta, debug=True, ) # CG = Ppcg(g, A=J, rhs=-c, matvec=hprod, debug=True) CG.Solve() Jd = numpy.empty(nlp.m, "d") J.matvec(CG.x, Jd) feas = norms.norm_infty(Jd) # feas = norms.norm_infty( c + Jd ) # for nonzero rhs constraint snorm = norms.norm2(CG.x) # The trust-region is in the l2 norm sys.stdout.write("Number of variables : %-d\n" % nlp.n) sys.stdout.write("Number of constraints : %-d\n" % nlp.m) sys.stdout.write("Converged : %-s\n" % repr(CG.converged)) sys.stdout.write("Trust-region radius : %8.1e\n" % delta) sys.stdout.write("Solution norm : %8.1e\n" % snorm) sys.stdout.write("Final/Initial Residual : ") sys.stdout.write("%8.1e / %8.1e\n" % (CG.residNorm, CG.residNorm0)) sys.stdout.write("Feasibility error : %8.1e\n" % feas) sys.stdout.write("Number of iterations : %-d\n" % CG.iter)
if probname[-3:] == '.nl': probname = probname[:-3] if not options.verbose: log.info(fmt % (probname, regqp.iter, regqp.obj_value, regqp.pResid, regqp.dResid, regqp.rgap, t_setup, regqp.solve_time, regqp.short_status)) if regqp.short_status == 'degn': log.info(' F') # Could not regularize sufficiently. qp.close() log.info('-'*len(hdr)) if not multiple_problems: x = regqp.x[:qp.original_n] log.info('Final x: %s, |x| = %7.1e' % (repr(x),norm2(x))) log.info('Final y: %s, |y| = %7.1e' % (repr(regqp.y),norm2(regqp.y))) log.info('Final z: %s, |z| = %7.1e' % (repr(regqp.z),norm2(regqp.z))) log.info(regqp.status) log.info('#Iterations: %-d' % regqp.iter) log.info('RelResidual: %7.1e' % regqp.kktResid) log.info('Final cost : %21.15e' % regqp.obj_value) log.info('Setup time : %6.2fs' % t_setup) log.info('Solve time : %6.2fs' % regqp.solve_time) # Plot linear system statistics. import matplotlib.pyplot as plt fig = plt.figure() ax = fig.gca() ax.semilogy(regqp.lres_history)
#regqp.dResid, regqp.cResid, regqp.rgap, regqp.qNorm, #regqp.wNorm) #output_line += format2 % (regqp.mu, regqp.alpha_p, regqp.alpha_d, #regqp.nres, regqp.regpr, regqp.regdu, regqp.rho_q, #regqp.del_w, regqp.mins, regqp.minz, regqp.maxs) #log.info(output_line) log.info('-' * len(regqp.header)) #sys.stdout.write(fmt % (probname, regqp.iter, regqp.obj_value, #regqp.pResid, regqp.dResid, regqp.rgap, #t_setup, regqp.solve_time, regqp.short_status)) #if regqp.short_status == 'degn': #sys.stdout.write(' F') # Could not regularize sufficiently. #sys.stdout.write('\n') x = regqp.x[0:n] norm_x0_x = norm2(.1*np.ones([1,n])-x) nnz = nnz_elements(x,1e-3) Probname = str(n)+'--'+str(m) numpy.set_printoptions(threshold=5) numpy.set_printoptions(threshold='nan') #print x[1:10] log.info('Non zero elements in minimizer %6.f'%nnz) x = regqp.x[0:n] log.info('#Iterations: %-d' % regqp.iter) log.info('RelResidual: %7.1e' % regqp.kktResid)
def SolveInner(self, **kwargs): """ Perform a series of inner iterations so as to minimize the primal-dual merit function with the current value of the barrier parameter to within some given tolerance. The only optional argument recognized is stopTol stopping tolerance (default: muerrfact * mu). """ merit = self.merit nx = merit.nlp.n nz = merit.nz rho = 1 # Dummy initial value for rho niter = 0 # Dummy initial value for number of inners status = '' # Dummy initial step status alpha = 0.0 # Fraction-to-the-boundary step size if self.inexact: cgtol = 1.0 else: cgtol = -1.0 inner_iter = 0 # Inner iteration counter # Obtain starting point. (x, z) = (self.x, self.z) # Obtain first-order data at starting point. if self.iter == 0: f = nlp.obj(x) gf = nlp.grad(x) else: f = self.f gf = self.gf psi = merit.obj(x, z, f=f) g = merit.grad(x, z, g=gf, check_optimal=True) gNorm = norm2(g) if self.optimal: return # Reset initial trust-region radius self.TR.Delta = 0.1 * gNorm # Set inner iteration stopping tolerance stopTol = kwargs.get('stopTol', self.muerrfact * self.mu) finished = (gNorm <= stopTol) or (self.iter >= self.maxiter) while not finished: # Print out header every so many iterations if self.verbose: if self.iter % self.printFrequency == 0: sys.stdout.write(self.hline) sys.stdout.write(self.header) sys.stdout.write(self.hline) if inner_iter == 0: sys.stdout.write(('*' + self.itFormat) % self.iter) else: sys.stdout.write((' ' + self.itFormat) % self.iter) sys.stdout.write( self.format % (f, max(norm_infty(self.dRes), norm_infty(self.cRes), norm_infty(self.pRes)), gNorm, self.mu, alpha, niter, rho, self.TR.Delta, status)) # Set stopping tolerance for trust-region subproblem. if self.inexact: cgtol = max(1.0e-8, min(0.1 * cgtol, sqrt(gNorm))) if self.debug: self._debugMsg('cgtol = ' + str(cgtol)) # Update Hessian matrix with current iteration information. # self.PDHess(x,z) if self.debug: self._debugMsg('g = ' + np.str(g)) self._debugMsg('gNorm = ' + str(gNorm)) self._debugMsg('stopTol = ' + str(stopTol)) self._debugMsg('dRes = ' + np.str(self.dRes)) self._debugMsg('cRes = ' + np.str(self.cRes)) self._debugMsg('pRes = ' + np.str(self.pRes)) self._debugMsg('optimal = ' + str(self.optimal)) # Set up the preconditioner if applicable. self.SetupPrecon() # Iteratively minimize the quadratic model in the trust region # m(s) = <g, s> + 1/2 <s, Hs> # Note that m(s) does not include f(x): m(0) = 0. H = SimpleLinearOperator(nx + nz, nx + nz, lambda v: merit.hprod(v), symmetric=True) subsolver = self.TrSolver( g, H, prec=self.Precon, radius=self.TR.Delta, reltol=cgtol, #fraction = 0.5, itmax=2 * (n + nz), #debug=True, #btol=.9, #cur_iter=np.concatenate((x,z)) ) subsolver.Solve() if self.debug: self._debugMsg('x = ' + np.str(x)) self._debugMsg('z = ' + np.str(z)) self._debugMsg('step = ' + np.str(solver.step)) self._debugMsg('step norm = ' + str(solver.stepNorm)) # Record total number of CG iterations. niter = subsolver.niter self.cgiter += subsolver.niter # Compute maximal step to the boundary and next candidate. alphax, alphaz = self.ftb(x, z, solver.step) alpha = min(alphax, alphaz) dx = subsolver.step[:n] dz = subsolver.step[n:] x_trial = x + alphax * dx z_trial = z + alphaz * dz f_trial = merit.nlp.obj(x_trial) psi_trial = merit.obj(x_trial, z_trial, f=f_trial) # Compute ratio of predicted versus achieved reduction. rho = self.TR.Rho(psi, psi_trial, subsolver.m) if self.debug: self._debugMsg('m = ' + str(solver.m)) self._debugMsg('x_trial = ' + np.str(x_trial)) self._debugMsg('z_trial = ' + np.str(z_trial)) self._debugMsg('psi_trial = ' + str(psi_trial)) self._debugMsg('rho = ' + str(rho)) # Accept or reject next candidate status = 'Rej' if rho >= self.TR.eta1: self.TR.UpdateRadius(rho, solver.stepNorm) x = x_trial z = z_trial f = f_trial psi = psi_trial gf = nlp.grad(x) g = self.GradPDMerit(x, z, g=gf, check_optimal=True) gNorm = norm2(g) if self.optimal: finished = True continue status = 'Acc' else: if self.ny: # Backtracking linesearch a la "Nocedal & Yuan" slope = np.dot(g, solver.step) target = psi + 1.0e-4 * alpha * slope j = 0 while (psi_trial >= target) and (j < self.nyMax): alphax /= 1.2 alphaz /= 1.2 alpha = min(alphax, alphaz) target = psi + 1.0e-4 * alpha * slope x_trial = x + alphax * dx z_trial = z + alphaz * dz f_trial = nlp.obj(x_trial) psi_trial = self.PDMerit(x_trial, z_trial, f=f_trial) j += 1 if self.opportunistic or (j < self.nyMax): x = x_trial z = z_trial #self.PrimalMultipliers(x) f = f_trial psi = psi_trial gf = nlp.grad(x) g = self.GradPDMerit(x, z, g=gf, check_optimal=True) gNorm = norm2(g) if self.optimal: finished = True continue self.TR.Delta = alpha * solver.stepNorm status = 'N-Y' else: self.TR.UpdateRadius(rho, solver.stepNorm) else: self.TR.UpdateRadius(rho, solver.stepNorm) self.UpdatePrecon() self.iter += 1 inner_iter += 1 finished = (gNorm <= stopTol) or (self.iter >= self.maxiter) if self.debug: sys.stderr.write('\n') # Store final iterate (self.x, self.z) = (x, z) self.f = f self.gf = gf self.g = g self.gNorm = gNorm self.psi = psi return
if not options.verbose: log.info( fmt % (probname, regqp.iter, regqp.obj_value, regqp.pResid, regqp.dResid, regqp.rgap, t_setup, regqp.solve_time, regqp.short_status)) if regqp.short_status == 'degn': log.info(' F') # Could not regularize sufficiently. qp.close() log.info('-' * len(hdr)) if not multiple_problems: x = regqp.x[:qp.original_n] log.info('Final x: %s, |x| = %7.1e' % (repr(x), norm2(x))) log.info('Final y: %s, |y| = %7.1e' % (repr(regqp.y), norm2(regqp.y))) log.info('Final z: %s, |z| = %7.1e' % (repr(regqp.z), norm2(regqp.z))) log.info(regqp.status) log.info('#Iterations: %-d' % regqp.iter) log.info('RelResidual: %7.1e' % regqp.kktResid) log.info('Final cost : %21.15e' % regqp.obj_value) log.info('Setup time : %6.2fs' % t_setup) log.info('Solve time : %6.2fs' % regqp.solve_time) # Plot linear system statistics. import matplotlib.pyplot as plt fig = plt.figure() ax = fig.gca() ax.semilogy(regqp.lres_history)
def solve(self, **kwargs): """ Solve the input problem with the primal-dual-regularized interior-point method. Accepted input keyword arguments are :keywords: :itermax: The maximum allowed number of iterations (default 10n) :tolerance: Stopping tolerance (default 1.0e-6) :PredictorCorrector: Use the predictor-corrector method (default `True`). If set to `False`, a variant of the long-step method is used. The long-step method is generally slower and less robust. :return: :x: final iterate :y: final value of the Lagrange multipliers associated to `A1 x + A2 s = b` :z: final value of the Lagrange multipliers associated to `s >= 0` :obj_value: final cost :iter: total number of iterations :kktResid: final relative residual :solve_time: time to solve the LP :status: string describing the exit status :short_status: short version of status, used for printing. """ lp = self.lp itermax = kwargs.get('itermax', max(100,10*lp.n)) tolerance = kwargs.get('tolerance', 1.0e-6) PredictorCorrector = kwargs.get('PredictorCorrector', True) check_infeasible = kwargs.get('check_infeasible', True) # Transfer pointers for convenience. m, n = self.A.shape ; on = lp.original_n A = self.A ; b = self.b ; c = self.c ; H = self.H regpr = self.regpr ; regdu = self.regdu regpr_min = self.regpr_min ; regdu_min = self.regdu_min # Obtain initial point from Mehrotra's heuristic. # set_initial_guess() initializes self.LBL which is reused below. (x,y,z) = self.set_initial_guess(self.lp, **kwargs) # Slack variables are the trailing variables in x. s = x[on:] ; ns = self.nSlacks # Initialize steps in dual variables. dz = np.zeros(ns) col_scale = np.empty(n) # Allocate room for right-hand side of linear systems. rhs = np.zeros(n+m) finished = False iter = 0 # Acceptance thresholds for primal and dual reg parameters. #t1 = t2 = 0.99 solve_time = cputime() # Main loop. while not finished: # Display initial header every so often. if self.verbose and iter % 20 == 0: sys.stdout.write('\n' + self.header + '\n') sys.stdout.write('-' * len(self.header) + '\n') # Compute residuals. pFeas = A*x - b comp = s*z ; sz = sum(comp) # comp = S z dFeas = y*A ; dFeas[:on] -= self.c # dFeas1 = A1'y - c dFeas[on:] += z # dFeas2 = A2'y + z mu = sz/ns # Compute residual norms and scaled residual norms. pResid = norm2(pFeas) ; spResid = pResid/(1+self.normb+self.normA) cResid = norm2(comp) ; scResid = cResid/self.normbc dResid = norm2(dFeas) ; sdResid = dResid/(1+self.normc+self.normA) # Compute relative duality gap. cx = np.dot(c,x[:on]) by = np.dot(b,y) rgap = cx - by rgap = abs(rgap) / (1 + abs(cx)) rgap2 = mu /(1 + abs(cx)) # Compute overall residual for stopping condition. kktResid = max(spResid, sdResid, rgap2) #kktResid = max(pResid, cResid, dResid) if kktResid <= tolerance: status = 'Optimal solution found' short_status = 'opt' finished = True continue if iter >= itermax: status = 'Maximum number of iterations reached' short_status= 'iter' finished = True continue # Adjust regularization parameters #mu = sum(comp)/ns #if mu < 1: # regpr = sqrt(mu) # regdu = sqrt(mu) # At the first iteration, initialize perturbation vectors # (q=primal, r=dual). if iter == 0: regpr = self.regpr ; regdu = self.regdu if regpr > 0: q = dFeas/regpr ; qNorm = norm2(q) ; rho_q = regpr * qNorm else: q = dFeas ; qNorm = norm2(q) ; rho_q = 0.0 rho_q_min = rho_q if regdu > 0: r = -pFeas/regdu ; rNorm = norm2(r) ; del_r = regdu * rNorm else: r = -pFeas ; rNorm = norm2(r) ; del_r = 0.0 del_r_min = del_r pr_infeas_count = 0 # Used to detect primal infeasibility. du_infeas_count = 0 # Used to detect dual infeasibility. pr_last_iter = 0 du_last_iter = 0 mu0 = mu else: # Adjust regularization parameters. #regpr = max(min(regpr/10, regpr**(1.1)), regpr_min) #regdu = max(min(regdu/10, regdu**(1.1)), regdu_min) # 1) rho+ |dx| <= const * s'z # 2) del+ |dy| <= const * s'z if regdu > 0: regdu = min(regdu/10, sz/normdy/10, (sz/normdy)**(1.1)) regdu = max(regdu, regdu_min) if regpr > 0: regpr = min(regpr/10, sz/normdx/10, (sz/normdx)**(1.1)) regpr = max(regpr, regpr_min) # Check for infeasible problem. if check_infeasible: #if mu < 1.0e-8 * mu0 and rho_q > 1.0e+3 * kktResid * self.normbc: #* mu * self.normbc: if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min: pr_infeas_count += 1 if pr_infeas_count > 1 and pr_last_iter == iter-1: if pr_infeas_count > 6: status = 'Problem seems to be (locally) dual infeasible' short_status = 'dInf' finished = True continue pr_last_iter = iter #if mu < 1.0e-8 * mu0 and del_r > 1.0e+3 * kktResid * self.normbc: # * mu * self.normbc: if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min: du_infeas_count += 1 if du_infeas_count > 1 and du_last_iter == iter-1: if du_infeas_count > 6: status='Problem seems to be (locally) primal infeasible' short_status = 'pInf' finished = True continue du_last_iter = iter # Display objective and residual data. if self.verbose: sys.stdout.write(self.format1 % (iter, cx, pResid, dResid, cResid, rgap, qNorm, rNorm)) # Record some quantities for display mins = np.min(s) minz = np.min(z) maxs = np.max(s) # Repeatedly assemble system and compute step until primal and # dual regularization parameters have appropriate values. # Reset primal and dual regularization parameters to best guess #if iter > 0: # regpr = max(regpr_min, 0.5*sigma*dResid/normds) # regdu = max(regdu_min, 0.5*sigma*pResid/normdy) step_acceptable = False while not step_acceptable: # Solve the linear system # # [-pI 0 A1'] [∆x] [c - A1' y ] # [ 0 -(S^{-1} Z + pI) A2'] [∆s] = [ - A2' y - µ S^{-1} e] # [ A1 A2 dI ] [∆y] [b - A1 x - A2 s ] # # where s are the slack variables, p is the primal # regularization parameter, d is the dual regularization # parameter, and A = [ A1 A2 ] where the columns of A1 # correspond to the original problem variables and those of A2 # correspond to slack variables. # # We recover ∆z = -z - S^{-1} (Z ∆s + µ e). # Compute augmented matrix and factorize it. factorized = False nb_bump = 0 while not factorized and nb_bump < 5: if self.stabilize: col_scale[:on] = sqrt(regpr) col_scale[on:] = np.sqrt(z/s + regpr) H.put(-sqrt(regdu), range(n)) H.put( sqrt(regdu), range(n,n+m)) AA = self.A.copy() AA.col_scale(1/col_scale) H[n:,:n] = AA else: if regpr > 0: H.put(-regpr, range(on)) H.put(-z/s - regpr, range(on,n)) if regdu > 0: H.put(regdu, range(n,n+m)) #if iter == 5: # # Export current matrix to file for futher inspection. # import os # name = os.path.basename(self.lp.name) # fname = '.'.join(name.split('.')[:-1]) + '.mtx' # H.exportMmf(fname) self.LBL.factorize(H) factorized = True # If the augmented matrix does not have full rank, bump up # regularization parameters. if not self.LBL.isFullRank: if self.verbose: sys.stderr.write('Primal-Dual Matrix ') sys.stderr.write('Rank Deficient') if regdu == 0.0: sys.stderr.write('... No regularization in effect') sys.stderr.write('... bailing out\n') factorized = False nb_bump = 5 continue else: sys.stderr.write('... bumping up reg parameters\n') regpr *= 10 ; regdu *= 10 nb_bump += 1 factorized = False # Abandon if regularization is unsuccessful. if not self.LBL.isFullRank and nb_bump >= 5: status = 'Unable to regularize sufficiently.' short_status = 'degn' finished = True continue # Does this get us out of the outer while? # Compute duality measure. mu = sz/ns if PredictorCorrector: # Use Mehrotra predictor-corrector method. # Compute affine-scaling step, i.e. with centering = 0. rhs[:n] = -dFeas rhs[on:n] += z rhs[n:] = -pFeas # if 'stabilize' is on, must scale right-hand side. if self.stabilize: rhs[:n] /= col_scale rhs[n:] /= sqrt(regdu) (step, nres, neig) = self.solveSystem(rhs) # Unscale step if 'stabilize' is on. if self.stabilize: step[:n] *= sqrt(regdu) / col_scale # Recover dx and dz. dx = step[:n] ds = dx[on:] dz = -z * (1 + ds/s) # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, ip) = self.maxStepLength(z, dz) # Estimate duality gap after affine-scaling step. muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns sigma = (muAff/mu)**3 # Incorporate predictor information for corrector step. comp += ds*dz else: # Use long-step method: Compute centering parameter. sigma = min(0.1, 100*mu) # Assemble right-hand side with centering information. comp -= sigma * mu if PredictorCorrector: # Only update rhs[on:n]; the rest of rhs did not change. if self.stabilize: rhs[on:n] += (comp/s - z)/col_scale[on:n] else: rhs[on:n] += comp/s - z else: rhs[:n] = -dFeas rhs[on:n] += comp/s rhs[n:] = -pFeas # If 'stabilize' is on, must scale right-hand side. # In the predictor-corrector method, this has already been # done. if self.stabilize: rhs[:n] /= col_scale rhs[n:] /= sqrt(regdu) # Solve augmented system. (step, nres, neig) = self.solveSystem(rhs) # Unscale step if 'stabilize' is on. if self.stabilize: step[:n] *= sqrt(regdu) / col_scale # Recover step. dx = step[:n] ds = dx[on:] dy = step[n:] normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx) step_acceptable = True # Must get rid of this # End while not step_acceptable # Recover step in z. dz = -(comp + z*ds)/s # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, id) = self.maxStepLength(z, dz) # Compute fraction-to-the-boundary factor. tau = max(.9995, 1.0-mu) if PredictorCorrector: # Compute actual stepsize using Mehrotra's heuristic mult = 0.1 # ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0 if (ip != -1 or id != -1) and ip != id: mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns if ip != -1 and ip != id: zip = z[ip] + alpha_d * dz[ip] gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip) alpha_p *= max(1-mult, gamma_p) if id != -1 and ip != id: sid = s[id] + alpha_p * ds[id] gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid) alpha_d *= max(1-mult, gamma_d) if ip==id and ip != -1: # There is a division by zero in Mehrotra's heuristic # Fall back on classical rule. alpha_p *= tau alpha_d *= tau else: alpha_p *= tau alpha_d *= tau # Display data. if self.verbose: sys.stdout.write(self.format2 % (mu, alpha_p, alpha_d, nres, regpr, regdu, rho_q, del_r, mins, minz, maxs)) # Update primal variables and slacks. x += alpha_p * dx # Update dual variables. y += alpha_d * dy z += alpha_d * dz # Update perturbation vectors. q *= (1-alpha_p) ; q += alpha_p * dx r *= (1-alpha_d) ; r += alpha_d * dy qNorm = norm2(q) ; rNorm = norm2(r) rho_q = regpr * qNorm/(1+self.normc) ; rho_q_min = min(rho_q_min, rho_q) del_r = regdu * rNorm/(1+self.normb) ; del_r_min = min(del_r_min, del_r) iter += 1 solve_time = cputime() - solve_time if self.verbose: sys.stdout.write('\n') sys.stdout.write('-' * len(self.header) + '\n') # Transfer final values to class members. self.x = x self.y = y self.z = z self.iter = iter self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid self.rgap = rgap self.kktResid = kktResid self.solve_time = solve_time self.status = status self.short_status = short_status # Unscale problem if applicable. if self.prob_scaled: self.unscale() # Recompute final objective value. self.obj_value = np.dot(self.c, x[:on]) + self.c0 return
def scale(self, **kwargs): """ Equilibrate the constraint matrix of the linear program. Equilibration is done by first dividing every row by its largest element in absolute value and then by dividing every column by its largest element in absolute value. In effect the original problem:: minimize c' x + 1/2 x' Q x subject to A1 x + A2 s = b, x >= 0 is converted to:: minimize (Cc)' x + 1/2 x' (CQC') x subject to R A1 C x + R A2 C s = Rb, x >= 0, where the diagonal matrices R and C operate row and column scaling respectively. Upon return, the matrix A and the right-hand side b are scaled and the members `row_scale` and `col_scale` are set to the row and column scaling factors. The scaling may be undone by subsequently calling :meth:`unscale`. It is necessary to unscale the problem in order to unscale the final dual variables. Normally, the :meth:`solve` method takes care of unscaling the problem upon termination. """ log = self.log m, n = self.A.shape row_scale = np.zeros(m) col_scale = np.zeros(n) (values,irow,jcol) = self.A.find() if self.verbose: log.info('Smallest and largest elements of A prior to scaling: ') log.info('%8.2e %8.2e' % (np.min(np.abs(values)), np.max(np.abs(values)))) # Find row scaling. for k in range(len(values)): row = irow[k] val = abs(values[k]) row_scale[row] = max(row_scale[row], val) row_scale[row_scale == 0.0] = 1.0 if self.verbose: log.info('Max row scaling factor = %8.2e' % np.max(row_scale)) # Apply row scaling to A and b. values /= row_scale[irow] self.b /= row_scale # Find column scaling. for k in range(len(values)): col = jcol[k] val = abs(values[k]) col_scale[col] = max(col_scale[col], val) col_scale[col_scale == 0.0] = 1.0 if self.verbose: log.info('Max column scaling factor = %8.2e' % np.max(col_scale)) # Apply column scaling to A and c. values /= col_scale[jcol] self.c[:self.qp.original_n] /= col_scale[:self.qp.original_n] if self.verbose: log.info('Smallest and largest elements of A after scaling: ') log.info('%8.2e %8.2e' % (np.min(np.abs(values)), np.max(np.abs(values)))) # Overwrite A with scaled values. self.A.put(values,irow,jcol) self.normA = norm2(values) # Frobenius norm of A. # Apply scaling to Hessian matrix Q. (values,irow,jcol) = self.Q.find() values /= col_scale[irow] values /= col_scale[jcol] self.Q.put(values,irow,jcol) self.normQ = norm2(values) # Frobenius norm of Q # Save row and column scaling. self.row_scale = row_scale self.col_scale = col_scale self.prob_scaled = True return
hdr_fmt = '%-13s %-11s %-11s %-11s %-7s %-7s %-5s\n' res_fmt = '%-13s %-11.5e %-11.5e %-11.5e %-7.3f %-7.3f %-5d\n' hdrs = ('Name', 'Rel. resid.', 'Residual', 'Resid itref', 'Analyze', 'Solve', 'neig') header = hdr_fmt % hdrs lhead = len(header) sys.stderr.write('-' * lhead + '\n') sys.stderr.write(header) sys.stderr.write('-' * lhead + '\n') # Solve example from the spec sheet (A, rhs) = Ma27SpecSheet() (x, r, nr, nr1, t_an, t_sl, neig) = SolveSystem(A, rhs) exact = numpy.arange(5, dtype = 'd') + 1 relres = norms.norm2(x - exact) / norms.norm2(exact) sys.stdout.write(res_fmt % ('Spec sheet',relres,nr,nr1,t_an,t_sl,neig)) # Solve example with Hilbert matrix n = 10 H = Hilbert(n) e = numpy.ones(n, 'd') rhs = numpy.empty(n, 'd') H.matvec(e, rhs) (x, r, nr, nr1, t_an, t_sl, neig) = SolveSystem(H, rhs) relres = norms.norm2(x - e) / norms.norm2(e) sys.stdout.write(res_fmt % ('Hilbert', relres, nr, nr1, t_an, t_sl, neig)) # Process matrices given on the command line for matrix in matrices: M = spmatrix.ll_mat_from_mtx(matrix)
**opts_solve) # Display summary line. probname=os.path.basename(probname) if probname[-3:] == '.nl': probname = probname[:-3] if not options.verbose: sys.stdout.write(fmt % (probname, regqp.iter, regqp.obj_value, regqp.pResid, regqp.dResid, regqp.rgap, t_setup, regqp.solve_time, regqp.short_status)) if regqp.short_status == 'degn': sys.stdout.write(' F') # Could not regularize sufficiently. sys.stdout.write('\n') qp.close() if not options.verbose: sys.stderr.write('-'*len(hdr) + '\n') else: x = regqp.x[:qp.original_n] print 'Final x: ', x, ', |x| = %7.1e' % norm2(x) print 'Final y: ', regqp.y, ', |y| = %7.1e' % norm2(regqp.y) print 'Final z: ', regqp.z, ', |z| = %7.1e' % norm2(regqp.z) sys.stdout.write('\n' + regqp.status + '\n') sys.stdout.write(' #Iterations: %-d\n' % regqp.iter) sys.stdout.write(' RelResidual: %7.1e\n' % regqp.kktResid) sys.stdout.write(' Final cost : %21.15e\n' % regqp.obj_value) sys.stdout.write(' Setup time : %6.2fs\n' % t_setup) sys.stdout.write(' Solve time : %6.2fs\n' % regqp.solve_time)
def solve(self, **kwargs): """ Solve the input problem with the primal-dual-regularized interior-point method. Accepted input keyword arguments are :keywords: :itermax: The maximum allowed number of iterations (default: 10n) :tolerance: Stopping tolerance (default: 1.0e-6) :PredictorCorrector: Use the predictor-corrector method (default: `True`). If set to `False`, a variant of the long-step method is used. The long-step method is generally slower and less robust. :returns: :x: final iterate :y: final value of the Lagrange multipliers associated to `A1 x + A2 s = b` :z: final value of the Lagrange multipliers associated to `s >= 0` :obj_value: final cost :iter: total number of iterations :kktResid: final relative residual :solve_time: time to solve the QP :status: string describing the exit status. :short_status: short version of status, used for printing. """ qp = self.qp itermax = kwargs.get('itermax', max(100, 10 * qp.n)) tolerance = kwargs.get('tolerance', 1.0e-6) PredictorCorrector = kwargs.get('PredictorCorrector', True) check_infeasible = kwargs.get('check_infeasible', True) # Transfer pointers for convenience. m, n = self.A.shape on = qp.original_n A = self.A b = self.b c = self.c Q = self.Q diagQ = self.diagQ H = self.H regpr = self.regpr regdu = self.regdu regpr_min = self.regpr_min regdu_min = self.regdu_min # Obtain initial point from Mehrotra's heuristic. (x, y, z) = self.set_initial_guess(**kwargs) # Slack variables are the trailing variables in x. s = x[on:] ns = self.nSlacks # Initialize steps in dual variables. dz = np.zeros(ns) # Allocate room for right-hand side of linear systems. rhs = self.initialize_rhs() finished = False iter = 0 setup_time = cputime() # Main loop. while not finished: # Display initial header every so often. if iter % 50 == 0: self.log.info(self.header) self.log.info('-' * len(self.header)) # Compute residuals. pFeas = A * x - b comp = s * z sz = sum(comp) # comp = Sz Qx = Q * x[:on] dFeas = y * A dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx dFeas[on:] += z # dFeas2 = A2'y + z # Compute duality measure. if ns > 0: mu = sz / ns else: mu = 0.0 # Compute residual norms and scaled residual norms. pResid = norm2(pFeas) spResid = pResid / (1 + self.normb + self.normA + self.normQ) dResid = norm2(dFeas) sdResid = dResid / (1 + self.normc + self.normA + self.normQ) if ns > 0: cResid = norm_infty(comp) / (self.normbc + self.normA + self.normQ) else: cResid = 0.0 # Compute relative duality gap. cx = np.dot(c, x[:on]) xQx = np.dot(x[:on], Qx) by = np.dot(b, y) rgap = cx + xQx - by rgap = abs(rgap) / (1 + abs(cx) + self.normA + self.normQ) rgap2 = mu / (1 + abs(cx) + self.normA + self.normQ) # Compute overall residual for stopping condition. kktResid = max(spResid, sdResid, rgap2) # At the first iteration, initialize perturbation vectors # (q=primal, r=dual). # Should probably get rid of q when regpr=0 and of r when regdu=0. if iter == 0: if regpr > 0: q = dFeas / regpr qNorm = dResid / regpr rho_q = dResid else: q = dFeas qNorm = dResid rho_q = 0.0 rho_q_min = rho_q if regdu > 0: r = -pFeas / regdu rNorm = pResid / regdu del_r = pResid else: r = -pFeas rNorm = pResid del_r = 0.0 del_r_min = del_r pr_infeas_count = 0 # Used to detect primal infeasibility. du_infeas_count = 0 # Used to detect dual infeasibility. pr_last_iter = 0 du_last_iter = 0 mu0 = mu else: if regdu > 0: regdu = regdu / 10 regdu = max(regdu, regdu_min) if regpr > 0: regpr = regpr / 10 regpr = max(regpr, regpr_min) # Check for infeasible problem. if check_infeasible: if mu < tolerance/100 * mu0 and \ rho_q > 1./tolerance/1.0e+6 * rho_q_min: pr_infeas_count += 1 if pr_infeas_count > 1 and pr_last_iter == iter - 1: if pr_infeas_count > 6: status = 'Problem seems to be (locally) dual' status += ' infeasible' short_status = 'dInf' finished = True continue pr_last_iter = iter else: pr_infeas_count = 0 if mu < tolerance/100 * mu0 and \ del_r > 1./tolerance/1.0e+6 * del_r_min: du_infeas_count += 1 if du_infeas_count > 1 and du_last_iter == iter - 1: if du_infeas_count > 6: status = 'Problem seems to be (locally) primal' status += ' infeasible' short_status = 'pInf' finished = True continue du_last_iter = iter else: du_infeas_count = 0 # Display objective and residual data. output_line = self.format1 % (iter, cx + 0.5 * xQx, pResid, dResid, cResid, rgap, qNorm, rNorm) if kktResid <= tolerance: status = 'Optimal solution found' short_status = 'opt' finished = True continue if iter >= itermax: status = 'Maximum number of iterations reached' short_status = 'iter' finished = True continue # Record some quantities for display if ns > 0: mins = np.min(s) minz = np.min(z) maxs = np.max(s) else: mins = minz = maxs = 0 # Compute augmented matrix and factorize it. factorized = False degenerate = False nb_bump = 0 while not factorized and not degenerate: self.update_linear_system(s, z, regpr, regdu) self.log.debug('Factorizing') self.LBL.factorize(H) factorized = True # If the augmented matrix does not have full rank, bump up the # regularization parameters. if not self.LBL.isFullRank: if self.verbose: self.log.info('Primal-Dual Matrix Rank Deficient' + \ '... bumping up reg parameters') if regpr == 0. and regdu == 0.: degenerate = True else: if regpr > 0: regpr *= 100 if regdu > 0: regdu *= 100 nb_bump += 1 degenerate = nb_bump > self.bump_max factorized = False # Abandon if regularization is unsuccessful. if not self.LBL.isFullRank and degenerate: status = 'Unable to regularize sufficiently.' short_status = 'degn' finished = True continue if PredictorCorrector: # Use Mehrotra predictor-corrector method. # Compute affine-scaling step, i.e. with centering = 0. self.set_affine_scaling_rhs(rhs, pFeas, dFeas, s, z) (step, nres, neig) = self.solveSystem(rhs) # Recover dx and dz. dx, ds, dy, dz = self.get_affine_scaling_dxsyz( step, x, s, y, z) # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, ip) = self.maxStepLength(z, dz) # Estimate duality gap after affine-scaling step. muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns sigma = (muAff / mu)**3 # Incorporate predictor information for corrector step. # Only update rhs[on:n]; the rest of the vector did not change. comp += ds * dz comp -= sigma * mu self.update_corrector_rhs(rhs, s, z, comp) else: # Use long-step method: Compute centering parameter. sigma = min(0.1, 100 * mu) comp -= sigma * mu # Assemble rhs. self.update_long_step_rhs(rhs, pFeas, dFeas, comp, s) # Solve augmented system. (step, nres, neig) = self.solveSystem(rhs) # Recover step. dx, ds, dy, dz = self.get_dxsyz(step, x, s, y, z, comp) normds = norm2(ds) normdy = norm2(dy) normdx = norm2(dx) # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, id) = self.maxStepLength(z, dz) # Compute fraction-to-the-boundary factor. tau = max(.9995, 1.0 - mu) if PredictorCorrector: # Compute actual stepsize using Mehrotra's heuristic. mult = 0.1 # ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0 if (ip != -1 or id != -1) and ip != id: mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns if ip != -1 and ip != id: zip = z[ip] + alpha_d * dz[ip] gamma_p = (mult * mu_tmp - s[ip] * zip) / (alpha_p * ds[ip] * zip) alpha_p *= max(1 - mult, gamma_p) if id != -1 and ip != id: sid = s[id] + alpha_p * ds[id] gamma_d = (mult * mu_tmp - z[id] * sid) / (alpha_d * dz[id] * sid) alpha_d *= max(1 - mult, gamma_d) if ip == id and ip != -1: # There is a division by zero in Mehrotra's heuristic # Fall back on classical rule. alpha_p *= tau alpha_d *= tau else: alpha_p *= tau alpha_d *= tau # Display data. output_line += self.format2 % (mu, alpha_p, alpha_d, nres, regpr, regdu, rho_q, del_r, mins, minz, maxs) self.log.info(output_line) # Update iterates and perturbation vectors. x += alpha_p * dx # This also updates slack variables. y += alpha_d * dy z += alpha_d * dz q *= (1 - alpha_p) q += alpha_p * dx r *= (1 - alpha_d) r += alpha_d * dy qNorm = norm2(q) rNorm = norm2(r) if regpr > 0: rho_q = regpr * qNorm / (1 + self.normc) rho_q_min = min(rho_q_min, rho_q) else: rho_q = 0.0 if regdu > 0: del_r = regdu * rNorm / (1 + self.normb) del_r_min = min(del_r_min, del_r) else: del_r = 0.0 iter += 1 solve_time = cputime() - setup_time self.log.info('-' * len(self.header)) # Transfer final values to class members. self.x = x self.y = y self.z = z self.iter = iter self.pResid = pResid self.cResid = cResid self.dResid = dResid self.rgap = rgap self.kktResid = kktResid self.solve_time = solve_time self.status = status self.short_status = short_status # Unscale problem if applicable. if self.prob_scaled: self.unscale() # Recompute final objective value. self.obj_value = self.c0 + cx + 0.5 * xQx return
def solveSystem(self, rhs, itref_threshold=1.0e-5, nitrefmax=3): self.LBL.solve(rhs) #nr = norm2(self.LBL.residual) self.LBL.refine(rhs, tol=itref_threshold, nitref=nitrefmax) nr = norm2(self.LBL.residual) return (self.LBL.x, nr, self.LBL.neig)
def solve(self, **kwargs): """ Solve the input problem with the primal-dual-regularized interior-point method. Accepted input keyword arguments are :keywords: :itermax: The maximum allowed number of iterations (default: 10n) :tolerance: Stopping tolerance (default: 1.0e-6) :PredictorCorrector: Use the predictor-corrector method (default: `True`). If set to `False`, a variant of the long-step method is used. The long-step method is generally slower and less robust. Upon exit, the following members of the class instance are set: * x..............final iterate * y..............final value of the Lagrange multipliers associated to A1 x + A2 s = b * z..............final value of the Lagrange multipliers associated to s>=0 * obj_value......final cost * iter...........total number of iterations * kktResid.......final relative residual * solve_time.....time to solve the QP * status.........string describing the exit status. * short_status...short version of status, used for printing. """ qp = self.qp itermax = kwargs.get('itermax', max(100, 10 * qp.n)) tolerance = kwargs.get('tolerance', 1.0e-6) PredictorCorrector = kwargs.get('PredictorCorrector', True) check_infeasible = kwargs.get('check_infeasible', True) # Transfer pointers for convenience. m, n = self.A.shape on = qp.original_n A = self.A b = self.b c = self.c Q = self.Q diagQ = self.diagQ H = self.H regpr = self.regpr regdu = self.regdu regpr_min = self.regpr_min regdu_min = self.regdu_min # Obtain initial point from Mehrotra's heuristic. (x, y, z) = self.set_initial_guess(self.qp, **kwargs) # Slack variables are the trailing variables in x. s = x[on:] ns = self.nSlacks # Initialize steps in dual variables. dz = np.zeros(ns) col_scale = np.empty(n) # Allocate room for right-hand side of linear systems. rhs = np.zeros(n + m) finished = False iter = 0 setup_time = cputime() # Main loop. while not finished: # Display initial header every so often. if self.verbose and iter % 20 == 0: sys.stdout.write('\n' + self.header + '\n') sys.stdout.write('-' * len(self.header) + '\n') # Compute residuals. pFeas = A * x - b comp = s * z sz = sum(comp) # comp = S z Qx = Q * x[:on] dFeas = y * A dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx dFeas[on:] += z # dFeas2 = A2'y + z # Compute duality measure. if ns > 0: mu = sz / ns else: mu = 0.0 # Compute residual norms and scaled residual norms. #pResid = norm_infty(pFeas + regdu * r)/(1+self.normc+self.normQ) #dResid = norm_infty(dFeas - regpr * q)/(1+self.normb+self.normQ) pResid = norm2(pFeas) spResid = pResid / (1 + self.normc + self.normQ) dResid = norm2(dFeas) sdResid = dResid / (1 + self.normb + self.normQ) if ns > 0: cResid = norm_infty(comp) / (self.normbc + self.normQ) else: cResid = 0.0 # Compute relative duality gap. cx = np.dot(c, x[:on]) xQx = np.dot(x[:on], Qx) by = np.dot(b, y) rgap = cx + xQx - by #rgap += regdu * (rNorm**2 + np.dot(r,y)) rgap = abs(rgap) / (1 + abs(cx) + self.normQ) rgap2 = mu / (1 + abs(cx) + self.normQ) # Compute overall residual for stopping condition. kktResid = max(spResid, sdResid, rgap2) #kktResid = max(pResid, cResid, dResid) # At the first iteration, initialize perturbation vectors # (q=primal, r=dual). # Should probably get rid of q when regpr=0 and of r when regdu=0. if iter == 0: if regpr > 0: q = dFeas / regpr qNorm = dResid / regpr rho_q = dResid else: q = dFeas qNorm = dResid rho_q = 0.0 rho_q_min = rho_q if regdu > 0: r = -pFeas / regdu rNorm = pResid / regdu del_r = pResid else: r = -pFeas rNorm = pResid del_r = 0.0 del_r_min = del_r pr_infeas_count = 0 # Used to detect primal infeasibility. du_infeas_count = 0 # Used to detect dual infeasibility. pr_last_iter = 0 du_last_iter = 0 mu0 = mu else: regdu = min(regdu / 10, sz / normdy / 10, (sz / normdy)**(1.1)) regdu = max(regdu, regdu_min) regpr = min(regpr / 10, sz / normdx / 10, (sz / normdx)**(1.1)) regpr = max(regpr, regpr_min) # Check for infeasible problem. if check_infeasible: if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min: pr_infeas_count += 1 if pr_infeas_count > 1 and pr_last_iter == iter - 1: if pr_infeas_count > 6: status = 'Problem seems to be (locally) dual infeasible' short_status = 'dInf' finished = True continue pr_last_iter = iter if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min: du_infeas_count += 1 if du_infeas_count > 1 and du_last_iter == iter - 1: if du_infeas_count > 6: status = 'Problem seems to be (locally) primal infeasible' short_status = 'pInf' finished = True continue du_last_iter = iter # Display objective and residual data. if self.verbose: sys.stdout.write(self.format1 % (iter, cx + 0.5 * xQx, pResid, dResid, cResid, rgap, qNorm, rNorm)) if kktResid <= tolerance: status = 'Optimal solution found' short_status = 'opt' finished = True continue if iter >= itermax: status = 'Maximum number of iterations reached' short_status = 'iter' finished = True continue # Record some quantities for display if ns > 0: mins = np.min(s) minz = np.min(z) maxs = np.max(s) else: mins = minz = maxs = 0 # Solve the linear system # # [ -(Q+pI) 0 A1' ] [∆x] = [c + Q x - A1' y ] # [ 0 -(S^{-1} Z + pI) A2' ] [∆s] [ - A2' y - µ S^{-1} e] # [ A1 A2 dI ] [∆y] [ b - A1 x - A2 s ] # # where s are the slack variables, p is the primal regularization # parameter, d is the dual regularization parameter, and # A = [ A1 A2 ] where the columns of A1 correspond to the original # problem variables and those of A2 correspond to slack variables. # # We recover ∆z = -z - S^{-1} (Z ∆s + µ e). # Compute augmented matrix and factorize it. factorized = False nb_bump = 0 while not factorized and nb_bump < 5: H.put(-diagQ - regpr, range(on)) H.put(-z / s - regpr, range(on, n)) H.put(regdu, range(n, n + m)) self.LBL.factorize(H) factorized = True # If the augmented matrix does not have full rank, bump up the # regularization parameters. if not self.LBL.isFullRank: if self.verbose: sys.stderr.write('Primal-Dual Matrix Rank Deficient') sys.stderr.write('... bumping up reg parameters\n') regpr *= 100 regdu *= 100 nb_bump += 1 factorized = False # Abandon if regularization is unsuccessful. if not self.LBL.isFullRank and nb_bump == 5: status = 'Unable to regularize sufficiently.' short_status = 'degn' finished = True continue if PredictorCorrector: # Use Mehrotra predictor-corrector method. # Compute affine-scaling step, i.e. with centering = 0. rhs[:n] = -dFeas rhs[on:n] += z rhs[n:] = -pFeas (step, nres, neig) = self.solveSystem(rhs) # Recover dx and dz. dx = step[:n] ds = dx[on:] dz = -z * (1 + ds / s) # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, ip) = self.maxStepLength(z, dz) # Estimate duality gap after affine-scaling step. muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns sigma = (muAff / mu)**3 # Incorporate predictor information for corrector step. comp += ds * dz else: # Use long-step method: Compute centering parameter. sigma = min(0.1, 100 * mu) # Assemble right-hand side with centering information. comp -= sigma * mu if PredictorCorrector: # Only update rhs[on:n]; the rest of the vector did not change. rhs[on:n] += comp / s - z else: rhs[:n] = -dFeas rhs[on:n] += comp / s rhs[n:] = -pFeas # Solve augmented system. (step, nres, neig) = self.solveSystem(rhs) # Recover step. dx = step[:n] ds = dx[on:] dy = step[n:] dz = -(comp + z * ds) / s normds = norm2(ds) normdy = norm2(dy) normdx = norm2(dx) # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, id) = self.maxStepLength(z, dz) # Compute fraction-to-the-boundary factor. tau = max(.9995, 1.0 - mu) if PredictorCorrector: # Compute actual stepsize using Mehrotra's heuristic mult = 0.1 # ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0 if (ip != -1 or id != -1) and ip != id: mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz) / ns if ip != -1 and ip != id: zip = z[ip] + alpha_d * dz[ip] gamma_p = (mult * mu_tmp - s[ip] * zip) / (alpha_p * ds[ip] * zip) alpha_p *= max(1 - mult, gamma_p) if id != -1 and ip != id: sid = s[id] + alpha_p * ds[id] gamma_d = (mult * mu_tmp - z[id] * sid) / (alpha_d * dz[id] * sid) alpha_d *= max(1 - mult, gamma_d) if ip == id and ip != -1: # There is a division by zero in Mehrotra's heuristic # Fall back on classical rule. alpha_p *= tau alpha_d *= tau else: alpha_p *= tau alpha_d *= tau # Display data. if self.verbose: sys.stdout.write(self.format2 % (mu, alpha_p, alpha_d, nres, regpr, regdu, rho_q, del_r, mins, minz, maxs)) # Update iterates and perturbation vectors. x += alpha_p * dx # This also updates slack variables. y += alpha_d * dy z += alpha_d * dz q *= (1 - alpha_p) q += alpha_p * dx r *= (1 - alpha_d) r += alpha_d * dy qNorm = norm2(q) rNorm = norm2(r) rho_q = regpr * qNorm / (1 + self.normc) rho_q_min = min(rho_q_min, rho_q) del_r = regdu * rNorm / (1 + self.normb) del_r_min = min(del_r_min, del_r) iter += 1 solve_time = cputime() - setup_time if self.verbose: sys.stdout.write('\n') sys.stdout.write('-' * len(self.header) + '\n') # Transfer final values to class members. self.x = x self.y = y self.z = z self.iter = iter self.pResid = pResid self.cResid = cResid self.dResid = dResid self.rgap = rgap self.kktResid = kktResid self.solve_time = solve_time self.status = status self.short_status = short_status # Unscale problem if applicable. if self.prob_scaled: self.unscale() # Recompute final objective value. self.obj_value = self.c0 + cx + 0.5 * xQx return
def SolveInner(self, **kwargs): """ Perform a series of inner iterations so as to minimize the primal-dual merit function with the current value of the barrier parameter to within some given tolerance. The only optional argument recognized is stopTol stopping tolerance (default: muerrfact * mu). """ merit = self.merit ; nx = merit.nlp.n ; nz = merit.nz rho = 1 # Dummy initial value for rho niter = 0 # Dummy initial value for number of inners status = '' # Dummy initial step status alpha = 0.0 # Fraction-to-the-boundary step size if self.inexact: cgtol = 1.0 else: cgtol = -1.0 inner_iter = 0 # Inner iteration counter # Obtain starting point. (x,z) = (self.x, self.z) # Obtain first-order data at starting point. if self.iter == 0: f = nlp.obj(x) ; gf = nlp.grad(x) else: f = self.f ; gf = self.gf psi = merit.obj(x, z, f=f) g = merit.grad(x, z, g=gf, check_optimal=True) gNorm = norm2(g) if self.optimal: return # Reset initial trust-region radius self.TR.Delta = 0.1*gNorm # Set inner iteration stopping tolerance stopTol = kwargs.get('stopTol', self.muerrfact * self.mu) finished = (gNorm <= stopTol) or (self.iter >= self.maxiter) while not finished: # Print out header every so many iterations if self.verbose: if self.iter % self.printFrequency == 0: sys.stdout.write(self.hline) sys.stdout.write(self.header) sys.stdout.write(self.hline) if inner_iter == 0: sys.stdout.write(('*' + self.itFormat) % self.iter) else: sys.stdout.write((' ' + self.itFormat) % self.iter) sys.stdout.write(self.format % (f, max(norm_infty(self.dRes), norm_infty(self.cRes), norm_infty(self.pRes)), gNorm, self.mu, alpha, niter, rho, self.TR.Delta, status)) # Set stopping tolerance for trust-region subproblem. if self.inexact: cgtol = max(1.0e-8, min(0.1 * cgtol, sqrt(gNorm))) if self.debug: self._debugMsg('cgtol = ' + str(cgtol)) # Update Hessian matrix with current iteration information. # self.PDHess(x,z) if self.debug: self._debugMsg('g = ' + np.str(g)) self._debugMsg('gNorm = ' + str(gNorm)) self._debugMsg('stopTol = ' + str(stopTol)) self._debugMsg('dRes = ' + np.str(self.dRes)) self._debugMsg('cRes = ' + np.str(self.cRes)) self._debugMsg('pRes = ' + np.str(self.pRes)) self._debugMsg('optimal = ' + str(self.optimal)) # Set up the preconditioner if applicable. self.SetupPrecon() # Iteratively minimize the quadratic model in the trust region # m(s) = <g, s> + 1/2 <s, Hs> # Note that m(s) does not include f(x): m(0) = 0. H = SimpleLinearOperator(nx+nz, nx+nz, lambda v: merit.hprod(v), symmetric=True) subsolver = self.TrSolver(g, H, prec = self.Precon, radius = self.TR.Delta, reltol = cgtol, #fraction = 0.5, itmax = 2*(n+nz), #debug=True, #btol=.9, #cur_iter=np.concatenate((x,z)) ) subsolver.Solve() if self.debug: self._debugMsg('x = ' + np.str(x)) self._debugMsg('z = ' + np.str(z)) self._debugMsg('step = ' + np.str(solver.step)) self._debugMsg('step norm = ' + str(solver.stepNorm)) # Record total number of CG iterations. niter = subsolver.niter self.cgiter += subsolver.niter # Compute maximal step to the boundary and next candidate. alphax, alphaz = self.ftb(x, z, solver.step) alpha = min(alphax, alphaz) dx = subsolver.step[:n] ; dz = subsolver.step[n:] x_trial = x + alphax * dx z_trial = z + alphaz * dz f_trial = merit.nlp.obj(x_trial) psi_trial = merit.obj(x_trial, z_trial, f=f_trial) # Compute ratio of predicted versus achieved reduction. rho = self.TR.Rho(psi, psi_trial, subsolver.m) if self.debug: self._debugMsg('m = ' + str(solver.m)) self._debugMsg('x_trial = ' + np.str(x_trial)) self._debugMsg('z_trial = ' + np.str(z_trial)) self._debugMsg('psi_trial = ' + str(psi_trial)) self._debugMsg('rho = ' + str(rho)) # Accept or reject next candidate status = 'Rej' if rho >= self.TR.eta1: self.TR.UpdateRadius(rho, solver.stepNorm) x = x_trial z = z_trial f = f_trial psi = psi_trial gf = nlp.grad(x) g = self.GradPDMerit(x, z, g=gf, check_optimal=True) gNorm = norm2(g) if self.optimal: finished = True continue status = 'Acc' else: if self.ny: # Backtracking linesearch a la "Nocedal & Yuan" slope = np.dot(g, solver.step) target = psi + 1.0e-4 * alpha * slope j = 0 while (psi_trial >= target) and (j < self.nyMax): alphax /= 1.2 ; alphaz /= 1.2 alpha = min(alphax, alphaz) target = psi + 1.0e-4 * alpha * slope x_trial = x + alphax * dx z_trial = z + alphaz * dz f_trial = nlp.obj(x_trial) psi_trial = self.PDMerit(x_trial, z_trial, f=f_trial) j += 1 if self.opportunistic or (j < self.nyMax): x = x_trial z = z_trial #self.PrimalMultipliers(x) f = f_trial psi = psi_trial gf = nlp.grad(x) g = self.GradPDMerit(x, z, g=gf, check_optimal=True) gNorm = norm2(g) if self.optimal: finished = True continue self.TR.Delta = alpha * solver.stepNorm status = 'N-Y' else: self.TR.UpdateRadius(rho, solver.stepNorm) else: self.TR.UpdateRadius(rho, solver.stepNorm) self.UpdatePrecon() self.iter += 1 inner_iter += 1 finished = (gNorm <= stopTol) or (self.iter >= self.maxiter) if self.debug: sys.stderr.write('\n') # Store final iterate (self.x, self.z) = (x, z) self.f = f self.gf = gf self.g = g self.gNorm = gNorm self.psi = psi return
# Display summary line. probname=os.path.basename(probname) if probname[-3:] == '.nl': probname = probname[:-3] if not options.verbose: sys.stdout.write(fmt % (probname, reglp.iter, reglp.obj_value, reglp.pResid, reglp.dResid, reglp.rgap, t_setup, reglp.solve_time, reglp.short_status)) if reglp.short_status == 'degn': sys.stdout.write(' F') # Could not regularize sufficiently. sys.stdout.write('\n') lp.close() if islp: if not options.verbose: sys.stderr.write('-'*len(hdr) + '\n') else: x = reglp.x[:lp.original_n] print 'Final x: ', x, ', |x| = %7.1e' % norm2(x) print 'Final y: ', reglp.y, ', |y| = %7.1e' % norm2(reglp.y) print 'Final z: ', reglp.z, ', |z| = %7.1e' % norm2(reglp.z) sys.stdout.write('\n' + reglp.status + '\n') sys.stdout.write(' #Iterations: %-d\n' % reglp.iter) sys.stdout.write(' RelResidual: %7.1e\n' % reglp.kktResid) sys.stdout.write(' Final cost : %21.15e\n' % reglp.obj_value) sys.stdout.write(' Setup time : %6.2fs\n' % t_setup) sys.stdout.write(' Solve time : %6.2fs\n' % reglp.solve_time)
def Solve(self, **kwargs): nlp = self.nlp # Gather initial information. self.f = self.nlp.obj(self.x) self.f0 = self.f self.g = self.nlp.grad(self.x) # Current gradient self.g_old = self.g # Previous gradient self.gnorm = norms.norm2(self.g) self.g0 = self.gnorm # Reset initial trust-region radius. # self.TR.Delta = 0.1 * self.g0 if self.inexact: cgtol = 1.0 else: cgtol = -1.0 stoptol = max(self.abstol, self.reltol * self.g0) step_status = None exitOptimal = exitIter = exitUser = False # Initialize non-monotonicity parameters. if not self.monotone: fMin = fRef = fCan = self.f0 l = 0 sigRef = sigCan = 0 t = cputime() # Print out header and initial log. if self.iter % 20 == 0 and self.verbose: self.log.info(self.hline) self.log.info(self.header) self.log.info(self.hline) self.log.info( self.format0 % (self.iter, self.f, self.gnorm, '', '', self.TR.Delta, '')) while not (exitUser or exitOptimal or exitIter): self.iter += 1 self.alpha = 1.0 # Save current gradient if self.save_g: self.g_old = self.g.copy() # Iteratively minimize the quadratic model in the trust region # m(s) = <g, s> + 1/2 <s, Hs> # Note that m(s) does not include f(x): m(0) = 0. if self.inexact: cgtol = max(1.0e-6, min(0.5 * cgtol, sqrt(self.gnorm))) H = SimpleLinearOperator(nlp.n, nlp.n, lambda v: self.hprod(v), symmetric=True) self.solver = self.TrSolver(self.g, H) self.solver.Solve( prec=self.precon, radius=self.TR.Delta, reltol=cgtol, #debug=True ) step = self.solver.step snorm = self.solver.stepNorm cgiter = self.solver.niter # Obtain model value at next candidate m = self.solver.m if m is None: m = numpy.dot(self.g, step) + 0.5 * numpy.dot(step, H * step) self.total_cgiter += cgiter x_trial = self.x + step f_trial = nlp.obj(x_trial) rho = self.TR.Rho(self.f, f_trial, m) if not self.monotone: rhoHis = (fRef - f_trial) / (sigRef - m) rho = max(rho, rhoHis) step_status = 'Rej' if rho >= self.TR.eta1: # Trust-region step is accepted. self.TR.UpdateRadius(rho, snorm) self.x = x_trial self.f = f_trial self.g = nlp.grad(self.x) self.gnorm = norms.norm2(self.g) step_status = 'Acc' # Update non-monotonicity parameters. if not self.monotone: sigRef = sigRef - m sigCan = sigCan - m if f_trial < fMin: fCan = f_trial fMin = f_trial sigCan = 0 l = 0 else: l = l + 1 if f_trial > fCan: fCan = f_trial sigCan = 0 if l == self.nIterNonMono: fRef = fCan sigRef = sigCan else: # Trust-region step is rejected. if self.ny: # Backtracking linesearch following "Nocedal & Yuan" slope = numpy.dot(self.g, step) bk = 0 while bk < self.nbk and \ f_trial >= self.f + 1.0e-4 * self.alpha * slope: bk = bk + 1 self.alpha /= 1.2 x_trial = self.x + self.alpha * step f_trial = nlp.obj(x_trial) self.x = x_trial self.f = f_trial self.g = nlp.grad(self.x) self.gnorm = norms.norm2(self.g) self.TR.Delta = self.alpha * snorm step_status = 'N-Y' else: self.TR.UpdateRadius(rho, snorm) self.step_status = step_status self.radii.append(self.TR.Delta) status = '' try: self.PostIteration() except UserExitRequest: status = 'usr' # Print out header, say, every 20 iterations if self.iter % 20 == 0 and self.verbose: self.log.info(self.hline) self.log.info(self.header) self.log.info(self.hline) if self.verbose: pstatus = step_status if step_status != 'Acc' else '' self.log.info(self.format % (self.iter, self.f, self.gnorm, cgiter, rho, self.TR.Delta, pstatus)) exitOptimal = self.gnorm < stoptol exitIter = self.iter > self.maxiter exitUser = status == 'usr' self.tsolve = cputime() - t # Solve time # Set final solver status. if status == 'usr': pass elif self.gnorm <= stoptol: status = 'opt' else: # self.iter > self.maxiter: status = 'itr' self.status = status
def solve(self, **kwargs): """ Solve the input problem with the primal-dual-regularized interior-point method. Accepted input keyword arguments are :keywords: :itermax: The maximum allowed number of iterations (default: 10n) :tolerance: Stopping tolerance (default: 1.0e-6) :PredictorCorrector: Use the predictor-corrector method (default: `True`). If set to `False`, a variant of the long-step method is used. The long-step method is generally slower and less robust. :returns: :x: final iterate :y: final value of the Lagrange multipliers associated to `A1 x + A2 s = b` :z: final value of the Lagrange multipliers associated to `s >= 0` :obj_value: final cost :iter: total number of iterations :kktResid: final relative residual :solve_time: time to solve the QP :status: string describing the exit status. :short_status: short version of status, used for printing. """ qp = self.qp itermax = kwargs.get('itermax', max(100,10*qp.n)) tolerance = kwargs.get('tolerance', 1.0e-6) PredictorCorrector = kwargs.get('PredictorCorrector', True) check_infeasible = kwargs.get('check_infeasible', True) # Transfer pointers for convenience. m, n = self.A.shape ; on = qp.original_n A = self.A ; b = self.b ; c = self.c ; Q = self.Q ; diagQ = self.diagQ H = self.H regpr = self.regpr ; regdu = self.regdu regpr_min = self.regpr_min ; regdu_min = self.regdu_min # Obtain initial point from Mehrotra's heuristic. (x,y,z) = self.set_initial_guess(**kwargs) # Slack variables are the trailing variables in x. s = x[on:] ; ns = self.nSlacks # Initialize steps in dual variables. dz = np.zeros(ns) # Allocate room for right-hand side of linear systems. rhs = self.initialize_rhs() finished = False iter = 0 setup_time = cputime() # Main loop. while not finished: # Display initial header every so often. if iter % 50 == 0: self.log.info(self.header) self.log.info('-' * len(self.header)) # Compute residuals. pFeas = A*x - b comp = s*z ; sz = sum(comp) # comp = Sz Qx = Q*x[:on] dFeas = y*A ; dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx dFeas[on:] += z # dFeas2 = A2'y + z # Compute duality measure. if ns > 0: mu = sz/ns else: mu = 0.0 # Compute residual norms and scaled residual norms. pResid = norm2(pFeas) spResid = pResid/(1+self.normb+self.normA+self.normQ) dResid = norm2(dFeas) sdResid = dResid/(1+self.normc+self.normA+self.normQ) if ns > 0: cResid = norm_infty(comp)/(self.normbc+self.normA+self.normQ) else: cResid = 0.0 # Compute relative duality gap. cx = np.dot(c,x[:on]) xQx = np.dot(x[:on],Qx) by = np.dot(b,y) rgap = cx + xQx - by rgap = abs(rgap) / (1 + abs(cx) + self.normA + self.normQ) rgap2 = mu / (1 + abs(cx) + self.normA + self.normQ) # Compute overall residual for stopping condition. kktResid = max(spResid, sdResid, rgap2) # At the first iteration, initialize perturbation vectors # (q=primal, r=dual). # Should probably get rid of q when regpr=0 and of r when regdu=0. if iter == 0: if regpr > 0: q = dFeas/regpr ; qNorm = dResid/regpr ; rho_q = dResid else: q = dFeas ; qNorm = dResid ; rho_q = 0.0 rho_q_min = rho_q if regdu > 0: r = -pFeas/regdu ; rNorm = pResid/regdu ; del_r = pResid else: r = -pFeas ; rNorm = pResid ; del_r = 0.0 del_r_min = del_r pr_infeas_count = 0 # Used to detect primal infeasibility. du_infeas_count = 0 # Used to detect dual infeasibility. pr_last_iter = 0 du_last_iter = 0 mu0 = mu else: if regdu > 0: regdu = regdu/10 regdu = max(regdu, regdu_min) if regpr > 0: regpr = regpr/10 regpr = max(regpr, regpr_min) # Check for infeasible problem. if check_infeasible: if mu < tolerance/100 * mu0 and \ rho_q > 1./tolerance/1.0e+6 * rho_q_min: pr_infeas_count += 1 if pr_infeas_count > 1 and pr_last_iter == iter-1: if pr_infeas_count > 6: status = 'Problem seems to be (locally) dual' status += ' infeasible' short_status = 'dInf' finished = True continue pr_last_iter = iter else: pr_infeas_count = 0 if mu < tolerance/100 * mu0 and \ del_r > 1./tolerance/1.0e+6 * del_r_min: du_infeas_count += 1 if du_infeas_count > 1 and du_last_iter == iter-1: if du_infeas_count > 6: status = 'Problem seems to be (locally) primal' status += ' infeasible' short_status = 'pInf' finished = True continue du_last_iter = iter else: du_infeas_count = 0 # Display objective and residual data. output_line = self.format1 % (iter, cx + 0.5 * xQx, pResid, dResid, cResid, rgap, qNorm, rNorm) if kktResid <= tolerance: status = 'Optimal solution found' short_status = 'opt' finished = True continue if iter >= itermax: status = 'Maximum number of iterations reached' short_status = 'iter' finished = True continue # Record some quantities for display if ns > 0: mins = np.min(s) minz = np.min(z) maxs = np.max(s) else: mins = minz = maxs = 0 # Compute augmented matrix and factorize it. factorized = False degenerate = False nb_bump = 0 while not factorized and not degenerate: self.update_linear_system(s, z, regpr, regdu) self.log.debug('Factorizing') self.LBL.factorize(H) factorized = True # If the augmented matrix does not have full rank, bump up the # regularization parameters. if not self.LBL.isFullRank: if self.verbose: self.log.info('Primal-Dual Matrix Rank Deficient' + \ '... bumping up reg parameters') if regpr == 0. and regdu == 0.: degenerate = True else: if regpr > 0: regpr *= 100 if regdu > 0: regdu *= 100 nb_bump += 1 degenerate = nb_bump > self.bump_max factorized = False # Abandon if regularization is unsuccessful. if not self.LBL.isFullRank and degenerate: status = 'Unable to regularize sufficiently.' short_status = 'degn' finished = True continue if PredictorCorrector: # Use Mehrotra predictor-corrector method. # Compute affine-scaling step, i.e. with centering = 0. self.set_affine_scaling_rhs(rhs, pFeas, dFeas, s, z) (step, nres, neig) = self.solveSystem(rhs) # Recover dx and dz. dx, ds, dy, dz = self.get_affine_scaling_dxsyz(step, x, s, y, z) # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, ip) = self.maxStepLength(z, dz) # Estimate duality gap after affine-scaling step. muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns sigma = (muAff/mu)**3 # Incorporate predictor information for corrector step. # Only update rhs[on:n]; the rest of the vector did not change. comp += ds*dz comp -= sigma * mu self.update_corrector_rhs(rhs, s, z, comp) else: # Use long-step method: Compute centering parameter. sigma = min(0.1, 100*mu) comp -= sigma * mu # Assemble rhs. self.update_long_step_rhs(rhs, pFeas, dFeas, comp, s) # Solve augmented system. (step, nres, neig) = self.solveSystem(rhs) # Recover step. dx, ds, dy, dz = self.get_dxsyz(step, x, s, y, z, comp) normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx) # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, id) = self.maxStepLength(z, dz) # Compute fraction-to-the-boundary factor. tau = max(.9995, 1.0-mu) if PredictorCorrector: # Compute actual stepsize using Mehrotra's heuristic. mult = 0.1 # ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0 if (ip != -1 or id != -1) and ip != id: mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns if ip != -1 and ip != id: zip = z[ip] + alpha_d * dz[ip] gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip) alpha_p *= max(1-mult, gamma_p) if id != -1 and ip != id: sid = s[id] + alpha_p * ds[id] gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid) alpha_d *= max(1-mult, gamma_d) if ip==id and ip != -1: # There is a division by zero in Mehrotra's heuristic # Fall back on classical rule. alpha_p *= tau alpha_d *= tau else: alpha_p *= tau alpha_d *= tau # Display data. output_line += self.format2 % (mu, alpha_p, alpha_d, nres, regpr, regdu, rho_q, del_r, mins, minz, maxs) self.log.info(output_line) # Update iterates and perturbation vectors. x += alpha_p * dx # This also updates slack variables. y += alpha_d * dy z += alpha_d * dz q *= (1-alpha_p) ; q += alpha_p * dx r *= (1-alpha_d) ; r += alpha_d * dy qNorm = norm2(q) ; rNorm = norm2(r) if regpr > 0: rho_q = regpr * qNorm/(1+self.normc) rho_q_min = min(rho_q_min, rho_q) else: rho_q = 0.0 if regdu > 0: del_r = regdu * rNorm/(1+self.normb) del_r_min = min(del_r_min, del_r) else: del_r = 0.0 iter += 1 solve_time = cputime() - setup_time self.log.info('-' * len(self.header)) # Transfer final values to class members. self.x = x self.y = y self.z = z self.iter = iter self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid self.rgap = rgap self.kktResid = kktResid self.solve_time = solve_time self.status = status self.short_status = short_status # Unscale problem if applicable. if self.prob_scaled: self.unscale() # Recompute final objective value. self.obj_value = self.c0 + cx + 0.5 * xQx return
def solve(self, **kwargs): """ Solve the input problem with the primal-dual-regularized interior-point method. Accepted input keyword arguments are :keywords: :itermax: The maximum allowed number of iterations (default: 10n) :tolerance: Stopping tolerance (default: 1.0e-6) :PredictorCorrector: Use the predictor-corrector method (default: `True`). If set to `False`, a variant of the long-step method is used. The long-step method is generally slower and less robust. Upon exit, the following members of the class instance are set: * x..............final iterate * y..............final value of the Lagrange multipliers associated to A1 x + A2 s = b * z..............final value of the Lagrange multipliers associated to s>=0 * obj_value......final cost * iter...........total number of iterations * kktResid.......final relative residual * solve_time.....time to solve the QP * status.........string describing the exit status. * short_status...short version of status, used for printing. """ qp = self.qp itermax = kwargs.get('itermax', max(100,10*qp.n)) tolerance = kwargs.get('tolerance', 1.0e-6) PredictorCorrector = kwargs.get('PredictorCorrector', True) check_infeasible = kwargs.get('check_infeasible', True) # Transfer pointers for convenience. m, n = self.A.shape ; on = qp.original_n A = self.A ; b = self.b ; c = self.c ; Q = self.Q ; diagQ = self.diagQ H = self.H regpr = self.regpr ; regdu = self.regdu regpr_min = self.regpr_min ; regdu_min = self.regdu_min # Obtain initial point from Mehrotra's heuristic. (x,y,z) = self.set_initial_guess(self.qp, **kwargs) # Slack variables are the trailing variables in x. s = x[on:] ; ns = self.nSlacks # Initialize steps in dual variables. dz = np.zeros(ns) col_scale = np.empty(n) # Allocate room for right-hand side of linear systems. rhs = np.zeros(n+m) finished = False iter = 0 setup_time = cputime() # Main loop. while not finished: # Display initial header every so often. if self.verbose and iter % 20 == 0: sys.stdout.write('\n' + self.header + '\n') sys.stdout.write('-' * len(self.header) + '\n') # Compute residuals. pFeas = A*x - b comp = s*z ; sz = sum(comp) # comp = S z Qx = Q*x[:on] dFeas = y*A ; dFeas[:on] -= self.c + Qx # dFeas1 = A1'y - c - Qx dFeas[on:] += z # dFeas2 = A2'y + z # Compute duality measure. if ns > 0: mu = sz/ns else: mu = 0.0 # Compute residual norms and scaled residual norms. #pResid = norm_infty(pFeas + regdu * r)/(1+self.normc+self.normQ) #dResid = norm_infty(dFeas - regpr * q)/(1+self.normb+self.normQ) pResid = norm2(pFeas) ; spResid = pResid/(1+self.normc+self.normQ) dResid = norm2(dFeas) ; sdResid = dResid/(1+self.normb+self.normQ) if ns > 0: cResid = norm_infty(comp)/(self.normbc+self.normQ) else: cResid = 0.0 # Compute relative duality gap. cx = np.dot(c,x[:on]) xQx = np.dot(x[:on],Qx) by = np.dot(b,y) rgap = cx + xQx - by #rgap += regdu * (rNorm**2 + np.dot(r,y)) rgap = abs(rgap) / (1 + abs(cx) + self.normQ) rgap2 = mu / (1 + abs(cx) + self.normQ) # Compute overall residual for stopping condition. kktResid = max(spResid, sdResid, rgap2) #kktResid = max(pResid, cResid, dResid) # At the first iteration, initialize perturbation vectors # (q=primal, r=dual). # Should probably get rid of q when regpr=0 and of r when regdu=0. if iter == 0: if regpr > 0: q = dFeas/regpr ; qNorm = dResid/regpr ; rho_q = dResid else: q = dFeas ; qNorm = dResid ; rho_q = 0.0 rho_q_min = rho_q if regdu > 0: r = -pFeas/regdu ; rNorm = pResid/regdu ; del_r = pResid else: r = -pFeas ; rNorm = pResid ; del_r = 0.0 del_r_min = del_r pr_infeas_count = 0 # Used to detect primal infeasibility. du_infeas_count = 0 # Used to detect dual infeasibility. pr_last_iter = 0 du_last_iter = 0 mu0 = mu else: regdu = min(regdu/10, sz/normdy/10, (sz/normdy)**(1.1)) regdu = max(regdu, regdu_min) regpr = min(regpr/10, sz/normdx/10, (sz/normdx)**(1.1)) regpr = max(regpr, regpr_min) # Check for infeasible problem. if check_infeasible: if mu < 1.0e-8 * mu0 and rho_q > 1.0e+2 * rho_q_min: pr_infeas_count += 1 if pr_infeas_count > 1 and pr_last_iter == iter-1: if pr_infeas_count > 6: status = 'Problem seems to be (locally) dual infeasible' short_status = 'dInf' finished = True continue pr_last_iter = iter if mu < 1.0e-8 * mu0 and del_r > 1.0e+2 * del_r_min: du_infeas_count += 1 if du_infeas_count > 1 and du_last_iter == iter-1: if du_infeas_count > 6: status='Problem seems to be (locally) primal infeasible' short_status = 'pInf' finished = True continue du_last_iter = iter # Display objective and residual data. if self.verbose: sys.stdout.write(self.format1 % (iter, cx + 0.5 * xQx, pResid, dResid, cResid, rgap, qNorm, rNorm)) if kktResid <= tolerance: status = 'Optimal solution found' short_status = 'opt' finished = True continue if iter >= itermax: status = 'Maximum number of iterations reached' short_status = 'iter' finished = True continue # Record some quantities for display if ns > 0: mins = np.min(s) minz = np.min(z) maxs = np.max(s) else: mins = minz = maxs = 0 # Solve the linear system # # [ -(Q+pI) 0 A1' ] [∆x] = [c + Q x - A1' y ] # [ 0 -(S^{-1} Z + pI) A2' ] [∆s] [ - A2' y - µ S^{-1} e] # [ A1 A2 dI ] [∆y] [ b - A1 x - A2 s ] # # where s are the slack variables, p is the primal regularization # parameter, d is the dual regularization parameter, and # A = [ A1 A2 ] where the columns of A1 correspond to the original # problem variables and those of A2 correspond to slack variables. # # We recover ∆z = -z - S^{-1} (Z ∆s + µ e). # Compute augmented matrix and factorize it. factorized = False nb_bump = 0 while not factorized and nb_bump < 5: H.put(-diagQ - regpr, range(on)) H.put(-z/s - regpr, range(on,n)) H.put(regdu, range(n,n+m)) self.LBL.factorize(H) factorized = True # If the augmented matrix does not have full rank, bump up the # regularization parameters. if not self.LBL.isFullRank: if self.verbose: sys.stderr.write('Primal-Dual Matrix Rank Deficient') sys.stderr.write('... bumping up reg parameters\n') regpr *= 100 ; regdu *= 100 nb_bump += 1 factorized = False # Abandon if regularization is unsuccessful. if not self.LBL.isFullRank and nb_bump == 5: status = 'Unable to regularize sufficiently.' short_status = 'degn' finished = True continue if PredictorCorrector: # Use Mehrotra predictor-corrector method. # Compute affine-scaling step, i.e. with centering = 0. rhs[:n] = -dFeas rhs[on:n] += z rhs[n:] = -pFeas (step, nres, neig) = self.solveSystem(rhs) # Recover dx and dz. dx = step[:n] ds = dx[on:] dz = -z * (1 + ds/s) # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, ip) = self.maxStepLength(z, dz) # Estimate duality gap after affine-scaling step. muAff = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns sigma = (muAff/mu)**3 # Incorporate predictor information for corrector step. comp += ds*dz else: # Use long-step method: Compute centering parameter. sigma = min(0.1, 100*mu) # Assemble right-hand side with centering information. comp -= sigma * mu if PredictorCorrector: # Only update rhs[on:n]; the rest of the vector did not change. rhs[on:n] += comp/s - z else: rhs[:n] = -dFeas rhs[on:n] += comp/s rhs[n:] = -pFeas # Solve augmented system. (step, nres, neig) = self.solveSystem(rhs) # Recover step. dx = step[:n] ds = dx[on:] dy = step[n:] dz = -(comp + z*ds)/s normds = norm2(ds) ; normdy = norm2(dy) ; normdx = norm2(dx) # Compute largest allowed primal and dual stepsizes. (alpha_p, ip) = self.maxStepLength(s, ds) (alpha_d, id) = self.maxStepLength(z, dz) # Compute fraction-to-the-boundary factor. tau = max(.9995, 1.0-mu) if PredictorCorrector: # Compute actual stepsize using Mehrotra's heuristic mult = 0.1 # ip=-1 if ds ≥ 0, and id=-1 if dz ≥ 0 if (ip != -1 or id != -1) and ip != id: mu_tmp = np.dot(s + alpha_p * ds, z + alpha_d * dz)/ns if ip != -1 and ip != id: zip = z[ip] + alpha_d * dz[ip] gamma_p = (mult*mu_tmp - s[ip]*zip)/(alpha_p*ds[ip]*zip) alpha_p *= max(1-mult, gamma_p) if id != -1 and ip != id: sid = s[id] + alpha_p * ds[id] gamma_d = (mult*mu_tmp - z[id]*sid)/(alpha_d*dz[id]*sid) alpha_d *= max(1-mult, gamma_d) if ip==id and ip != -1: # There is a division by zero in Mehrotra's heuristic # Fall back on classical rule. alpha_p *= tau alpha_d *= tau else: alpha_p *= tau alpha_d *= tau # Display data. if self.verbose: sys.stdout.write(self.format2 % (mu, alpha_p, alpha_d, nres, regpr, regdu, rho_q, del_r, mins, minz, maxs)) # Update iterates and perturbation vectors. x += alpha_p * dx # This also updates slack variables. y += alpha_d * dy z += alpha_d * dz q *= (1-alpha_p) ; q += alpha_p * dx r *= (1-alpha_d) ; r += alpha_d * dy qNorm = norm2(q) ; rNorm = norm2(r) rho_q = regpr * qNorm/(1+self.normc) ; rho_q_min = min(rho_q_min, rho_q) del_r = regdu * rNorm/(1+self.normb) ; del_r_min = min(del_r_min, del_r) iter += 1 solve_time = cputime() - setup_time if self.verbose: sys.stdout.write('\n') sys.stdout.write('-' * len(self.header) + '\n') # Transfer final values to class members. self.x = x self.y = y self.z = z self.iter = iter self.pResid = pResid ; self.cResid = cResid ; self.dResid = dResid self.rgap = rgap self.kktResid = kktResid self.solve_time = solve_time self.status = status self.short_status = short_status # Unscale problem if applicable. if self.prob_scaled: self.unscale() # Recompute final objective value. self.obj_value = self.c0 + cx + 0.5 * xQx return
def Solve(self, **kwargs): nlp = self.nlp # Gather initial information. self.f = self.nlp.obj(self.x) self.f0 = self.f self.g = self.nlp.grad(self.x) # Current gradient self.g_old = self.g # Previous gradient self.gnorm = norms.norm2(self.g) self.g0 = self.gnorm # Reset initial trust-region radius. # self.TR.Delta = 0.1 * self.g0 if self.inexact: cgtol = 1.0 else: cgtol = -1.0 stoptol = max(self.abstol, self.reltol * self.g0) step_status = None exitOptimal = exitIter = exitUser = False # Initialize non-monotonicity parameters. if not self.monotone: fMin = fRef = fCan = self.f0 l = 0 sigRef = sigCan = 0 t = cputime() # Print out header and initial log. if self.iter % 20 == 0 and self.verbose: self.log.info(self.hline) self.log.info(self.header) self.log.info(self.hline) self.log.info(self.format0 % (self.iter, self.f, self.gnorm, '', '', self.TR.Delta, '')) while not (exitUser or exitOptimal or exitIter): self.iter += 1 self.alpha = 1.0 # Save current gradient if self.save_g: self.g_old = self.g.copy() # Iteratively minimize the quadratic model in the trust region # m(s) = <g, s> + 1/2 <s, Hs> # Note that m(s) does not include f(x): m(0) = 0. if self.inexact: cgtol = max(1.0e-6, min(0.5 * cgtol, sqrt(self.gnorm))) H = SimpleLinearOperator(nlp.n, nlp.n, lambda v: self.hprod(v), symmetric=True) self.solver = self.TrSolver(self.g, H) self.solver.Solve(prec=self.precon, radius=self.TR.Delta, reltol=cgtol, #debug=True ) step = self.solver.step snorm = self.solver.stepNorm cgiter = self.solver.niter # Obtain model value at next candidate m = self.solver.m if m is None: m = numpy.dot(self.g, step) + 0.5*numpy.dot(step, H * step) self.total_cgiter += cgiter x_trial = self.x + step f_trial = nlp.obj(x_trial) rho = self.TR.Rho(self.f, f_trial, m) if not self.monotone: rhoHis = (fRef - f_trial)/(sigRef - m) rho = max(rho, rhoHis) step_status = 'Rej' if rho >= self.TR.eta1: # Trust-region step is accepted. self.TR.UpdateRadius(rho, snorm) self.x = x_trial self.f = f_trial self.g = nlp.grad(self.x) self.gnorm = norms.norm2(self.g) step_status = 'Acc' # Update non-monotonicity parameters. if not self.monotone: sigRef = sigRef - m sigCan = sigCan - m if f_trial < fMin: fCan = f_trial fMin = f_trial sigCan = 0 l = 0 else: l = l + 1 if f_trial > fCan: fCan = f_trial sigCan = 0 if l == self.nIterNonMono: fRef = fCan sigRef = sigCan else: # Trust-region step is rejected. if self.ny: # Backtracking linesearch following "Nocedal & Yuan" slope = numpy.dot(self.g, step) bk = 0 while bk < self.nbk and \ f_trial >= self.f + 1.0e-4 * self.alpha * slope: bk = bk + 1 self.alpha /= 1.2 x_trial = self.x + self.alpha * step f_trial = nlp.obj(x_trial) self.x = x_trial self.f = f_trial self.g = nlp.grad(self.x) self.gnorm = norms.norm2(self.g) self.TR.Delta = self.alpha * snorm step_status = 'N-Y' else: self.TR.UpdateRadius(rho, snorm) self.step_status = step_status self.radii.append(self.TR.Delta) status = '' try: self.PostIteration() except UserExitRequest: status = 'usr' # Print out header, say, every 20 iterations if self.iter % 20 == 0 and self.verbose: self.log.info(self.hline) self.log.info(self.header) self.log.info(self.hline) if self.verbose: pstatus = step_status if step_status != 'Acc' else '' self.log.info(self.format % (self.iter, self.f, self.gnorm, cgiter, rho, self.TR.Delta, pstatus)) exitOptimal = self.gnorm < stoptol exitIter = self.iter > self.maxiter exitUser = status == 'usr' self.tsolve = cputime() - t # Solve time # Set final solver status. if status == 'usr': pass elif self.gnorm <= stoptol: status = 'opt' else: # self.iter > self.maxiter: status = 'itr' self.status = status
def scale(self, **kwargs): """ Equilibrate the constraint matrix of the linear program. Equilibration is done by first dividing every row by its largest element in absolute value and then by dividing every column by its largest element in absolute value. In effect the original problem:: minimize c' x + 1/2 x' Q x subject to A1 x + A2 s = b, x >= 0 is converted to:: minimize (Cc)' x + 1/2 x' (CQC') x subject to R A1 C x + R A2 C s = Rb, x >= 0, where the diagonal matrices R and C operate row and column scaling respectively. Upon return, the matrix A and the right-hand side b are scaled and the members `row_scale` and `col_scale` are set to the row and column scaling factors. The scaling may be undone by subsequently calling :meth:`unscale`. It is necessary to unscale the problem in order to unscale the final dual variables. Normally, the :meth:`solve` method takes care of unscaling the problem upon termination. """ w = sys.stdout.write m, n = self.A.shape row_scale = np.zeros(m) col_scale = np.zeros(n) (values, irow, jcol) = self.A.find() if self.verbose: w('Smallest and largest elements of A prior to scaling: ') w('%8.2e %8.2e\n' % (np.min(np.abs(values)), np.max(np.abs(values)))) # Find row scaling. for k in range(len(values)): row = irow[k] val = abs(values[k]) row_scale[row] = max(row_scale[row], val) row_scale[row_scale == 0.0] = 1.0 if self.verbose: w('Largest row scaling factor = %8.2e\n' % np.max(row_scale)) # Apply row scaling to A and b. values /= row_scale[irow] self.b /= row_scale # Find column scaling. for k in range(len(values)): col = jcol[k] val = abs(values[k]) col_scale[col] = max(col_scale[col], val) col_scale[col_scale == 0.0] = 1.0 if self.verbose: w('Largest column scaling factor = %8.2e\n' % np.max(col_scale)) # Apply column scaling to A and c. values /= col_scale[jcol] self.c[:self.qp.original_n] /= col_scale[:self.qp.original_n] if self.verbose: w('Smallest and largest elements of A after scaling: ') w('%8.2e %8.2e\n' % (np.min(np.abs(values)), np.max(np.abs(values)))) # Overwrite A with scaled values. self.A.put(values, irow, jcol) # Apply scaling to Hessian matrix Q. (values, irow, jcol) = self.Q.find() self.normQ = norm2(values) # Frobenius norm of Q values /= col_scale[irow] values /= col_scale[jcol] self.Q.put(values, irow, jcol) # Save row and column scaling. self.row_scale = row_scale self.col_scale = col_scale self.prob_scaled = True return