def msGmres(A, b, shifts, m=200, x0=None, tol=1e-5, maxiter=None, M=None, callback=None): """ Multi-Shift Gmres(m) """ A, M, x, b, postprocess = make_system(A, M, x0, b, xtype=None) n = len(b) if maxiter is None: maxiter = n * 10 matvec = A.matvec psolve = M.matvec xtype = x.dtype axpy, dot, norm = get_blas_funcs(['axpy', 'dot', 'nrm2'], dtype=xtype) # Relative tolerance tolb = tol * norm(b) # Compute initial residual: r = b - matvec(x) rnrm2 = norm(r) # We should call the callback func # Initialization V = np.zeros((n, m + 1), dtype=xtype) H = np.zeros((m + 1, m), dtype=xtype) cs = np.zeros(m + 1, dtype=xtype) sn = np.zeros(m + 1, dtype=xtype) e1 = np.zeros(m + 1, dtype=xtype) e1[0] = 1.0 # Begin iteration #for _iter in range(0, maxit): V[:, 0] = r / rnrm2 s = rnrm2 * e1 for i in range(0, m): # Construct orthonormal basis using Gram-Schmidt w = psolve(matvec(V[:, i])) for k in range(0, i + 1): H[k, i] = dot(V[:, k], w) #w = w - H[k, i]*V[:, k] axpy(V[:, k], w, None, -H[k, i]) H[i + 1, i] = norm(w) V[:, i + 1] = w / H[i + 1, i] # Multi-Shift with \bar{H} # We should call the callback func return V, H
def conjugate_gradient(A, b, x0=None, tol=1e-5, norm=np.linalg.norm, maxiter=None, callback=None, init_seed=None): dtype = A.dtype A, B, x, b, postprocess = make_system(A, None, x0, b) n = len(b) if maxiter is None: maxiter = n * 10 r = b - A.dot(x) rnrm = norm(r) p = r.copy() if callback is not None: callback(x) tolr = tol * norm(b) if rnrm < tolr: return postprocess(x)[:, None] for i in range(maxiter): r_norm = r.T.dot(r) Ap = A.dot(p) alpha = r_norm / p.T.dot(Ap) x += alpha * p if callback is not None: callback(x) r -= alpha * Ap rnrm = norm(r) if rnrm < tolr: break beta = r.T.dot(r) p = r + p * (beta / r_norm) return postprocess(x)[:, None]
def bicgstab(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None, callback=None, residuals=None): '''Biconjugate Gradient Algorithm with Stabilization Solves the linear system Ax = b. Left preconditioning is supported. Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by ||r_0||_2 maxiter : int maximum number of allowed iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A A.H x = M b. callback : function User-supplied function is called after each iteration as callback(xk), where xk is the current solution vector residuals : list residuals has the residual norm history, including the initial residual, appended to it Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of bicgstab == ====================================== 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. <0 numerical breakdown, or illegal input == ====================================== Notes ----- The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. Examples -------- >>> from pyamg.krylov.bicgstab import bicgstab >>> from pyamg.util.linalg import norm >>> import numpy >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = numpy.ones((A.shape[0],)) >>> (x,flag) = bicgstab(A,b, maxiter=2, tol=1e-8) >>> print norm(b - A*x) 4.68163045309 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 231-234, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' # Convert inputs to linear system, with error checking A, M, x, b, postprocess = make_system(A, M, x0, b, xtype) ## # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._bicgstab') # Check iteration numbers if maxiter is None: maxiter = len(x) + 5 elif maxiter < 1: raise ValueError('Number of iterations must be positive') # Prep for method r = b - A*x normr = norm(r) if residuals is not None: residuals[:] = [normr] # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol*normb: return (postprocess(x), 0) # Scale tol by ||r_0||_2 if normr != 0.0: tol = tol*normr # Is this a one dimensional matrix? if A.shape[0] == 1: entry = ravel(A*array([1.0], dtype=xtype)) return (postprocess(b/entry), 0) rstar = r.copy() p = r.copy() rrstarOld = inner(rstar.conjugate(), r) iter = 0 # Begin BiCGStab while True: Mp = M*p AMp = A*Mp # alpha = (r_j, rstar) / (A*M*p_j, rstar) alpha = rrstarOld/inner(rstar.conjugate(), AMp) # s_j = r_j - alpha*A*M*p_j s = r - alpha*AMp Ms = M*s AMs = A*Ms # omega = (A*M*s_j, s_j)/(A*M*s_j, A*M*s_j) omega = inner(AMs.conjugate(), s)/inner(AMs.conjugate(), AMs) # x_{j+1} = x_j + alpha*M*p_j + omega*M*s_j x = x + alpha*Mp + omega*Ms # r_{j+1} = s_j - omega*A*M*s r = s - omega*AMs # beta_j = (r_{j+1}, rstar)/(r_j, rstar) * (alpha/omega) rrstarNew = inner(rstar.conjugate(), r) beta = (rrstarNew / rrstarOld) * (alpha / omega) rrstarOld = rrstarNew # p_{j+1} = r_{j+1} + beta*(p_j - omega*A*M*p) p = r + beta*(p - omega*AMp) iter += 1 normr = norm(r) if residuals is not None: residuals.append(normr) if callback is not None: callback(x) if normr < tol: return (postprocess(x), 0) if iter == maxiter: return (postprocess(x), iter)
def lgmres(A, b, x0=None, tol=1e-5, maxiter=1000, M=None, callback=None, inner_m=30, outer_k=3, outer_v=None, store_outer_Av=True, prepend_outer_v=False, res = "both"): """ Solve a matrix equation using the LGMRES algorithm. The LGMRES algorithm [1]_ [2]_ is designed to avoid some problems in the convergence in restarted GMRES, and often converges in fewer iterations. Parameters ---------- A : {sparse matrix, dense matrix, LinearOperator} The real or complex N-by-N matrix of the linear system. b : {array, matrix} Right hand side of the linear system. Has shape (N,) or (N,1). x0 : {array, matrix} Starting guess for the solution. tol : float, optional Tolerance to achieve. The algorithm terminates when either the relative or the absolute residual is below `tol`. res: string, optional choose between relative, absolute or both (scipy default) to terminate the algo maxiter : int, optional Maximum number of iterations. Iteration will stop after maxiter steps even if the specified tolerance has not been achieved. M : {sparse matrix, dense matrix, LinearOperator}, optional Preconditioner for A. The preconditioner should approximate the inverse of A. Effective preconditioning dramatically improves the rate of convergence, which implies that fewer iterations are needed to reach a given error tolerance. callback : function, optional User-supplied function to call after each iteration. It is called as callback(xk), where xk is the current solution vector. inner_m : int, optional Number of inner GMRES iterations per each outer iteration. outer_k : int, optional Number of vectors to carry between inner GMRES iterations. According to [1]_, good values are in the range of 1...3. However, note that if you want to use the additional vectors to accelerate solving multiple similar problems, larger values may be beneficial. outer_v : list of tuples, optional List containing tuples ``(v, Av)`` of vectors and corresponding matrix-vector products, used to augment the Krylov subspace, and carried between inner GMRES iterations. The element ``Av`` can be `None` if the matrix-vector product should be re-evaluated. This parameter is modified in-place by `lgmres`, and can be used to pass "guess" vectors in and out of the algorithm when solving similar problems. store_outer_Av : bool, optional Whether LGMRES should store also A*v in addition to vectors `v` in the `outer_v` list. Default is True. prepend_outer_v : bool, optional Whether to put outer_v augmentation vectors before Krylov iterates. In standard LGMRES, prepend_outer_v=False. Returns ------- x : array or matrix The converged solution. info : int Provides convergence information: - 0 : successful exit - >0 : convergence to tolerance not achieved, number of iterations - <0 : illegal input or breakdown Notes ----- The LGMRES algorithm [1]_ [2]_ is designed to avoid the slowing of convergence in restarted GMRES, due to alternating residual vectors. Typically, it often outperforms GMRES(m) of comparable memory requirements by some measure, or at least is not much worse. Another advantage in this algorithm is that you can supply it with 'guess' vectors in the `outer_v` argument that augment the Krylov subspace. If the solution lies close to the span of these vectors, the algorithm converges faster. This can be useful if several very similar matrices need to be inverted one after another, such as in Newton-Krylov iteration where the Jacobian matrix often changes little in the nonlinear steps. References ---------- .. [1] A.H. Baker and E.R. Jessup and T. Manteuffel, "A Technique for Accelerating the Convergence of Restarted GMRES", SIAM J. Matrix Anal. Appl. 26, 962 (2005). .. [2] A.H. Baker, "On Improving the Performance of the Linear Solver restarted GMRES", PhD thesis, University of Colorado (2003). Examples -------- >>> from scipy.sparse import csc_matrix >>> from scipy.sparse.linalg import lgmres >>> A = csc_matrix([[3, 2, 0], [1, -1, 0], [0, 5, 1]], dtype=float) >>> b = np.array([2, 4, -1], dtype=float) >>> x, exitCode = lgmres(A, b) >>> print(exitCode) # 0 indicates successful convergence 0 >>> np.allclose(A.dot(x), b) True """ A,M,x,b,postprocess = make_system(A,M,x0,b) if not np.isfinite(b).all(): raise ValueError("RHS must contain only finite numbers") matvec = A.matvec psolve = M.matvec if outer_v is None: outer_v = [] axpy, dot, scal = None, None, None nrm2 = get_blas_funcs('nrm2', [b]) b_norm = nrm2(b) if b_norm == 0: b_norm = 1 for k_outer in xrange(maxiter): r_outer = matvec(x) - b # -- callback if callback is not None: callback(x) # -- determine input type routines if axpy is None: if np.iscomplexobj(r_outer) and not np.iscomplexobj(x): x = x.astype(r_outer.dtype) axpy, dot, scal, nrm2 = get_blas_funcs(['axpy', 'dot', 'scal', 'nrm2'], (x, r_outer)) trtrs = get_lapack_funcs('trtrs', (x, r_outer)) # -- check stopping condition r_norm = nrm2(r_outer) if res == "both": if r_norm <= tol * b_norm or r_norm <= tol: break elif res == "absolute": if r_norm <= tol: break elif res == "relative": if r_norm <= tol * b_norm: break else: raise ValueError("res must be absolute, relative or both") # -- inner LGMRES iteration v0 = -psolve(r_outer) inner_res_0 = nrm2(v0) if inner_res_0 == 0: rnorm = nrm2(r_outer) raise RuntimeError("Preconditioner returned a zero vector; " "|v| ~ %.1g, |M v| = 0" % rnorm) v0 = scal(1.0/inner_res_0, v0) try: Q, R, B, vs, zs, y = _fgmres(matvec, v0, inner_m, lpsolve=psolve, atol=tol*b_norm/r_norm, outer_v=outer_v, prepend_outer_v=prepend_outer_v) y *= inner_res_0 if not np.isfinite(y).all(): # Overflow etc. in computation. There's no way to # recover from this, so we have to bail out. raise LinAlgError() except LinAlgError: # Floating point over/underflow, non-finite result from # matmul etc. -- report failure. return postprocess(x), k_outer + 1 # -- GMRES terminated: eval solution dx = zs[0]*y[0] for w, yc in zip(zs[1:], y[1:]): dx = axpy(w, dx, dx.shape[0], yc) # dx += w*yc # -- Store LGMRES augmentation vectors nx = nrm2(dx) if nx > 0: if store_outer_Av: q = Q.dot(R.dot(y)) ax = vs[0]*q[0] for v, qc in zip(vs[1:], q[1:]): ax = axpy(v, ax, ax.shape[0], qc) outer_v.append((dx/nx, ax/nx)) else: outer_v.append((dx/nx, None)) # -- Retain only a finite number of augmentation vectors while len(outer_v) > outer_k: del outer_v[0] # -- Apply step x += dx else: # didn't converge ... return postprocess(x), maxiter return postprocess(x), 0
def steepest_descent(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None, callback=None, residuals=None): '''Steepest descent algorithm Solves the linear system Ax = b. Left preconditioning is supported. Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by the preconditioner norm of r_0, or ||r_0||_M. maxiter : int maximum number of allowed iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A x = M b. callback : function User-supplied function is called after each iteration as callback(xk), where xk is the current solution vector residuals : list residuals contains the residual norm history, including the initial residual. The preconditioner norm is used, instead of the Euclidean norm. Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of cg == ======================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. <0 numerical breakdown, or illegal input == ======================================= Notes ----- The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. The residual in the preconditioner norm is both used for halting and returned in the residuals list. Examples -------- >>> from pyamg.krylov import steepest_descent >>> from pyamg.util.linalg import norm >>> import numpy >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = numpy.ones((A.shape[0],)) >>> (x,flag) = steepest_descent(A,b, maxiter=2, tol=1e-8) >>> print norm(b - A*x) 7.89436429704 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 137--142, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' A, M, x, b, postprocess = make_system(A, M, x0, b, xtype=None) n = len(b) ## # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._steepest_descent') # determine maxiter if maxiter is None: maxiter = int(len(b)) elif maxiter < 1: raise ValueError('Number of iterations must be positive') # setup method r = b - A * x z = M * r rz = inner(r.conjugate(), z) # use preconditioner norm normr = sqrt(rz) if residuals is not None: residuals[:] = [normr] #initial residual # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol * normb: return (postprocess(x), 0) # Scale tol by ||r_0||_M if normr != 0.0: tol = tol * normr # How often should r be recomputed recompute_r = 50 iter = 0 while True: iter = iter + 1 q = A * z zAz = inner(z.conjugate(), q) # check curvature of A if zAz < 0.0: warn( "\nIndefinite matrix detected in steepest descent, aborting\n") return (postprocess(x), -1) alpha = rz / zAz # step size x = x + alpha * z if mod(iter, recompute_r) and iter > 0: r = b - A * x else: r = r - alpha * q z = M * r rz = inner(r.conjugate(), z) if rz < 0.0: # check curvature of M warn( "\nIndefinite preconditioner detected in steepest descent, aborting\n" ) return (postprocess(x), -1) normr = sqrt(rz) # use preconditioner norm if residuals is not None: residuals.append(normr) if callback is not None: callback(x) if normr < tol: return (postprocess(x), 0) elif rz == 0.0: # important to test after testing normr < tol. rz == 0.0 is an # indicator of convergence when r = 0.0 warn( "\nSingular preconditioner detected in steepest descent, ceasing iterations\n" ) return (postprocess(x), -1) if iter == maxiter: return (postprocess(x), iter)
def gmres_householder(A, b, x0=None, tol=1e-5, restrt=None, maxiter=None, xtype=None, M=None, callback=None, residuals=None): ''' Generalized Minimum Residual Method (GMRES) GMRES iteratively refines the initial solution guess to the system Ax = b Householder reflections are used for orthogonalization Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n, 1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by the norm of the initial preconditioned residual restrt : {None, int} - if int, restrt is max number of inner iterations and maxiter is the max number of outer iterations - if None, do not restart GMRES, and max number of inner iterations is maxiter maxiter : {None, int} - if restrt is None, maxiter is the max number of inner iterations and GMRES does not restart - if restrt is int, maxiter is the max number of outer iterations, and restrt is the max number of inner iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A x = M b. callback : function User-supplied function is called after each iteration as callback( ||rk||_2 ), where rk is the current preconditioned residual vector residuals : list residuals contains the preconditioned residual norm history, including the initial residual. Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of gmres == ============================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. This value is precisely the order of the Krylov space. <0 numerical breakdown, or illegal input == ============================================= Notes ----- - The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. - For robustness, Householder reflections are used to orthonormalize the Krylov Space Givens Rotations are used to provide the residual norm each iteration Examples -------- >>> from pyamg.krylov import gmres >>> from pyamg.util.linalg import norm >>> import numpy as np >>> from pyamg.gallery import poisson >>> A = poisson((10, 10)) >>> b = np.ones((A.shape[0],)) >>> (x, flag) = gmres(A, b, maxiter=2, tol=1e-8, orthog='householder') >>> print norm(b - A*x) 6.5428213057 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' # Convert inputs to linear system, with error checking A, M, x, b, postprocess = make_system(A, M, x0, b) dimen = A.shape[0] # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._gmres_householder') # Choose type if not hasattr(A, 'dtype'): Atype = upcast(x.dtype, b.dtype) else: Atype = A.dtype if not hasattr(M, 'dtype'): Mtype = upcast(x.dtype, b.dtype) else: Mtype = M.dtype xtype = upcast(Atype, x.dtype, b.dtype, Mtype) if restrt is not None: restrt = int(restrt) if maxiter is not None: maxiter = int(maxiter) # Should norm(r) be kept if residuals == []: keep_r = True else: keep_r = False # Set number of outer and inner iterations if restrt: if maxiter: max_outer = maxiter else: max_outer = 1 if restrt > dimen: warn('Setting number of inner iterations (restrt) to maximum \ allowed, which is A.shape[0] ') restrt = dimen max_inner = restrt else: max_outer = 1 if maxiter > dimen: warn('Setting number of inner iterations (maxiter) to maximum \ allowed, which is A.shape[0] ') maxiter = dimen elif maxiter is None: maxiter = min(dimen, 40) max_inner = maxiter # Get fast access to underlying LAPACK routine [lartg] = get_lapack_funcs(['lartg'], [x]) # Is this a one dimensional matrix? if dimen == 1: entry = ravel(A * array([1.0], dtype=xtype)) return (postprocess(b / entry), 0) # Prep for method r = b - ravel(A * x) # Apply preconditioner r = ravel(M * r) normr = norm(r) if keep_r: residuals.append(normr) # Check for nan, inf # if isnan(r).any() or isinf(r).any(): # warn('inf or nan after application of preconditioner') # return(postprocess(x), -1) # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol * normb: if callback is not None: callback(norm(r)) return (postprocess(x), 0) # Scale tol by ||r_0||_2, we use the preconditioned residual # because this is left preconditioned GMRES. if normr != 0.0: tol = tol * normr # Use separate variable to track iterations. If convergence fails, we # cannot simply report niter = (outer-1)*max_outer + inner. Numerical # error could cause the inner loop to halt while the actual ||r|| > tol. niter = 0 # Begin GMRES for outer in range(max_outer): # Calculate vector w, which defines the Householder reflector # Take shortcut in calculating, # w = r + sign(r[1])*||r||_2*e_1 w = r beta = mysign(w[0]) * normr w[0] = w[0] + beta w[:] = w / norm(w) # Preallocate for Krylov vectors, Householder reflectors and # Hessenberg matrix # Space required is O(dimen*max_inner) # Givens Rotations Q = zeros((4 * max_inner, ), dtype=xtype) # upper Hessenberg matrix (made upper tri with Givens Rotations) H = zeros((max_inner, max_inner), dtype=xtype) # Householder reflectors W = zeros((max_inner + 1, dimen), dtype=xtype) W[0, :] = w # Multiply r with (I - 2*w*w.T), i.e. apply the Householder reflector # This is the RHS vector for the problem in the Krylov Space g = zeros((dimen, ), dtype=xtype) g[0] = -beta for inner in range(max_inner): # Calculate Krylov vector in two steps # (1) Calculate v = P_j = (I - 2*w*w.T)v, where k = inner v = -2.0 * conjugate(w[inner]) * w v[inner] = v[inner] + 1.0 # (2) Calculate the rest, v = P_1*P_2*P_3...P_{j-1}*ej. # for j in range(inner-1,-1,-1): # v -= 2.0*dot(conjugate(W[j,:]), v)*W[j,:] amg_core.apply_householders(v, ravel(W), dimen, inner - 1, -1, -1) # Calculate new search direction v = ravel(A * v) # Apply preconditioner v = ravel(M * v) # Check for nan, inf # if isnan(v).any() or isinf(v).any(): # warn('inf or nan after application of preconditioner') # return(postprocess(x), -1) # Factor in all Householder orthogonal reflections on new search # direction # for j in range(inner+1): # v -= 2.0*dot(conjugate(W[j,:]), v)*W[j,:] amg_core.apply_householders(v, ravel(W), dimen, 0, inner + 1, 1) # Calculate next Householder reflector, w # w = v[inner+1:] + sign(v[inner+1])*||v[inner+1:]||_2*e_{inner+1) # Note that if max_inner = dimen, then this is unnecessary for the # last inner iteration, when inner = dimen-1. Here we do not need # to calculate a Householder reflector or Givens rotation because # nnz(v) is already the desired length, i.e. we do not need to # zero anything out. if inner != dimen - 1: if inner < (max_inner - 1): w = W[inner + 1, :] vslice = v[inner + 1:] alpha = norm(vslice) if alpha != 0: alpha = mysign(vslice[0]) * alpha # do not need the final reflector for future calculations if inner < (max_inner - 1): w[inner + 1:] = vslice w[inner + 1] += alpha w[:] = w / norm(w) # Apply new reflector to v # v = v - 2.0*w*(w.T*v) v[inner + 1] = -alpha v[inner + 2:] = 0.0 if inner > 0: # Apply all previous Givens Rotations to v amg_core.apply_givens(Q, v, dimen, inner) # Calculate the next Givens rotation, where j = inner Note that if # max_inner = dimen, then this is unnecessary for the last inner # iteration, when inner = dimen-1. Here we do not need to # calculate a Householder reflector or Givens rotation because # nnz(v) is already the desired length, i.e. we do not need to zero # anything out. if inner != dimen - 1: if v[inner + 1] != 0: [c, s, r] = lartg(v[inner], v[inner + 1]) Qblock = array([[c, s], [-conjugate(s), c]], dtype=xtype) Q[(inner * 4):((inner + 1) * 4)] = ravel(Qblock).copy() # Apply Givens Rotation to g, the RHS for the linear system # in the Krylov Subspace. Note that this dot does a matrix # multiply, not an actual dot product where a conjugate # transpose is taken g[inner:inner + 2] = dot(Qblock, g[inner:inner + 2]) # Apply effect of Givens Rotation to v v[inner] = dot(Qblock[0, :], v[inner:inner + 2]) v[inner + 1] = 0.0 # Write to upper Hessenberg Matrix, # the LHS for the linear system in the Krylov Subspace H[:, inner] = v[0:max_inner] niter += 1 # Don't update normr if last inner iteration, because # normr is calculated directly after this loop ends. if inner < max_inner - 1: normr = abs(g[inner + 1]) if normr < tol: break # Allow user access to residual if callback is not None: callback(normr) if keep_r: residuals.append(normr) # end inner loop, back to outer loop # Find best update to x in Krylov Space, V. Solve inner+1 x inner+1 # system. Apparently this is the best way to solve a triangular system # in the magical world of scipy # piv = arange(inner+1) # y = lu_solve((H[0:(inner+1), 0:(inner+1)], piv), g[0:(inner+1)], # trans=0) y = sp.linalg.solve(H[0:(inner + 1), 0:(inner + 1)], g[0:(inner + 1)]) # Use Horner like Scheme to map solution, y, back to original space. # Note that we do not use the last reflector. update = zeros(x.shape, dtype=xtype) # for j in range(inner,-1,-1): # update[j] += y[j] # # Apply j-th reflector, (I - 2.0*w_j*w_j.T)*upadate # update -= 2.0*dot(conjugate(W[j,:]), update)*W[j,:] amg_core.householder_hornerscheme(update, ravel(W), ravel(y), dimen, inner, -1, -1) x[:] = x + update r = b - ravel(A * x) # Apply preconditioner r = ravel(M * r) normr = norm(r) # Check for nan, inf # if isnan(r).any() or isinf(r).any(): # warn('inf or nan after application of preconditioner') # return(postprocess(x), -1) # Allow user access to residual if callback is not None: callback(normr) if keep_r: residuals.append(normr) # Has GMRES stagnated? indices = (x != 0) if indices.any(): change = max(abs(update[indices] / x[indices])) if change < 1e-12: # No change, halt return (postprocess(x), -1) # test for convergence if normr < tol: return (postprocess(x), 0) # end outer loop return (postprocess(x), niter)
def gmres_mgs(A, b, x0=None, tol=1e-5, restrt=None, maxiter=None, xtype=None, M=None, callback=None, residuals=None, reorth=False): ''' Generalized Minimum Residual Method (GMRES) GMRES iteratively refines the initial solution guess to the system Ax = b Modified Gram-Schmidt version Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by the norm of the initial preconditioned residual restrt : {None, int} - if int, restrt is max number of inner iterations and maxiter is the max number of outer iterations - if None, do not restart GMRES, and max number of inner iterations is maxiter maxiter : {None, int} - if restrt is None, maxiter is the max number of inner iterations and GMRES does not restart - if restrt is int, maxiter is the max number of outer iterations, and restrt is the max number of inner iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A x = M b. callback : function User-supplied function is called after each iteration as callback( ||rk||_2 ), where rk is the current preconditioned residual vector residuals : list residuals contains the preconditioned residual norm history, including the initial residual. reorth : boolean If True, then a check is made whether to re-orthogonalize the Krylov space each GMRES iteration Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of gmres == ============================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. This value is precisely the order of the Krylov space. <0 numerical breakdown, or illegal input == ============================================= Notes ----- - The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. - For robustness, modified Gram-Schmidt is used to orthogonalize the Krylov Space Givens Rotations are used to provide the residual norm each iteration Examples -------- >>> from pyamg.krylov import gmres >>> from pyamg.util.linalg import norm >>> import numpy as np >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = np.ones((A.shape[0],)) >>> (x,flag) = gmres(A,b, maxiter=2, tol=1e-8, orthog='mgs') >>> print norm(b - A*x) >>> 6.5428213057 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003 http://www-users.cs.umn.edu/~saad/books.html .. [2] C. T. Kelley, http://www4.ncsu.edu/~ctk/matlab_roots.html ''' # Convert inputs to linear system, with error checking A, M, x, b, postprocess = make_system(A, M, x0, b) dimen = A.shape[0] # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._gmres_mgs') # Choose type if not hasattr(A, 'dtype'): Atype = upcast(x.dtype, b.dtype) else: Atype = A.dtype if not hasattr(M, 'dtype'): Mtype = upcast(x.dtype, b.dtype) else: Mtype = M.dtype xtype = upcast(Atype, x.dtype, b.dtype, Mtype) if restrt is not None: restrt = int(restrt) if maxiter is not None: maxiter = int(maxiter) # Get fast access to underlying BLAS routines # dotc is the conjugate dot, dotu does no conjugation [lartg] = get_lapack_funcs(['lartg'], [x] ) if iscomplexobj(zeros((1,), dtype=xtype)): [axpy, dotu, dotc, scal] =\ get_blas_funcs(['axpy', 'dotu', 'dotc', 'scal'], [x]) else: # real type [axpy, dotu, dotc, scal] =\ get_blas_funcs(['axpy', 'dot', 'dot', 'scal'], [x]) # Make full use of direct access to BLAS by defining own norm def norm(z): return sqrt(real(dotc(z, z))) # Should norm(r) be kept if residuals == []: keep_r = True else: keep_r = False # Set number of outer and inner iterations if restrt: if maxiter: max_outer = maxiter else: max_outer = 1 if restrt > dimen: warn('Setting number of inner iterations (restrt) to maximum\ allowed, which is A.shape[0] ') restrt = dimen max_inner = restrt else: max_outer = 1 if maxiter > dimen: warn('Setting number of inner iterations (maxiter) to maximum\ allowed, which is A.shape[0] ') maxiter = dimen elif maxiter is None: maxiter = min(dimen, 40) max_inner = maxiter # Is this a one dimensional matrix? if dimen == 1: entry = ravel(A*array([1.0], dtype=xtype)) return (postprocess(b/entry), 0) # Prep for method r = b - ravel(A*x) # Apply preconditioner r = ravel(M*r) normr = norm(r) if keep_r: residuals.append(normr) # Check for nan, inf # if isnan(r).any() or isinf(r).any(): # warn('inf or nan after application of preconditioner') # return(postprocess(x), -1) # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol*normb: if callback is not None: callback(norm(r)) return (postprocess(x), 0) # Scale tol by ||r_0||_2, we use the preconditioned residual # because this is left preconditioned GMRES. if normr != 0.0: tol = tol*normr # Use separate variable to track iterations. If convergence fails, we # cannot simply report niter = (outer-1)*max_outer + inner. Numerical # error could cause the inner loop to halt while the actual ||r|| > tol. niter = 0 # Begin GMRES for outer in range(max_outer): # Preallocate for Givens Rotations, Hessenberg matrix and Krylov Space # Space required is O(dimen*max_inner). # NOTE: We are dealing with row-major matrices, so we traverse in a # row-major fashion, # i.e., H and V's transpose is what we store. Q = [] # Givens Rotations # Upper Hessenberg matrix, which is then # converted to upper tri with Givens Rots H = zeros((max_inner+1, max_inner+1), dtype=xtype) V = zeros((max_inner+1, dimen), dtype=xtype) # Krylov Space # vs store the pointers to each column of V. # This saves a considerable amount of time. vs = [] # v = r/normr V[0, :] = scal(1.0/normr, r) vs.append(V[0, :]) # This is the RHS vector for the problem in the Krylov Space g = zeros((dimen,), dtype=xtype) g[0] = normr for inner in range(max_inner): # New Search Direction v = V[inner+1, :] v[:] = ravel(M*(A*vs[-1])) vs.append(v) normv_old = norm(v) # Check for nan, inf # if isnan(V[inner+1, :]).any() or isinf(V[inner+1, :]).any(): # warn('inf or nan after application of preconditioner') # return(postprocess(x), -1) # Modified Gram Schmidt for k in range(inner+1): vk = vs[k] alpha = dotc(vk, v) H[inner, k] = alpha v[:] = axpy(vk, v, dimen, -alpha) normv = norm(v) H[inner, inner+1] = normv # Re-orthogonalize if (reorth is True) and (normv_old == normv_old + 0.001*normv): for k in range(inner+1): vk = vs[k] alpha = dotc(vk, v) H[inner, k] = H[inner, k] + alpha v[:] = axpy(vk, v, dimen, -alpha) # Check for breakdown if H[inner, inner+1] != 0.0: v[:] = scal(1.0/H[inner, inner+1], v) # Apply previous Givens rotations to H if inner > 0: apply_givens(Q, H[inner, :], inner) # Calculate and apply next complex-valued Givens Rotation # ==> Note that if max_inner = dimen, then this is unnecessary # for the last inner # iteration, when inner = dimen-1. if inner != dimen-1: if H[inner, inner+1] != 0: [c, s, r] = lartg(H[inner, inner], H[inner, inner+1]) Qblock = array([[c, s], [-conjugate(s), c]], dtype=xtype) Q.append(Qblock) # Apply Givens Rotation to g, # the RHS for the linear system in the Krylov Subspace. g[inner:inner+2] = sp.dot(Qblock, g[inner:inner+2]) # Apply effect of Givens Rotation to H H[inner, inner] = dotu(Qblock[0, :], H[inner, inner:inner+2]) H[inner, inner+1] = 0.0 niter += 1 # Don't update normr if last inner iteration, because # normr is calculated directly after this loop ends. if inner < max_inner-1: normr = abs(g[inner+1]) if normr < tol: break # Allow user access to residual if callback is not None: callback(normr) if keep_r: residuals.append(normr) # end inner loop, back to outer loop # Find best update to x in Krylov Space V. Solve inner x inner system. y = sp.linalg.solve(H[0:inner+1, 0:inner+1].T, g[0:inner+1]) update = ravel(sp.mat(V[:inner+1, :]).T*y.reshape(-1, 1)) x = x + update r = b - ravel(A*x) # Apply preconditioner r = ravel(M*r) normr = norm(r) # Check for nan, inf # if isnan(r).any() or isinf(r).any(): # warn('inf or nan after application of preconditioner') # return(postprocess(x), -1) # Allow user access to residual if callback is not None: callback(normr) if keep_r: residuals.append(normr) # Has GMRES stagnated? indices = (x != 0) if indices.any(): change = max(abs(update[indices] / x[indices])) if change < 1e-12: # No change, halt return (postprocess(x), -1) # test for convergence if normr < tol: return (postprocess(x), 0) # end outer loop return (postprocess(x), niter)
def gcrotmk(A, b, x0=None, tol=1e-5, maxiter=1000, M=None, callback=None, m=20, k=None, CU=None, discard_C=False, truncate='oldest'): """ Solve a matrix equation using flexible GCROT(m,k) algorithm. Parameters ---------- A : {sparse matrix, dense matrix, LinearOperator} The real or complex N-by-N matrix of the linear system. b : {array, matrix} Right hand side of the linear system. Has shape (N,) or (N,1). x0 : {array, matrix} Starting guess for the solution. tol : float, optional Tolerance to achieve. The algorithm terminates when either the relative or the absolute residual is below `tol`. maxiter : int, optional Maximum number of iterations. Iteration will stop after maxiter steps even if the specified tolerance has not been achieved. M : {sparse matrix, dense matrix, LinearOperator}, optional Preconditioner for A. The preconditioner should approximate the inverse of A. gcrotmk is a 'flexible' algorithm and the preconditioner can vary from iteration to iteration. Effective preconditioning dramatically improves the rate of convergence, which implies that fewer iterations are needed to reach a given error tolerance. callback : function, optional User-supplied function to call after each iteration. It is called as callback(xk), where xk is the current solution vector. m : int, optional Number of inner FGMRES iterations per each outer iteration. Default: 20 k : int, optional Number of vectors to carry between inner FGMRES iterations. According to [2]_, good values are around m. Default: m CU : list of tuples, optional List of tuples ``(c, u)`` which contain the columns of the matrices C and U in the GCROT(m,k) algorithm. For details, see [2]_. The list given and vectors contained in it are modified in-place. If not given, start from empty matrices. The ``c`` elements in the tuples can be ``None``, in which case the vectors are recomputed via ``c = A u`` on start and orthogonalized as described in [3]_. discard_C : bool, optional Discard the C-vectors at the end. Useful if recycling Krylov subspaces for different linear systems. truncate : {'oldest', 'smallest'}, optional Truncation scheme to use. Drop: oldest vectors, or vectors with smallest singular values using the scheme discussed in [1,2]. See [2]_ for detailed comparison. Default: 'oldest' Returns ------- x : array or matrix The solution found. info : int Provides convergence information: * 0 : successful exit * >0 : convergence to tolerance not achieved, number of iterations References ---------- .. [1] E. de Sturler, ''Truncation strategies for optimal Krylov subspace methods'', SIAM J. Numer. Anal. 36, 864 (1999). .. [2] J.E. Hicken and D.W. Zingg, ''A simplified and flexible variant of GCROT for solving nonsymmetric linear systems'', SIAM J. Sci. Comput. 32, 172 (2010). .. [3] M.L. Parks, E. de Sturler, G. Mackey, D.D. Johnson, S. Maiti, ''Recycling Krylov subspaces for sequences of linear systems'', SIAM J. Sci. Comput. 28, 1651 (2006). """ A, M, x, b, postprocess = make_system(A, M, x0, b) if not np.isfinite(b).all(): raise ValueError("RHS must contain only finite numbers") if truncate not in ('oldest', 'smallest'): raise ValueError("Invalid value for 'truncate': %r" % (truncate, )) matvec = A.matvec psolve = M.matvec if CU is None: CU = [] if k is None: k = m axpy, dot, scal = None, None, None r = b - matvec(x) axpy, dot, scal, nrm2 = get_blas_funcs(['axpy', 'dot', 'scal', 'nrm2'], (x, r)) b_norm = nrm2(b) if b_norm == 0: b_norm = 1 if discard_C: CU[:] = [(None, u) for c, u in CU] # Reorthogonalize old vectors if CU: # Sort already existing vectors to the front CU.sort(key=lambda cu: cu[0] is not None) # Fill-in missing ones C = np.empty((A.shape[0], len(CU)), dtype=r.dtype, order='F') us = [] j = 0 while CU: # More memory-efficient: throw away old vectors as we go c, u = CU.pop(0) if c is None: c = matvec(u) C[:, j] = c j += 1 us.append(u) # Orthogonalize Q, R, P = qr(C, overwrite_a=True, mode='economic', pivoting=True) del C # C := Q cs = list(Q.T) # U := U P R^-1, back-substitution new_us = [] for j in xrange(len(cs)): u = us[P[j]] for i in xrange(j): u = axpy(us[P[i]], u, u.shape[0], -R[i, j]) if abs(R[j, j]) < 1e-12 * abs(R[0, 0]): # discard rest of the vectors break u = scal(1.0 / R[j, j], u) new_us.append(u) # Form the new CU lists CU[:] = list(zip(cs, new_us))[::-1] if CU: axpy, dot = get_blas_funcs(['axpy', 'dot'], (r, )) # Solve first the projection operation with respect to the CU # vectors. This corresponds to modifying the initial guess to # be # # x' = x + U y # y = argmin_y || b - A (x + U y) ||^2 # # The solution is y = C^H (b - A x) for c, u in CU: yc = dot(c, r) x = axpy(u, x, x.shape[0], yc) r = axpy(c, r, r.shape[0], -yc) # GCROT main iteration for j_outer in xrange(maxiter): # -- callback if callback is not None: callback(x) beta = nrm2(r) # -- check stopping condition if beta <= max(tol, tol * b_norm): j_outer = -1 break ml = m + max(k - len(CU), 0) cs = [c for c, u in CU] try: Q, R, B, vs, zs, y = _fgmres(matvec, r / beta, ml, rpsolve=psolve, atol=tol * b_norm / beta, cs=cs) y *= beta except LinAlgError: # Floating point over/underflow, non-finite result from # matmul etc. -- report failure. break # # At this point, # # [A U, A Z] = [C, V] G; G = [ I B ] # [ 0 H ] # # where [C, V] has orthonormal columns, and r = beta v_0. Moreover, # # || b - A (x + Z y + U q) ||_2 = || r - C B y - V H y - C q ||_2 = min! # # from which y = argmin_y || beta e_1 - H y ||_2, and q = -B y # # # GCROT(m,k) update # # Define new outer vectors # ux := (Z - U B) y ux = zs[0] * y[0] for z, yc in zip(zs[1:], y[1:]): ux = axpy(z, ux, ux.shape[0], yc) # ux += z*yc by = B.dot(y) for cu, byc in zip(CU, by): c, u = cu ux = axpy(u, ux, ux.shape[0], -byc) # ux -= u*byc # cx := V H y hy = Q.dot(R.dot(y)) cx = vs[0] * hy[0] for v, hyc in zip(vs[1:], hy[1:]): cx = axpy(v, cx, cx.shape[0], hyc) # cx += v*hyc # Normalize cx, maintaining cx = A ux # This new cx is orthogonal to the previous C, by construction try: alpha = 1 / nrm2(cx) if not np.isfinite(alpha): raise FloatingPointError() except (FloatingPointError, ZeroDivisionError): # Cannot update, so skip it continue cx = scal(alpha, cx) ux = scal(alpha, ux) # Update residual and solution gamma = dot(cx, r) r = axpy(cx, r, r.shape[0], -gamma) # r -= gamma*cx x = axpy(ux, x, x.shape[0], gamma) # x += gamma*ux # Truncate CU if truncate == 'oldest': while len(CU) >= k and CU: del CU[0] elif truncate == 'smallest': if len(CU) >= k and CU: # cf. [1,2] D = solve(R[:-1, :].T, B.T).T W, sigma, V = svd(D) # C := C W[:,:k-1], U := U W[:,:k-1] new_CU = [] for j, w in enumerate(W[:, :k - 1].T): c, u = CU[0] c = c * w[0] u = u * w[0] for cup, wp in zip(CU[1:], w[1:]): cp, up = cup c = axpy(cp, c, c.shape[0], wp) u = axpy(up, u, u.shape[0], wp) # Reorthogonalize at the same time; not necessary # in exact arithmetic, but floating point error # tends to accumulate here for cp, up in new_CU: alpha = dot(cp, c) c = axpy(cp, c, c.shape[0], -alpha) u = axpy(up, u, u.shape[0], -alpha) alpha = nrm2(c) c = scal(1.0 / alpha, c) u = scal(1.0 / alpha, u) new_CU.append((c, u)) CU[:] = new_CU # Add new vector to CU CU.append((cx, ux)) # Include the solution vector to the span CU.append((None, x.copy())) if discard_C: CU[:] = [(None, uz) for cz, uz in CU] return postprocess(x), j_outer + 1
def bicgstab(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None, callback=None, residuals=None): '''Biconjugate Gradient Algorithm with Stabilization Solves the linear system Ax = b. Left preconditioning is supported. Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by ||r_0||_2 maxiter : int maximum number of allowed iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A A.H x = M b. callback : function User-supplied function is called after each iteration as callback(xk), where xk is the current solution vector residuals : list residuals has the residual norm history, including the initial residual, appended to it Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of bicgstab == ====================================== 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. <0 numerical breakdown, or illegal input == ====================================== Notes ----- The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. Examples -------- >>> from pyamg.krylov.bicgstab import bicgstab >>> from pyamg.util.linalg import norm >>> import numpy as np >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = np.ones((A.shape[0],)) >>> (x,flag) = bicgstab(A,b, maxiter=2, tol=1e-8) >>> print norm(b - A*x) 4.68163045309 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 231-234, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' # Convert inputs to linear system, with error checking A, M, x, b, postprocess = make_system(A, M, x0, b, xtype) # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._bicgstab') # Check iteration numbers if maxiter is None: maxiter = len(x) + 5 elif maxiter < 1: raise ValueError('Number of iterations must be positive') # Prep for method r = b - A*x normr = norm(r) if residuals is not None: residuals[:] = [normr] # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol*normb: return (postprocess(x), 0) # Scale tol by ||r_0||_2 if normr != 0.0: tol = tol*normr # Is this a one dimensional matrix? if A.shape[0] == 1: entry = np.ravel(A*np.array([1.0], dtype=xtype)) return (postprocess(b/entry), 0) rstar = r.copy() p = r.copy() rrstarOld = np.inner(rstar.conjugate(), r) iter = 0 # Begin BiCGStab while True: Mp = M*p AMp = A*Mp # alpha = (r_j, rstar) / (A*M*p_j, rstar) alpha = rrstarOld/np.inner(rstar.conjugate(), AMp) # s_j = r_j - alpha*A*M*p_j s = r - alpha*AMp Ms = M*s AMs = A*Ms # omega = (A*M*s_j, s_j)/(A*M*s_j, A*M*s_j) omega = np.inner(AMs.conjugate(), s)/np.inner(AMs.conjugate(), AMs) # x_{j+1} = x_j + alpha*M*p_j + omega*M*s_j x = x + alpha*Mp + omega*Ms # r_{j+1} = s_j - omega*A*M*s r = s - omega*AMs # beta_j = (r_{j+1}, rstar)/(r_j, rstar) * (alpha/omega) rrstarNew = np.inner(rstar.conjugate(), r) beta = (rrstarNew / rrstarOld) * (alpha / omega) rrstarOld = rrstarNew # p_{j+1} = r_{j+1} + beta*(p_j - omega*A*M*p) p = r + beta*(p - omega*AMp) iter += 1 normr = norm(r) if residuals is not None: residuals.append(normr) if callback is not None: callback(x) if normr < tol: return (postprocess(x), 0) if iter == maxiter: return (postprocess(x), iter)
def cgnr(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None, callback=None, residuals=None): '''Conjugate Gradient, Normal Residual algorithm Applies CG to the normal equations, A.H A x = b. Left preconditioning is supported. Note that unless A is well-conditioned, the use of CGNR is inadvisable Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by ||r_0||_2 maxiter : int maximum number of allowed iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A.H A x = b. callback : function User-supplied function is called after each iteration as callback(xk), where xk is the current solution vector residuals : list residuals has the residual norm history, including the initial residual, appended to it Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of cgnr == ======================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. <0 numerical breakdown, or illegal input == ======================================= Notes ----- The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. Examples -------- >>> from pyamg.krylov.cgnr import cgnr >>> from pyamg.util.linalg import norm >>> import numpy as np >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = np.ones((A.shape[0],)) >>> (x,flag) = cgnr(A,b, maxiter=2, tol=1e-8) >>> print norm(b - A*x) 9.3910201849 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 276-7, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' # Store the conjugate transpose explicitly as it will be used much later on if isspmatrix(A): AH = A.H else: # TODO avoid doing this since A may be a different sparse type AH = aslinearoperator(asmatrix(A).H) # Convert inputs to linear system, with error checking A, M, x, b, postprocess = make_system(A, M, x0, b) dimen = A.shape[0] # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._cgnr') # Choose type if not hasattr(A, 'dtype'): Atype = upcast(x.dtype, b.dtype) else: Atype = A.dtype if not hasattr(M, 'dtype'): Mtype = upcast(x.dtype, b.dtype) else: Mtype = M.dtype xtype = upcast(Atype, x.dtype, b.dtype, Mtype) # Should norm(r) be kept if residuals == []: keep_r = True else: keep_r = False # How often should r be recomputed recompute_r = 8 # Check iteration numbers. CGNR suffers from loss of orthogonality quite # easily, so we arbitrarily let the method go up to 130% over the # theoretically necessary limit of maxiter=dimen if maxiter is None: maxiter = int(ceil(1.3*dimen)) + 2 elif maxiter < 1: raise ValueError('Number of iterations must be positive') elif maxiter > (1.3*dimen): warn('maximum allowed inner iterations (maxiter) are the 130% times \ the number of dofs') maxiter = int(ceil(1.3*dimen)) + 2 # Prep for method r = b - A*x rhat = AH*r normr = norm(r) if keep_r: residuals.append(normr) # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol*normb: if callback is not None: callback(x) return (postprocess(x), 0) # Scale tol by ||r_0||_2 if normr != 0.0: tol = tol*normr # Begin CGNR # Apply preconditioner and calculate initial search direction z = M*rhat p = z.copy() old_zr = inner(z.conjugate(), rhat) for iter in range(maxiter): # w_j = A p_j w = A*p # alpha = (z_j, rhat_j) / (w_j, w_j) alpha = old_zr / inner(w.conjugate(), w) # x_{j+1} = x_j + alpha*p_j x += alpha*p # r_{j+1} = r_j - alpha*w_j if mod(iter, recompute_r) and iter > 0: r -= alpha*w else: r = b - A*x # rhat_{j+1} = A.H*r_{j+1} rhat = AH*r # z_{j+1} = M*r_{j+1} z = M*rhat # beta = (z_{j+1}, rhat_{j+1}) / (z_j, rhat_j) new_zr = inner(z.conjugate(), rhat) beta = new_zr / old_zr old_zr = new_zr # p_{j+1} = A.H*z_{j+1} + beta*p_j p *= beta p += z # Allow user access to residual if callback is not None: callback(x) # test for convergence normr = norm(r) if keep_r: residuals.append(normr) if normr < tol: return (postprocess(x), 0) # end loop return (postprocess(x), iter+1)
def gmres(A, b, x0=None, tol=1e-5, restrt=None, maxiter=None, xtype=None, M=None, callback=None, residuals=None): ''' Generalized Minimum Residual Method (GMRES) GMRES iteratively refines the initial solution guess to the system Ax = b For robustness, Householder reflections are used to orthonormalize the Krylov Space Givens Rotations are used to provide the residual norm each iteration Parameters ---------- A : array, matrix or sparse matrix n x n, linear system to solve b : array n x 1, right hand side x0 : array n x 1, initial guess default is a vector of zeros tol : float convergence tolerance restrt : int number of restarts total iterations = restrt*maxiter maxiter : int maximum number of allowed inner iterations xtype : type dtype for the solution M : matrix-like n x n, inverted preconditioner, i.e. solve M A x = b. For preconditioning with a mat-vec routine, set A.psolve = func, where func := M y callback : function callback( ||resid||_2 ) is called each iteration, residuals : {None, empty-list} If empty-list, residuals holds the residual norm history, including the initial residual, upon completion Returns ------- (xNew, info) xNew -- an updated guess to the solution of Ax = b info -- halting status of gmres 0 : successful exit >0 : convergence to tolerance not achieved, return iteration count instead. This value is precisely the order of the Krylov space. <0 : numerical breakdown, or illegal input Notes ----- Examples -------- >>>from pyamg.krylov import * >>>from scipy import rand >>>import pyamg >>>A = pyamg.poisson((50,50)) >>>b = rand(A.shape[0],) >>>(x,flag) = gmres(A,b) >>>print pyamg.util.linalg.norm(b - A*x) References ---------- Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003 ''' # Convert inputs to linear system, with error checking A, M, x, b, postprocess = make_system(A, M, x0, b, xtype) dimen = A.shape[0] # Choose type xtype = upcast(A.dtype, x.dtype, b.dtype, M.dtype) # We assume henceforth that shape=(n,) for all arrays b = ravel(array(b, xtype)) x = ravel(array(x, xtype)) # Should norm(r) be kept if residuals == []: keep_r = True else: keep_r = False # check number of iterations if restrt == None: restrt = 1 elif restrt < 1: raise ValueError('Number of restarts must be positive') if maxiter == None: maxiter = int(max(ceil(dimen / restrt))) elif maxiter < 1: raise ValueError('Number of iterations must be positive') elif maxiter > dimen: warn( 'maximimum allowed inner iterations (maxiter) are the number of degress of freedom' ) maxiter = dimen # Scale tol by normb normb = linalg.norm(b) if normb == 0: pass # if callback != None: # callback(0.0) # # return (postprocess(zeros((dimen,)), dtype=xtype),0) else: tol = tol * normb # Is this a one dimensional matrix? if dimen == 1: entry = ravel(A * array([1.0], dtype=xtype)) return (postprocess(b / entry), 0) # Prep for method r = b - ravel(A * x) normr = linalg.norm(r) if keep_r: residuals.append(normr) # Is initial guess sufficient? if normr <= tol: if callback != None: callback(norm(r)) return (postprocess(x), 0) #Apply preconditioner r = ravel(M * r) normr = linalg.norm(r) # Check for nan, inf if any(isnan(r)) or any(isinf(r)): warn('inf or nan after application of preconditioner') return (postprocess(x), -1) # Use separate variable to track iterations. If convergence fails, we cannot # simply report niter = (outer-1)*maxiter + inner. Numerical error could cause # the inner loop to halt before reaching maxiter while the actual ||r|| > tol. niter = 0 # Begin GMRES for outer in range(restrt): # Calculate vector w, which defines the Householder reflector # Take shortcut in calculating, # w = r + sign(r[1])*||r||_2*e_1 w = r beta = mysign(w[0]) * normr w[0] += beta w = w / linalg.norm(w) # Preallocate for Krylov vectors, Householder reflectors and Hessenberg matrix # Space required is O(dimen*maxiter) H = zeros( (maxiter, maxiter), dtype=xtype ) # upper Hessenberg matrix (actually made upper tri with Given's Rotations) W = zeros((dimen, maxiter), dtype=xtype) # Householder reflectors W[:, 0] = w # Multiply r with (I - 2*w*w.T), i.e. apply the Householder reflector # This is the RHS vector for the problem in the Krylov Space g = zeros((dimen, ), dtype=xtype) g[0] = -beta for inner in range(maxiter): # Calcute Krylov vector in two steps # (1) Calculate v = P_j = (I - 2*w*w.T)v, where k = inner v = -2.0 * conjugate(w[inner]) * w v[inner] += 1.0 # (2) Calculate the rest, v = P_1*P_2*P_3...P_{j-1}*ej. for j in range(inner - 1, -1, -1): v = v - 2.0 * dot(conjugate(W[:, j]), v) * W[:, j] # Calculate new search direction v = ravel(A * v) #Apply preconditioner v = ravel(M * v) # Check for nan, inf if any(isnan(v)) or any(isinf(v)): warn('inf or nan after application of preconditioner') return (postprocess(x), -1) # Factor in all Householder orthogonal reflections on new search direction for j in range(inner + 1): v = v - 2.0 * dot(conjugate(W[:, j]), v) * W[:, j] # Calculate next Householder reflector, w # w = v[inner+1:] + sign(v[inner+1])*||v[inner+1:]||_2*e_{inner+1) # Note that if maxiter = dimen, then this is unnecessary for the last inner # iteration, when inner = dimen-1. Here we do not need to calculate a # Householder reflector or Given's rotation because nnz(v) is already the # desired length, i.e. we do not need to zero anything out. if inner != dimen - 1: w = zeros((dimen, ), dtype=xtype) vslice = v[inner + 1:] alpha = linalg.norm(vslice) if alpha != 0: alpha = mysign(vslice[0]) * alpha # We do not need the final reflector for future calculations if inner < (maxiter - 1): w[inner + 1:] = vslice w[inner + 1] += alpha w = w / linalg.norm(w) W[:, inner + 1] = w # Apply new reflector to v # v = v - 2.0*w*(w.T*v) v[inner + 1] = -alpha v[inner + 2:] = 0.0 # Apply all previous Given's Rotations to v if inner == 0: # Q will store the cumulative effect of all Given's Rotations Q = scipy.sparse.eye(dimen, dimen, format='csr', dtype=xtype) # Declare initial Qj, which will be the current Given's Rotation rowptr = hstack((array([0, 2, 4], int), arange(5, dimen + 3, dtype=int))) colindices = hstack((array([0, 1, 0, 1], int), arange(2, dimen, dtype=int))) data = ones((dimen + 2, ), dtype=xtype) Qj = csr_matrix((data, colindices, rowptr), shape=(dimen, dimen), dtype=xtype) else: # Could avoid building a global Given's Rotation, by storing # and applying each 2x2 matrix individually. # But that would require looping, the bane of wonderful Python Q = Qj * Q v = Q * v # Calculate Qj, the next Given's rotation, where j = inner # Note that if maxiter = dimen, then this is unnecessary for the last inner # iteration, when inner = dimen-1. Here we do not need to calculate a # Householder reflector or Given's rotation because nnz(v) is already the # desired length, i.e. we do not need to zero anything out. if inner != dimen - 1: if v[inner + 1] != 0: # Calculate terms for complex 2x2 Given's Rotation # Note that abs(x) takes the complex modulus h1 = v[inner] h2 = v[inner + 1] h1_mag = abs(h1) h2_mag = abs(h2) if h1_mag < h2_mag: mu = h1 / h2 tau = conjugate(mu) / abs(mu) else: mu = h2 / h1 tau = mu / abs(mu) denom = sqrt(h1_mag**2 + h2_mag**2) c = h1_mag / denom s = h2_mag * tau / denom Qblock = array([[c, conjugate(s)], [-s, c]], dtype=xtype) # Modify Qj in csr-format so that it represents the current # global Given's Rotation equivalent to Qblock if inner != 0: Qj.data[inner - 1] = 1.0 Qj.indices[inner - 1] = inner - 1 Qj.indptr[inner - 1] = inner - 1 Qj.data[inner:inner + 4] = ravel(Qblock) Qj.indices[inner:inner + 4] = [inner, inner + 1, inner, inner + 1] Qj.indptr[inner:inner + 3] = [inner, inner + 2, inner + 4] # Apply Given's Rotation to g, # the RHS for the linear system in the Krylov Subspace. # Note that this dot does a matrix multiply, not an actual # dot product where a conjugate transpose is taken g[inner:inner + 2] = dot(Qblock, g[inner:inner + 2]) # Apply effect of Given's Rotation to v v[inner] = dot(Qblock[0, :], v[inner:inner + 2]) v[inner + 1] = 0.0 # Write to upper Hessenberg Matrix, # the LHS for the linear system in the Krylov Subspace H[:, inner] = v[0:maxiter] # Don't update normr if last inner iteration, because # normr is calculated directly after this loop ends. if inner < maxiter - 1: normr = abs(g[inner + 1]) if normr < tol: break # Allow user access to residual if callback != None: callback(normr) if keep_r: residuals.append(normr) niter += 1 # end inner loop, back to outer loop # Find best update to x in Krylov Space, V. Solve inner+1 x inner+1 system. # Apparently this is the best way to solve a triangular # system in the magical world of scipy piv = arange(inner + 1) y = lu_solve((H[0:(inner + 1), 0:(inner + 1)], piv), g[0:(inner + 1)], trans=0) # Use Horner like Scheme to map solution, y, back to original space. # Note that we do not use the last reflector. update = zeros(x.shape, dtype=xtype) for j in range(inner, -1, -1): update[j] += y[j] # Apply j-th reflector, (I - 2.0*w_j*w_j.T)*upadate update = update - 2.0 * dot(conjugate(W[:, j]), update) * W[:, j] x = x + update r = b - ravel(A * x) #Apply preconditioner r = ravel(M * r) normr = linalg.norm(r) # Check for nan, inf if any(isnan(r)) or any(isinf(r)): warn('inf or nan after application of preconditioner') return (postprocess(x), -1) # Allow user access to residual if callback != None: callback(normr) if keep_r: residuals.append(normr) # Has GMRES stagnated? indices = (x != 0) if indices.any(): change = max(abs(update[indices] / x[indices])) if change < 1e-12: # No change, halt return (postprocess(x), -1) # test for convergence if normr < tol: return (postprocess(x), 0) # end outer loop return (postprocess(x), niter)
def presid_gmres(A, b, verbose, x0=None, tol=1e-05, restart=None, maxiter=None, M=None): callback = gmres_counter(verbose) A, M, x, b, postprocess = make_system(A, M, x0, b) n = len(b) if maxiter is None: maxiter = n * 10 if restart is None: restart = 20 restart = min(restart, n) matvec = A.matvec psolve = M.matvec ltr = _type_conv[x.dtype.char] revcom = getattr(_iterative, ltr + 'gmresrevcom') bnrm2 = np.linalg.norm(b) Mb_nrm2 = np.linalg.norm(psolve(b)) get_residual = lambda: np.linalg.norm(matvec(x) - b) atol = tol if bnrm2 == 0: return postprocess(b), 0 # Tolerance passed to GMRESREVCOM applies to the inner iteration # and deals with the left-preconditioned residual. ptol_max_factor = 1.0 ptol = Mb_nrm2 * min(ptol_max_factor, atol / bnrm2) resid = np.nan presid = np.nan ndx1 = 1 ndx2 = -1 # Use _aligned_zeros to work around a f2py bug in Numpy 1.9.1 work = _aligned_zeros((6 + restart) * n, dtype=x.dtype) work2 = _aligned_zeros((restart + 1) * (2 * restart + 2), dtype=x.dtype) ijob = 1 info = 0 ftflag = True iter_ = maxiter old_ijob = ijob first_pass = True resid_ready = False iter_num = 1 while True: ### begin my modifications if presid / bnrm2 < atol: resid = presid / bnrm2 info = 1 if info: ptol = 10000 ### end my modifications x, iter_, presid, info, ndx1, ndx2, sclr1, sclr2, ijob = \ revcom(b, x, restart, work, work2, iter_, presid, info, ndx1, ndx2, ijob, ptol) slice1 = slice(ndx1 - 1, ndx1 - 1 + n) slice2 = slice(ndx2 - 1, ndx2 - 1 + n) if (ijob == -1): # gmres success, update last residual if resid_ready and callback is not None: callback(presid / bnrm2) resid_ready = False break elif (ijob == 1): work[slice2] *= sclr2 work[slice2] += sclr1 * matvec(x) elif (ijob == 2): work[slice1] = psolve(work[slice2]) if not first_pass and old_ijob == 3: resid_ready = True first_pass = False elif (ijob == 3): work[slice2] *= sclr2 work[slice2] += sclr1 * matvec(work[slice1]) if resid_ready and callback is not None: callback(presid / bnrm2) resid_ready = False iter_num = iter_num + 1 elif (ijob == 4): if ftflag: info = -1 ftflag = False resid, info = _stoptest(work[slice1], atol) # Inner loop tolerance control if info or presid > ptol: ptol_max_factor = min(1.0, 1.5 * ptol_max_factor) else: # Inner loop tolerance OK, but outer loop not. ptol_max_factor = max(1e-16, 0.25 * ptol_max_factor) if resid != 0: ptol = presid * min(ptol_max_factor, atol / resid) else: ptol = presid * ptol_max_factor old_ijob = ijob ijob = 2 if iter_num > maxiter: info = maxiter break if info >= 0 and not (resid <= atol): # info isn't set appropriately otherwise info = maxiter return postprocess(x), info, mydict['resnorms']
def idrs(A, b, x0=None, tol=1e-5, norm=np.linalg.norm, s=4, maxiter=None, callback=None, init_seed=None): """ References ---------- .. [1] P. Sonneveld and M. B. van Gijzen SIAM J. Sci. Comput. Vol. 31, No. 2, pp. 1035--1062, (2008). .. [2] M. B. van Gijzen and P. Sonneveld ACM Trans. Math. Software,, Vol. 38, No. 1, pp. 5:1-5:19, (2011). """ dtype = A.dtype A, _, x, b, postprocess = make_system(A, None, x0, b) n = len(b) if maxiter is None: maxiter = n * 10 if callback is not None: callback(x) r = b - A.dot(x) np.random.seed(init_seed) P = np.random.randn(n, s) P[:, 0] = r # For comparison with BiCGStab P = orth(P).T rnrm = norm(r) # Relative tolerance tolr = tol * norm(b) if rnrm < tolr: # Initial guess is a good enough solution return postprocess(x) # Initialization dR = np.zeros((n, s), dtype=dtype) dX = np.zeros_like(dR) M = np.zeros((s, s), dtype=dtype) for i in range(s): v = A.dot(r) om = np.dot(v, r) / np.dot(v, v) dX[:, i] = om * r dR[:, i] = -om * v x += dX[:, i] if callback is not None: callback(x) r += dR[:, i] M[:, i] = P.dot(dR[:, i]) iter_ = s current = 0 m = P.dot(r) while rnrm > tolr and iter_ < maxiter: for k in range(s + 1): c = np.linalg.solve(M, m) q = -dR.dot(c) v = r + q if k == 0: t = A.dot(v) om = np.dot(t, v) / np.dot(v, v) dR[:, current] = q - np.dot(om, t) dX[:, current] = -dX.dot(c) + om * v else: dX[:, current] = -dX.dot(c) + om * v dR[:, current] = -A.dot(dX[:, current]) x += dX[:, current] if callback is not None: callback(x) r += dR[:, current] iter_ += 1 dm = P.dot(dR[:, current]) M[:, current] = dm m += dm current = (current + 1) % s rnrm = norm(r) return postprocess(x)[:, None]
def minres(A, b, x0=None, shift=0.0, tol=1e-5, maxiter=None, M=None, callback=None, show=False, check=False): A, M, x, b, postprocess = make_system(A, M, x0, b) matvec = A.matvec psolve = M.matvec first = 'Enter minres. ' last = 'Exit minres. ' n = A.shape[0] if maxiter is None: maxiter = 5 * n msg = [ ' beta2 = 0. If M = I, b and x are eigenvectors ', # -1 ' beta1 = 0. The exact solution is x = 0 ', # 0 ' A solution to Ax = b was found, given rtol ', # 1 ' A least-squares solution was found, given rtol ', # 2 ' Reasonable accuracy achieved, given eps ', # 3 ' x has converged to an eigenvector ', # 4 ' acond has exceeded 0.1/eps ', # 5 ' The iteration limit was reached ', # 6 ' A does not define a symmetric matrix ', # 7 ' M does not define a symmetric matrix ', # 8 ' M does not define a pos-def preconditioner ' ] # 9 if show: print(first + 'Solution of symmetric Ax = b') print(first + 'n = %3g shift = %23.14e' % (n, shift)) print(first + 'itnlim = %3g rtol = %11.2e' % (maxiter, tol)) print() istop = 0 itn = 0 Anorm = 0 Acond = 0 rnorm = 0 ynorm = 0 xtype = x.dtype eps = finfo(xtype).eps x = zeros(n, dtype=xtype) # Set up y and v for the first Lanczos vector v1. # y = beta1 P' v1, where P = C**(-1). # v is really P' v1. y = b r1 = b y = psolve(b) beta1 = inner(b, y) if beta1 < 0: raise ValueError('indefinite preconditioner') elif beta1 == 0: return (postprocess(x), 0) beta1 = sqrt(beta1) if check: # are these too strict? # see if A is symmetric w = matvec(y) r2 = matvec(w) s = inner(w, w) t = inner(y, r2) z = abs(s - t) epsa = (s + eps) * eps**(1.0 / 3.0) if z > epsa: raise ValueError('non-symmetric matrix') # see if M is symmetric r2 = psolve(y) s = inner(y, y) t = inner(r1, r2) z = abs(s - t) epsa = (s + eps) * eps**(1.0 / 3.0) if z > epsa: raise ValueError('non-symmetric preconditioner') # Initialize other quantities oldb = 0 beta = beta1 dbar = 0 epsln = 0 qrnorm = beta1 phibar = beta1 rhs1 = beta1 rhs2 = 0 tnorm2 = 0 ynorm2 = 0 cs = -1 sn = 0 w = zeros(n, dtype=xtype) w2 = zeros(n, dtype=xtype) r2 = r1 if show: print() print() print( ' Itn x(1) Compatible LS norm(A) cond(A) gbar/|A|' ) while itn < maxiter: itn += 1 s = 1.0 / beta v = s * y y = matvec(v) y = y - shift * v if itn >= 2: y = y - (beta / oldb) * r1 alfa = inner(v, y) y = y - (alfa / beta) * r2 r1 = r2 r2 = y y = psolve(r2) oldb = beta beta = inner(r2, y) if beta < 0: raise ValueError('non-symmetric matrix') beta = sqrt(beta) tnorm2 += alfa**2 + oldb**2 + beta**2 if itn == 1: if beta / beta1 <= 10 * eps: istop = -1 # Terminate later # tnorm2 = alfa**2 ?? gmax = abs(alfa) gmin = gmax # Apply previous rotation Qk-1 to get # [deltak epslnk+1] = [cs sn][dbark 0 ] # [gbar k dbar k+1] [sn -cs][alfak betak+1]. oldeps = epsln delta = cs * dbar + sn * alfa # delta1 = 0 deltak gbar = sn * dbar - cs * alfa # gbar 1 = alfa1 gbar k epsln = sn * beta # epsln2 = 0 epslnk+1 dbar = -cs * beta # dbar 2 = beta2 dbar k+1 root = norm([gbar, dbar]) Arnorm = phibar * root # Compute the next plane rotation Qk gamma = norm([gbar, beta]) # gammak gamma = max(gamma, eps) cs = gbar / gamma # ck sn = beta / gamma # sk phi = cs * phibar # phik phibar = sn * phibar # phibark+1 # Update x. denom = 1.0 / gamma w1 = w2 w2 = w w = (v - oldeps * w1 - delta * w2) * denom x = x + phi * w # Go round again. gmax = max(gmax, gamma) gmin = min(gmin, gamma) z = rhs1 / gamma ynorm2 = z**2 + ynorm2 rhs1 = rhs2 - delta * z rhs2 = -epsln * z # Estimate various norms and test for convergence. Anorm = sqrt(tnorm2) ynorm = sqrt(ynorm2) epsa = Anorm * eps epsx = Anorm * ynorm * eps epsr = Anorm * ynorm * tol diag = gbar if diag == 0: diag = epsa qrnorm = phibar rnorm = qrnorm test1 = rnorm / (Anorm * ynorm) # ||r|| / (||A|| ||x||) test2 = root / Anorm # ||Ar|| / (||A|| ||r||) # Estimate cond(A). # In this version we look at the diagonals of R in the # factorization of the lower Hessenberg matrix, Q * H = R, # where H is the tridiagonal matrix from Lanczos with one # extra row, beta(k+1) e_k^T. Acond = gmax / (gmin + eps) # See if any of the stopping criteria are satisfied. # In rare cases, istop is already -1 from above (Abar = const*I). if istop == 0: t1 = 1 + test1 # These tests work if tol < eps t2 = 1 + test2 if t2 <= 1: istop = 2 if t1 <= 1: istop = 1 if itn >= maxiter: istop = 6 if Acond >= 0.1 / eps: istop = 4 if epsx >= beta: istop = 3 # if rnorm <= epsx : istop = 2 # if rnorm <= epsr : istop = 1 if test2 <= tol: istop = 2 if test1 <= tol: istop = 1 # See if it is time to print something. prnt = False if n <= 40: prnt = True if itn <= 10: prnt = True if itn >= maxiter - 10: prnt = True if itn % 10 == 0: prnt = True if qrnorm <= 10 * epsx: prnt = True if qrnorm <= 10 * epsr: prnt = True if Acond <= 1e-2 / eps: prnt = True if istop != 0: prnt = True if show and prnt: str1 = '%6g %12.5e %10.3e' % (itn, x[0], test1) str2 = ' %10.3e' % (test2, ) str3 = ' %8.1e %8.1e %8.1e' % (Anorm, Acond, gbar / Anorm) print(str1 + str2 + str3) if itn % 10 == 0: print() if callback is not None: callback(x) if istop in [1, 2]: pass #break # TODO check this if show: print() print(last + ' istop = %3g itn =%5g' % (istop, itn)) print(last + ' Anorm = %12.4e Acond = %12.4e' % (Anorm, Acond)) print(last + ' rnorm = %12.4e ynorm = %12.4e' % (rnorm, ynorm)) print(last + ' Arnorm = %12.4e' % (Arnorm, )) print(last + msg[istop + 1]) if istop == 6: info = maxiter else: info = 0 return (postprocess(x), info)
def minimal_residual(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None, callback=None, residuals=None): '''Minimal residual (MR) algorithm Solves the linear system Ax = b. Left preconditioning is supported. Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by the preconditioner norm of r_0, or ||r_0||_M. maxiter : int maximum number of allowed iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A x = M b. callback : function User-supplied function is called after each iteration as callback(xk), where xk is the current solution vector residuals : list residuals contains the residual norm history, including the initial residual. The preconditioner norm is used, instead of the Euclidean norm. Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of cg == ======================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. <0 numerical breakdown, or illegal input == ======================================= Notes ----- The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. The residual in the preconditioner norm is both used for halting and returned in the residuals list. Examples -------- >>> from pyamg.krylov import minimal_residual >>> from pyamg.util.linalg import norm >>> import numpy >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = numpy.ones((A.shape[0],)) >>> (x,flag) = minimal_residual(A,b, maxiter=2, tol=1e-8) >>> print norm(b - A*x) 7.26369350856 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 137--142, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' A,M,x,b,postprocess = make_system(A,M,x0,b,xtype=None) n = len(b) ## # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._minimal_residual') # determine maxiter if maxiter is None: maxiter = int(len(b)) elif maxiter < 1: raise ValueError('Number of iterations must be positive') # setup method r = M*(b - A*x) normr = norm(r) # store initial residual if residuals is not None: residuals[:] = [normr] # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol*normb: return (postprocess(x), 0) # Scale tol by ||r_0||_M if normr != 0.0: tol = tol*normr # How often should r be recomputed recompute_r = 50 iter = 0 while True: iter = iter+1 p = M*(A*r) rMAr = inner(p.conjugate(), r) # check curvature of M^-1 A if rMAr < 0.0: warn("\nIndefinite matrix detected in minimal residual, aborting\n") return (postprocess(x), -1) alpha = rMAr / inner(p.conjugate(), p) x = x + alpha*r if mod(iter, recompute_r) and iter > 0: r = M*(b - A*x) else: r = r - alpha*p normr = norm(r) if residuals is not None: residuals.append(normr) if callback is not None: callback(x) if normr < tol: return (postprocess(x), 0) if iter == maxiter: return (postprocess(x), iter)
def fgmres(A, b, x0=None, tol=1e-5, restrt=None, maxiter=None, xtype=None, M=None, callback=None, residuals=None): '''Flexible Generalized Minimum Residual Method (fGMRES) fGMRES iteratively refines the initial solution guess to the system Ax = b. fGMRES is flexible in the sense that the right preconditioner (M) can vary from iteration to iteration. Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by ||r_0||_2 restrt : {None, int} - if int, restrt is max number of inner iterations and maxiter is the max number of outer iterations - if None, do not restart GMRES, and max number of inner iterations is maxiter maxiter : {None, int} - if restrt is None, maxiter is the max number of inner iterations and GMRES does not restart - if restrt is int, maxiter is the max number of outer iterations, and restrt is the max number of inner iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve A M x = M b. M need not be stationary for fgmres callback : function User-supplied function is called after each iteration as callback( ||rk||_2 ), where rk is the current residual vector residuals : list residuals has the residual norm history, including the initial residual, appended to it Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of gmres == ============================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. This value is precisely the order of the Krylov space. <0 numerical breakdown, or illegal input == ============================================= Notes ----- - The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. - fGMRES allows for non-stationary preconditioners, as opposed to GMRES - For robustness, Householder reflections are used to orthonormalize the Krylov Space Givens Rotations are used to provide the residual norm each iteration Flexibility implies that the right preconditioner, M or A.psolve, can vary from iteration to iteration Examples -------- >>> from pyamg.krylov.fgmres import fgmres >>> from pyamg.util.linalg import norm >>> import numpy as np >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = np.ones((A.shape[0],)) >>> (x,flag) = fgmres(A,b, maxiter=2, tol=1e-8) >>> print norm(b - A*x) 6.5428213057 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' # Convert inputs to linear system, with error checking A, M, x, b, postprocess = make_system(A, M, x0, b, xtype) dimen = A.shape[0] # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._fgmres') # Choose type if not hasattr(A, 'dtype'): Atype = upcast(x.dtype, b.dtype) else: Atype = A.dtype if not hasattr(M, 'dtype'): Mtype = upcast(x.dtype, b.dtype) else: Mtype = M.dtype xtype = upcast(Atype, x.dtype, b.dtype, Mtype) # Should norm(r) be kept if residuals == []: keep_r = True else: keep_r = False # Set number of outer and inner iterations if restrt: if maxiter: max_outer = maxiter else: max_outer = 1 if restrt > dimen: warn('Setting number of inner iterations (restrt) to maximum \ allowed, which is A.shape[0] ') restrt = dimen max_inner = restrt else: max_outer = 1 if maxiter > dimen: warn('Setting number of inner iterations (maxiter) to maximum \ allowed, which is A.shape[0] ') maxiter = dimen elif maxiter is None: maxiter = min(dimen, 40) max_inner = maxiter # Get fast access to underlying BLAS routines [rotg] = get_blas_funcs(['rotg'], [x]) # Is this a one dimensional matrix? if dimen == 1: entry = ravel(A*array([1.0], dtype=xtype)) return (postprocess(b/entry), 0) # Prep for method r = b - ravel(A*x) normr = norm(r) if keep_r: residuals.append(normr) # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol*normb: if callback is not None: callback(norm(r)) return (postprocess(x), 0) # Scale tol by ||r_0||_2, we don't use the preconditioned # residual because this is right preconditioned GMRES. if normr != 0.0: tol = tol*normr # Use separate variable to track iterations. If convergence fails, # we cannot simply report niter = (outer-1)*max_outer + inner. Numerical # error could cause the inner loop to halt while the actual ||r|| > tol. niter = 0 # Begin fGMRES for outer in range(max_outer): # Calculate vector w, which defines the Householder reflector # Take shortcut in calculating, # w = r + sign(r[1])*||r||_2*e_1 w = r beta = mysign(w[0])*normr w[0] += beta w /= norm(w) # Preallocate for Krylov vectors, Householder reflectors and Hessenberg # matrix # Space required is O(dimen*max_inner) # Givens Rotations Q = zeros((4*max_inner,), dtype=xtype) # upper Hessenberg matrix (made upper tri with Givens Rotations) H = zeros((max_inner, max_inner), dtype=xtype) W = zeros((max_inner, dimen), dtype=xtype) # Householder reflectors # For fGMRES, preconditioned vectors must be stored # No Horner-like scheme exists that allow us to avoid this Z = zeros((dimen, max_inner), dtype=xtype) W[0, :] = w # Multiply r with (I - 2*w*w.T), i.e. apply the Householder reflector # This is the RHS vector for the problem in the Krylov Space g = zeros((dimen,), dtype=xtype) g[0] = -beta for inner in range(max_inner): # Calculate Krylov vector in two steps # (1) Calculate v = P_j = (I - 2*w*w.T)v, where k = inner v = -2.0*conjugate(w[inner])*w v[inner] += 1.0 # (2) Calculate the rest, v = P_1*P_2*P_3...P_{j-1}*ej. # for j in range(inner-1,-1,-1): # v = v - 2.0*dot(conjugate(W[j,:]), v)*W[j,:] amg_core.apply_householders(v, ravel(W), dimen, inner-1, -1, -1) # Apply preconditioner v = ravel(M*v) # Check for nan, inf # if isnan(v).any() or isinf(v).any(): # warn('inf or nan after application of preconditioner') # return(postprocess(x), -1) Z[:, inner] = v # Calculate new search direction v = ravel(A*v) # Factor in all Householder orthogonal reflections on new search # direction # for j in range(inner+1): # v = v - 2.0*dot(conjugate(W[j,:]), v)*W[j,:] amg_core.apply_householders(v, ravel(W), dimen, 0, inner+1, 1) # Calculate next Householder reflector, w # w = v[inner+1:] + sign(v[inner+1])*||v[inner+1:]||_2*e_{inner+1) # Note that if max_inner = dimen, then this is unnecessary for # the last inner iteration, when inner = dimen-1. Here we do # not need to calculate a Householder reflector or Givens # rotation because nnz(v) is already the desired length, # i.e. we do not need to zero anything out. if inner != dimen-1: if inner < (max_inner-1): w = W[inner+1, :] vslice = v[inner+1:] alpha = norm(vslice) if alpha != 0: alpha = mysign(vslice[0])*alpha # do not need the final reflector for future calculations if inner < (max_inner-1): w[inner+1:] = vslice w[inner+1] += alpha w /= norm(w) # Apply new reflector to v # v = v - 2.0*w*(w.T*v) v[inner+1] = -alpha v[inner+2:] = 0.0 if inner > 0: # Apply all previous Givens Rotations to v amg_core.apply_givens(Q, v, dimen, inner) # Calculate the next Givens rotation, where j = inner Note that if # max_inner = dimen, then this is unnecessary for the last inner # iteration, when inner = dimen-1. Here we do not need to # calculate a Householder reflector or Givens rotation because # nnz(v) is already the desired length, i.e. we do not need to zero # anything out. if inner != dimen-1: if v[inner+1] != 0: [c, s] = rotg(v[inner], v[inner+1]) Qblock = array([[c, s], [-conjugate(s), c]], dtype=xtype) Q[(inner*4): ((inner+1)*4)] = ravel(Qblock).copy() # Apply Givens Rotation to g, the RHS for the linear system # in the Krylov Subspace. Note that this dot does a matrix # multiply, not an actual dot product where a conjugate # transpose is taken g[inner:inner+2] = dot(Qblock, g[inner:inner+2]) # Apply effect of Givens Rotation to v v[inner] = dot(Qblock[0, :], v[inner:inner+2]) v[inner+1] = 0.0 # Write to upper Hessenberg Matrix, # the LHS for the linear system in the Krylov Subspace H[:, inner] = v[0:max_inner] # Don't update normr if last inner iteration, because # normr is calculated directly after this loop ends. if inner < max_inner-1: normr = abs(g[inner+1]) if normr < tol: break # Allow user access to residual if callback is not None: callback(normr) if keep_r: residuals.append(normr) niter += 1 # end inner loop, back to outer loop # Find best update to x in Krylov Space, V. Solve inner+1 x inner+1 # system. Apparently this is the best way to solve a triangular system # in the magical world of scipy # piv = arange(inner+1) # y = lu_solve((H[0:(inner+1),0:(inner+1)], piv), # g[0:(inner+1)], trans=0) y = sp.linalg.solve(H[0:(inner+1), 0:(inner+1)], g[0:(inner+1)]) # No Horner like scheme exists because the preconditioner can change # each iteration # Hence, we must store each preconditioned vector update = dot(Z[:, 0:inner+1], y) x = x + update r = b - ravel(A*x) normr = norm(r) # Allow user access to residual if callback is not None: callback(normr) if keep_r: residuals.append(normr) # Has fGMRES stagnated? indices = (x != 0) if indices.any(): change = max(abs(update[indices] / x[indices])) if change < 1e-12: # No change, halt return (postprocess(x), -1) # test for convergence if normr < tol: return (postprocess(x), 0) # end outer loop return (postprocess(x), niter)
def cg(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None, callback=None, residuals=None): '''Conjugate Gradient algorithm Solves the linear system Ax = b. Left preconditioning is supported. Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by ||b|| maxiter : int maximum number of allowed iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A A.H x = b. callback : function User-supplied funtion is called after each iteration as callback(xk), where xk is the current solution vector residuals : list residuals has the residual norm history, including the initial residual, appended to it Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of cg == ======================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. <0 numerical breakdown, or illegal input == ======================================= Notes ----- The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. Examples -------- >>> from pyamg.krylov.cg import cg >>> from pyamg.util.linalg import norm >>> import numpy >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = numpy.ones((A.shape[0],)) >>> (x,flag) = cg(A,b, maxiter=2, tol=1e-8) >>> print norm(b - A*x) 10.9370700187 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 262-67, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' A,M,x,b,postprocess = make_system(A,M,x0,b,xtype=None) n = len(b) # Determine maxiter if maxiter is None: maxiter = int(1.3*len(b)) + 2 elif maxiter < 1: raise ValueError('Number of iterations must be positive') # Scale tol by normb normb = norm(b) if normb != 0: tol = tol*normb # setup method r = b - A*x z = M*r p = z.copy() rz = inner(conjugate(r), z) normr = norm(r) if residuals is not None: residuals[:] = [normr] #initial residual if normr < tol: return (postprocess(x), 0) iter = 0 while True: Ap = A*p rz_old = rz alpha = rz/inner(conjugate(Ap), p) # 3 (step # in Saad's pseudocode) x += alpha * p # 4 r -= alpha * Ap # 5 z = M*r # 6 rz = inner(conjugate(r), z) beta = rz/rz_old # 7 p *= beta # 8 p += z iter += 1 normr = norm(r) if residuals is not None: residuals.append(normr) if callback is not None: callback(x) if normr < tol: return (postprocess(x), 0) if iter == maxiter: return (postprocess(x), iter)
def cgne(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None, callback=None, residuals=None): '''Conjugate Gradient, Normal Error algorithm Applies CG to the normal equations, A.H A x = b. Left preconditioning is supported. Note that unless A is well-conditioned, the use of CGNE is inadvisable Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by ||r_0||_2 maxiter : int maximum number of allowed iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A A.H x = M b. callback : function User-supplied function is called after each iteration as callback(xk), where xk is the current solution vector residuals : list residuals has the residual norm history, including the initial residual, appended to it Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of cgne == ======================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. <0 numerical breakdown, or illegal input == ======================================= Notes ----- - The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. Examples -------- >>> from pyamg.krylov.cgne import cgne >>> from pyamg.util.linalg import norm >>> import numpy as np >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = np.ones((A.shape[0],)) >>> (x,flag) = cgne(A,b, maxiter=2, tol=1e-8) >>> print norm(b - A*x) 46.1547104367 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 276-7, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' # Store the conjugate transpose explicitly as it will be used much later on if isspmatrix(A): AH = A.H else: # TODO avoid doing this since A may be a different sparse type AH = aslinearoperator(np.asmatrix(A).H) # Convert inputs to linear system, with error checking A, M, x, b, postprocess = make_system(A, M, x0, b, xtype) dimen = A.shape[0] # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._cgne') # Choose type if not hasattr(A, 'dtype'): Atype = upcast(x.dtype, b.dtype) else: Atype = A.dtype if not hasattr(M, 'dtype'): Mtype = upcast(x.dtype, b.dtype) else: Mtype = M.dtype xtype = upcast(Atype, x.dtype, b.dtype, Mtype) # Should norm(r) be kept if residuals == []: keep_r = True else: keep_r = False # How often should r be recomputed recompute_r = 8 # Check iteration numbers. CGNE suffers from loss of orthogonality quite # easily, so we arbitrarily let the method go up to 130% over the # theoretically necessary limit of maxiter=dimen if maxiter is None: maxiter = int(np.ceil(1.3 * dimen)) + 2 elif maxiter < 1: raise ValueError('Number of iterations must be positive') elif maxiter > (1.3 * dimen): warn('maximum allowed inner iterations (maxiter) are the 130% times \ the number of dofs') maxiter = int(np.ceil(1.3 * dimen)) + 2 # Prep for method r = b - A * x normr = norm(r) if keep_r: residuals.append(normr) # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol * normb: if callback is not None: callback(x) return (postprocess(x), 0) # Scale tol by ||r_0||_2 if normr != 0.0: tol = tol * normr # Begin CGNE # Apply preconditioner and calculate initial search direction z = M * r p = AH * z old_zr = np.inner(z.conjugate(), r) for iter in range(maxiter): # alpha = (z_j, r_j) / (p_j, p_j) alpha = old_zr / np.inner(p.conjugate(), p) # x_{j+1} = x_j + alpha*p_j x += alpha * p # r_{j+1} = r_j - alpha*w_j, where w_j = A*p_j if np.mod(iter, recompute_r) and iter > 0: r -= alpha * (A * p) else: r = b - A * x # z_{j+1} = M*r_{j+1} z = M * r # beta = (z_{j+1}, r_{j+1}) / (z_j, r_j) new_zr = np.inner(z.conjugate(), r) beta = new_zr / old_zr old_zr = new_zr # p_{j+1} = A.H*z_{j+1} + beta*p_j p *= beta p += AH * z # Allow user access to residual if callback is not None: callback(x) # test for convergence normr = norm(r) if keep_r: residuals.append(normr) if normr < tol: return (postprocess(x), 0) # end loop return (postprocess(x), iter + 1)
def idrs_biortho(A, b, x0=None, tol=1e-5, norm=np.linalg.norm, s=4, maxiter=None, callback=None, init_seed=None): """ References ---------- .. [1] P. Sonneveld and M. B. van Gijzen SIAM J. Sci. Comput. Vol. 31, No. 2, pp. 1035--1062, (2008). .. [2] M. B. van Gijzen and P. Sonneveld ACM Trans. Math. Software,, Vol. 38, No. 1, pp. 5:1-5:19, (2011). """ dtype = A.dtype A, B, x, b, postprocess = make_system(A, None, x0, b) n = len(b) if maxiter is None: maxiter = n * 10 if callback is not None: callback(x) r = b - A.dot(x) rnrm = norm(r) np.random.seed(init_seed) P = np.random.randn(n, s) P[:, 0] = r # For comparison with BiCdRStab P = orth(P).T if callback is not None: callback(x) tolr = tol * norm(b) if rnrm < tolr: return postprocess(x) dR = np.zeros((n, s), dtype=dtype) dX = np.zeros((n, s), dtype=dtype) M = np.eye(s, dtype=dtype) om = 1.0 iter_ = 0 while rnrm >= tolr and iter_ < maxiter: m = P.dot(r) for k in range(0, s): c = np.linalg.solve(M[k:s, k:s], m[k:s]) v = r - dR[:, k:s].dot(c) v = B.dot(v) dX[:, k] = dX[:, k:s].dot(c) + om * v dR[:, k] = A.dot(dX[:, k]) for i in range(0, k): alpha = np.dot(P[i, :], dR[:, k]) / M[i, i] dR[:, k] = dR[:, k] - alpha * dR[:, i] dX[:, k] = dX[:, k] - alpha * dX[:, i] M[k:s, k] = np.dot(P[k:s, :], dR[:, k]) beta = m[k] / M[k, k] r -= beta * dR[:, k] rnrm = np.linalg.norm(r) x += beta * dX[:, k] if callback is not None: callback(x) iter_ += 1 if k + 1 < s: m[k + 1:s] -= beta * M[k + 1:s, k] if rnrm < tolr or iter_ >= maxiter: break v = B.dot(r) t = A.dot(v) om = np.dot(t, r) / np.dot(t, t) x += om * v if callback is not None: callback(x) r -= om * t rnrm = norm(r) iter_ += 1 return postprocess(x)[:, None]
def cr(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None, callback=None, residuals=None): '''Conjugate Residual algorithm Solves the linear system Ax = b. Left preconditioning is supported. The matrix A must be Hermitian symmetric (but not necessarily definite). Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by the preconditioner norm of r_0, or ||r_0||_M. maxiter : int maximum number of allowed iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A x = M b. callback : function User-supplied function is called after each iteration as callback(xk), where xk is the current solution vector residuals : list residuals contains the residual norm history, including the initial residual. The preconditioner norm is used, instead of the Euclidean norm. Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of cr == ======================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. <0 numerical breakdown, or illegal input == ======================================= Notes ----- The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. The 2-norm of the preconditioned residual is used both for halting and returned in the residuals list. Examples -------- >>> from pyamg.krylov.cr import cr >>> from pyamg.util.linalg import norm >>> import numpy as np >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = np.ones((A.shape[0],)) >>> (x,flag) = cr(A,b, maxiter=2, tol=1e-8) >>> print norm(b - A*x) 10.9370700187 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 262-67, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' A, M, x, b, postprocess = make_system(A, M, x0, b, xtype=None) # n = len(b) # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._cr') # determine maxiter if maxiter is None: maxiter = int(1.3*len(b)) + 2 elif maxiter < 1: raise ValueError('Number of iterations must be positive') # choose tolerance for numerically zero values # t = A.dtype.char # eps = np.finfo(np.float).eps # feps = np.finfo(np.single).eps # geps = np.finfo(np.longfloat).eps # _array_precision = {'f': 0, 'd': 1, 'g': 2, 'F': 0, 'D': 1, 'G': 2} # numerically_zero = {0: feps*1e3, 1: eps*1e6, # 2: geps*1e6}[_array_precision[t]] # setup method r = b - A*x z = M*r p = z.copy() zz = inner(z.conjugate(), z) # use preconditioner norm normr = sqrt(zz) if residuals is not None: residuals[:] = [normr] # initial residual # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol*normb: return (postprocess(x), 0) # Scale tol by ||r_0||_M if normr != 0.0: tol = tol*normr # How often should r be recomputed recompute_r = 8 iter = 0 Az = A*z rAz = inner(r.conjugate(), Az) Ap = A*p while True: rAz_old = rAz alpha = rAz / inner(Ap.conjugate(), Ap) # 3 x += alpha * p # 4 if mod(iter, recompute_r) and iter > 0: # 5 r -= alpha * Ap else: r = b - A*x z = M*r Az = A*z rAz = inner(r.conjugate(), Az) beta = rAz/rAz_old # 6 p *= beta # 7 p += z Ap *= beta # 8 Ap += Az iter += 1 zz = inner(z.conjugate(), z) normr = sqrt(zz) # use preconditioner norm if residuals is not None: residuals.append(normr) if callback is not None: callback(x) if normr < tol: return (postprocess(x), 0) elif zz == 0.0: # important to test after testing normr < tol. rz == 0.0 is an # indicator of convergence when r = 0.0 warn("\nSingular preconditioner detected in CR, ceasing \ iterations\n") return (postprocess(x), -1) if iter == maxiter: return (postprocess(x), iter)
def msGmres(A, b, shifts, m=200, x0=None, tol=1e-5, maxiter=None, M=None, callback=None): """ Multi-Shift Gmres(m) """ A, M, x, b, postprocess = make_system(A, M, x0, b, xtype=None) n = len(b) if maxiter is None: maxiter = n * 10 matvec = A.matvec psolve = M.matvec xtype = x.dtype # Initialization # BLAS functions axpy, dot, norm, rotmat = get_blas_funcs(['axpy', 'dot', 'nrm2', 'rotg'], dtype=xtype) # Relative tolerance tolb = tol * norm(b) # Compute initial residual: r = b - matvec(x) rnrm2 = norm(r) nshifts = len(shifts) X = np.zeros((n, nshifts), dtype=xtype) for j in xrange(0, nshifts): X[:, j] = np.array(x) errors = np.zeros(nshifts, dtype=xtype) V = np.zeros((n, m + 1), dtype=xtype) H = np.zeros((m + 1, m), dtype=xtype) # Compute residuals converged = True for j in xrange(0, nshifts): rj = r + shifts[j] * X[:, j] errors[j] = norm(rj) converged = converged and (errors[j] < tolb) if callback is not None: callback(errors) if converged: return V, H # Begin iteration # for _iter in range(0, maxit): V[:, 0] = r / rnrm2 for i in xrange(0, m): # Construct orthonormal basis using Gram-Schmidt w = psolve(matvec(V[:, i])) for k in range(0, i + 1): H[k, i] = dot(V[:, k], w) axpy(V[:, k], w, None, -H[k, i]) H[i + 1, i] = norm(w) V[:, i + 1] = w / H[i + 1, i] # Multi-Shift part for j in xrange(0, nshifts): # Hj = H - w_j I Hj = np.array(H) for k in xrange(0, m): Hj[k, k] -= shifts[j] # s = |r_j| e_1 s = np.zeros(m + 1, dtype=xtype) s[0] = errors[j] # Apply Givens rotation to H_j[:m+1, :m] and s # Solve Triangular system with scipy.linalg.solve_triangular # Update X # Residuals? ¯\_(ツ)_/¯ return V, H
def cr(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None, callback=None, residuals=None): '''Conjugate Residual algorithm Solves the linear system Ax = b. Left preconditioning is supported. The matrix A must be Hermitian symmetric (but not necessarily definite). Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by the preconditioner norm of r_0, or ||r_0||_M. maxiter : int maximum number of allowed iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A x = M b. callback : function User-supplied function is called after each iteration as callback(xk), where xk is the current solution vector residuals : list residuals contains the residual norm history, including the initial residual. The preconditioner norm is used, instead of the Euclidean norm. Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of cr == ======================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. <0 numerical breakdown, or illegal input == ======================================= Notes ----- The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. The 2-norm of the preconditioned residual is used both for halting and returned in the residuals list. Examples -------- >>> from pyamg.krylov.cr import cr >>> from pyamg.util.linalg import norm >>> import numpy >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = numpy.ones((A.shape[0],)) >>> (x,flag) = cr(A,b, maxiter=2, tol=1e-8) >>> print norm(b - A*x) 10.9370700187 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 262-67, 2003 http://www-users.cs.umn.edu/~saad/books.html ''' A, M, x, b, postprocess = make_system(A, M, x0, b, xtype=None) n = len(b) ## # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._cr') # determine maxiter if maxiter is None: maxiter = int(1.3*len(b)) + 2 elif maxiter < 1: raise ValueError('Number of iterations must be positive') # choose tolerance for numerically zero values t = A.dtype.char eps = numpy.finfo(numpy.float).eps feps = numpy.finfo(numpy.single).eps geps = numpy.finfo(numpy.longfloat).eps _array_precision = {'f': 0, 'd': 1, 'g': 2, 'F': 0, 'D': 1, 'G': 2} numerically_zero = {0: feps*1e3, 1: eps*1e6, 2: geps*1e6}[_array_precision[t]] # setup method r = b - A*x z = M*r p = z.copy() zz = inner(z.conjugate(), z) # use preconditioner norm normr = sqrt(zz) if residuals is not None: residuals[:] = [normr] # initial residual # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol*normb: return (postprocess(x), 0) # Scale tol by ||r_0||_M if normr != 0.0: tol = tol*normr # How often should r be recomputed recompute_r = 8 iter = 0 Az = A*z rAz = inner(r.conjugate(), Az) Ap = A*p while True: rAz_old = rAz alpha = rAz / inner(Ap.conjugate(), Ap) # 3 x += alpha * p # 4 if mod(iter, recompute_r) and iter > 0: # 5 r -= alpha * Ap else: r = b - A*x z = M*r Az = A*z rAz = inner(r.conjugate(), Az) beta = rAz/rAz_old # 6 p *= beta # 7 p += z Ap *= beta # 8 Ap += Az iter += 1 zz = inner(z.conjugate(), z) normr = sqrt(zz) # use preconditioner norm if residuals is not None: residuals.append(normr) if callback is not None: callback(x) if normr < tol: return (postprocess(x), 0) elif zz == 0.0: # important to test after testing normr < tol. rz == 0.0 is an # indicator of convergence when r = 0.0 warn("\nSingular preconditioner detected in CR, ceasing \ iterations\n") return (postprocess(x), -1) if iter == maxiter: return (postprocess(x), iter)
def idrs(A, b, x0=None, tol=1e-5, s=4, maxiter=None, xtype=None, M=None, callback=None): """ Use Induced Dimension Reduction method [IDR(s)] to solve A x = b Parameters ---------- A : {sparse matrix, dense matrix, LinearOperator} The real or complex N-by-N matrix of the linear system. b : {array, matrix} Right hand side of the linear system. Has shape (N,) or (N,1). x0 : {array, matrix} Starting guess for the solution. tol : float, optional Tolerance to achieve. The algorithm terminates when either the relative or the absolute residual is below `tol`. s : integer, optional specifies the dimension of the shadow space. Normally, a higher s gives faster convergence, but also makes the method more expensive. Default is 4. maxiter : integer, optional Maximum number of iterations. Iteration will stop after maxiter steps even if the specified tolerance has not been achieved. xtype : {'f','d','F','D'} This parameter is deprecated -- avoid using it. M : {sparse matrix, dense matrix, LinearOperator}, optional Preconditioner for A. The preconditioner should approximate the inverse of A. Effective preconditioning dramatically improves the rate of convergence, which implies that fewer iterations are needed to reach a given error tolerance. callback : function, optional User-supplied function to call after each iteration. It is called as callback(xk), where xk is the current solution vector. Returns ------- x : array or matrix The converged solution. info : integer Provides convergence information: - 0 : successful exit - >0 : convergence to tolerance not achieved, number of iterations - <0 : illegal input or breakdown References ---------- .. [1] P. Sonneveld and M. B. van Gijzen SIAM J. Sci. Comput. Vol. 31, No. 2, pp. 1035--1062, (2008). .. [2] M. B. van Gijzen and P. Sonneveld ACM Trans. Math. Software,, Vol. 38, No. 1, pp. 5:1-5:19, (2011). .. [3] This file is a translation of the following MATLAB implementation: http://ta.twi.tudelft.nl/nw/users/gijzen/idrs.m """ A, M, x, b, postprocess = make_system(A, M, x0, b, xtype) n = len(b) if maxiter is None: maxiter = n*10 matvec = A.matvec psolve = M.matvec xtype = x.dtype axpy, dot, scal = get_blas_funcs(['axpy', 'dot', 'scal'], dtype=xtype) np.random.seed(0) P = np.random.rand(s, n) bnrm = np.linalg.norm(b) info = 0 # Check for zero rhs: if bnrm == 0.0: # Solution is null-vector return postprocess(np.zeros(n, dtype=xtype)), 0 # Compute initial residual: r = b - matvec(x) rnrm = np.linalg.norm(r) # Relative tolerance tolb = tol*np.linalg.norm(b) if rnrm < tolb: # Initial guess is a good enough solution return postprocess(x), 0 # Initialization angle = 0.7 G = np.zeros((n, s), dtype=xtype) U = np.zeros((n, s), dtype=xtype) Ms = np.eye(s, dtype=xtype) om = 1.0 iter_ = 0 # Main iteration loop, build G-spaces: while rnrm >= tolb and iter_ < maxiter: # New right-hand size for small system: f = P.dot(r) for k in xrange(s): # Solve small system and make v orthogonal to P: c = np.linalg.solve(Ms[k:s, k:s], f[k:s]) v = r - G[:, k:s].dot(c) # Preconditioning: v = psolve(v) # Compute new U(:,k) and G(:,k), G(:,k) is in space G_j U[:, k] = axpy(v, U[:, k:s].dot(c), None, om) # Matrix-vector product G[:, k] = matvec(U[:, k]) # Bi-Orthogonalize the new basis vectors: for i in xrange(0, k): alpha = dot(P[i, :], G[:, k]) / Ms[i, i] G[:, k] = axpy(G[:, i], G[:, k], None, -alpha) U[:, k] = axpy(U[:, i], U[:, k], None, -alpha) # New column of M = P'*G (first k-1 entries are zero) for i in xrange(k, s): Ms[i, k] = dot(P[i, :], G[:, k]) if Ms[k, k] == 0.0: # Breakdown return postprocess(x), -1 # Make r orthogonal to q_i, i = 1..k beta = f[k] / Ms[k, k] x = axpy(U[:, k], x, None, beta) r = axpy(G[:, k], r, None, -beta) rnrm = np.linalg.norm(r) if callback is not None: callback(x) iter_ += 1 if rnrm < tolb or iter_ >= maxiter: break # New f = P'*r (first k components are zero) if k < s - 1: f[k + 1:s] = axpy(Ms[k + 1:s, k], f[k + 1:s], None, -beta) # Now we have sufficient vectors in G_j to compute residual in G_j+1 # Note: r is already perpendicular to P so v = r if rnrm < tolb or iter_ >= maxiter: break # Preconditioning: v = psolve(r) # Matrix-vector product t = matvec(v) # Computation of a new omega ns = np.linalg.norm(r) nt = np.linalg.norm(t) ts = dot(t, r) rho = abs(ts / (nt * ns)) om = ts / (nt * nt) if rho < angle: om = om * angle / rho # New vector in G_j+1 x = axpy(v, x, None, om) r = axpy(t, r, None, -om) rnrm = np.linalg.norm(r) if callback is not None: callback(x) iter_ += 1 if rnrm >= tolb: info = iter_ return postprocess(x), info
def gmres_mgs(A, b, x0=None, tol=1e-5, restrt=None, maxiter=None, xtype=None, M=None, callback=None, residuals=None, reorth=False): ''' Generalized Minimum Residual Method (GMRES) GMRES iteratively refines the initial solution guess to the system Ax = b Modified Gram-Schmidt version Parameters ---------- A : {array, matrix, sparse matrix, LinearOperator} n x n, linear system to solve b : {array, matrix} right hand side, shape is (n,) or (n,1) x0 : {array, matrix} initial guess, default is a vector of zeros tol : float relative convergence tolerance, i.e. tol is scaled by the norm of the initial preconditioned residual restrt : {None, int} - if int, restrt is max number of inner iterations and maxiter is the max number of outer iterations - if None, do not restart GMRES, and max number of inner iterations is maxiter maxiter : {None, int} - if restrt is None, maxiter is the max number of inner iterations and GMRES does not restart - if restrt is int, maxiter is the max number of outer iterations, and restrt is the max number of inner iterations xtype : type dtype for the solution, default is automatic type detection M : {array, matrix, sparse matrix, LinearOperator} n x n, inverted preconditioner, i.e. solve M A x = M b. callback : function User-supplied function is called after each iteration as callback(xk), where xk is the current solution vector residuals : list residuals contains the preconditioned residual norm history, including the initial residual. reorth : boolean If True, then a check is made whether to re-orthogonalize the Krylov space each GMRES iteration Returns ------- (xNew, info) xNew : an updated guess to the solution of Ax = b info : halting status of gmres == ============================================= 0 successful exit >0 convergence to tolerance not achieved, return iteration count instead. This value is precisely the order of the Krylov space. <0 numerical breakdown, or illegal input == ============================================= Notes ----- - The LinearOperator class is in scipy.sparse.linalg.interface. Use this class if you prefer to define A or M as a mat-vec routine as opposed to explicitly constructing the matrix. A.psolve(..) is still supported as a legacy. - For robustness, modified Gram-Schmidt is used to orthogonalize the Krylov Space Givens Rotations are used to provide the residual norm each iteration Examples -------- >>> from pyamg.krylov import gmres >>> from pyamg.util.linalg import norm >>> import numpy as np >>> from pyamg.gallery import poisson >>> A = poisson((10,10)) >>> b = np.ones((A.shape[0],)) >>> (x,flag) = gmres(A,b, maxiter=2, tol=1e-8, orthog='mgs') >>> print norm(b - A*x) >>> 6.5428213057 References ---------- .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003 http://www-users.cs.umn.edu/~saad/books.html .. [2] C. T. Kelley, http://www4.ncsu.edu/~ctk/matlab_roots.html ''' # Convert inputs to linear system, with error checking A, M, x, b, postprocess = make_system(A, M, x0, b, xtype) dimen = A.shape[0] # Ensure that warnings are always reissued from this function import warnings warnings.filterwarnings('always', module='pyamg\.krylov\._gmres_mgs') # Choose type if not hasattr(A, 'dtype'): Atype = upcast(x.dtype, b.dtype) else: Atype = A.dtype if not hasattr(M, 'dtype'): Mtype = upcast(x.dtype, b.dtype) else: Mtype = M.dtype xtype = upcast(Atype, x.dtype, b.dtype, Mtype) if restrt is not None: restrt = int(restrt) if maxiter is not None: maxiter = int(maxiter) # Get fast access to underlying BLAS routines # dotc is the conjugate dot, dotu does no conjugation [lartg] = get_lapack_funcs(['lartg'], [x]) if np.iscomplexobj(np.zeros((1, ), dtype=xtype)): [axpy, dotu, dotc, scal] =\ get_blas_funcs(['axpy', 'dotu', 'dotc', 'scal'], [x]) else: # real type [axpy, dotu, dotc, scal] =\ get_blas_funcs(['axpy', 'dot', 'dot', 'scal'], [x]) # Make full use of direct access to BLAS by defining own norm def norm(z): return np.sqrt(np.real(dotc(z, z))) # Should norm(r) be kept if residuals == []: keep_r = True else: keep_r = False # Set number of outer and inner iterations if restrt: if maxiter: max_outer = maxiter else: max_outer = 1 if restrt > dimen: warn('Setting number of inner iterations (restrt) to maximum\ allowed, which is A.shape[0] ') restrt = dimen max_inner = restrt else: max_outer = 1 if maxiter > dimen: warn('Setting number of inner iterations (maxiter) to maximum\ allowed, which is A.shape[0] ') maxiter = dimen elif maxiter is None: maxiter = min(dimen, 40) max_inner = maxiter # Is this a one dimensional matrix? if dimen == 1: entry = np.ravel(A * np.array([1.0], dtype=xtype)) return (postprocess(b / entry), 0) # Prep for method r = b - np.ravel(A * x) # Apply preconditioner r = np.ravel(M * r) normr = norm(r) if keep_r: residuals.append(normr) # Check for nan, inf # if isnan(r).any() or isinf(r).any(): # warn('inf or nan after application of preconditioner') # return(postprocess(x), -1) # Check initial guess ( scaling by b, if b != 0, # must account for case when norm(b) is very small) normb = norm(b) if normb == 0.0: normb = 1.0 if normr < tol * normb: return (postprocess(x), 0) # Scale tol by ||r_0||_2, we use the preconditioned residual # because this is left preconditioned GMRES. if normr != 0.0: tol = tol * normr # Use separate variable to track iterations. If convergence fails, we # cannot simply report niter = (outer-1)*max_outer + inner. Numerical # error could cause the inner loop to halt while the actual ||r|| > tol. niter = 0 # Begin GMRES for outer in range(max_outer): # Preallocate for Givens Rotations, Hessenberg matrix and Krylov Space # Space required is O(dimen*max_inner). # NOTE: We are dealing with row-major matrices, so we traverse in a # row-major fashion, # i.e., H and V's transpose is what we store. Q = [] # Givens Rotations # Upper Hessenberg matrix, which is then # converted to upper tri with Givens Rots H = np.zeros((max_inner + 1, max_inner + 1), dtype=xtype) V = np.zeros((max_inner + 1, dimen), dtype=xtype) # Krylov Space # vs store the pointers to each column of V. # This saves a considerable amount of time. vs = [] # v = r/normr V[0, :] = scal(1.0 / normr, r) vs.append(V[0, :]) # This is the RHS vector for the problem in the Krylov Space g = np.zeros((dimen, ), dtype=xtype) g[0] = normr for inner in range(max_inner): # New Search Direction v = V[inner + 1, :] v[:] = np.ravel(M * (A * vs[-1])) vs.append(v) normv_old = norm(v) # Check for nan, inf # if isnan(V[inner+1, :]).any() or isinf(V[inner+1, :]).any(): # warn('inf or nan after application of preconditioner') # return(postprocess(x), -1) # Modified Gram Schmidt for k in range(inner + 1): vk = vs[k] alpha = dotc(vk, v) H[inner, k] = alpha v[:] = axpy(vk, v, dimen, -alpha) normv = norm(v) H[inner, inner + 1] = normv # Re-orthogonalize if (reorth is True) and (normv_old == normv_old + 0.001 * normv): for k in range(inner + 1): vk = vs[k] alpha = dotc(vk, v) H[inner, k] = H[inner, k] + alpha v[:] = axpy(vk, v, dimen, -alpha) # Check for breakdown if H[inner, inner + 1] != 0.0: v[:] = scal(1.0 / H[inner, inner + 1], v) # Apply previous Givens rotations to H if inner > 0: apply_givens(Q, H[inner, :], inner) # Calculate and apply next complex-valued Givens Rotation # ==> Note that if max_inner = dimen, then this is unnecessary # for the last inner # iteration, when inner = dimen-1. if inner != dimen - 1: if H[inner, inner + 1] != 0: [c, s, r] = lartg(H[inner, inner], H[inner, inner + 1]) Qblock = np.array([[c, s], [-np.conjugate(s), c]], dtype=xtype) Q.append(Qblock) # Apply Givens Rotation to g, # the RHS for the linear system in the Krylov Subspace. g[inner:inner + 2] = np.dot(Qblock, g[inner:inner + 2]) # Apply effect of Givens Rotation to H H[inner, inner] = dotu(Qblock[0, :], H[inner, inner:inner + 2]) H[inner, inner + 1] = 0.0 niter += 1 # Don't update normr if last inner iteration, because # normr is calculated directly after this loop ends. if inner < max_inner - 1: normr = np.abs(g[inner + 1]) if normr < tol: break # Allow user access to the iterates if callback is not None: callback(x) if keep_r: residuals.append(normr) # end inner loop, back to outer loop # Find best update to x in Krylov Space V. Solve inner x inner system. y = sp.linalg.solve(H[0:inner + 1, 0:inner + 1].T, g[0:inner + 1]) update = np.ravel(np.mat(V[:inner + 1, :]).T * y.reshape(-1, 1)) x = x + update r = b - np.ravel(A * x) # Apply preconditioner r = np.ravel(M * r) normr = norm(r) # Check for nan, inf # if isnan(r).any() or isinf(r).any(): # warn('inf or nan after application of preconditioner') # return(postprocess(x), -1) # Allow user access to the iterates if callback is not None: callback(x) if keep_r: residuals.append(normr) # Has GMRES stagnated? indices = (x != 0) if indices.any(): change = np.max(np.abs(update[indices] / x[indices])) if change < 1e-12: # No change, halt return (postprocess(x), -1) # test for convergence if normr < tol: return (postprocess(x), 0) # end outer loop return (postprocess(x), niter)
def gcrotmk(A, b, x0=None, tol=1e-5, maxiter=1000, M=None, callback=None, m=20, k=None, CU=None, discard_C=False, truncate='oldest', atol=None): """ Solve a matrix equation using flexible GCROT(m,k) algorithm. Parameters ---------- A : {sparse matrix, dense matrix, LinearOperator} The real or complex N-by-N matrix of the linear system. b : {array, matrix} Right hand side of the linear system. Has shape (N,) or (N,1). x0 : {array, matrix} Starting guess for the solution. tol, atol : float, optional Tolerances for convergence, ``norm(residual) <= max(tol*norm(b), atol)``. The default for ``atol`` is `tol`. .. warning:: The default value for `atol` will be changed in a future release. For future compatibility, specify `atol` explicitly. maxiter : int, optional Maximum number of iterations. Iteration will stop after maxiter steps even if the specified tolerance has not been achieved. M : {sparse matrix, dense matrix, LinearOperator}, optional Preconditioner for A. The preconditioner should approximate the inverse of A. gcrotmk is a 'flexible' algorithm and the preconditioner can vary from iteration to iteration. Effective preconditioning dramatically improves the rate of convergence, which implies that fewer iterations are needed to reach a given error tolerance. callback : function, optional User-supplied function to call after each iteration. It is called as callback(xk), where xk is the current solution vector. m : int, optional Number of inner FGMRES iterations per each outer iteration. Default: 20 k : int, optional Number of vectors to carry between inner FGMRES iterations. According to [2]_, good values are around m. Default: m CU : list of tuples, optional List of tuples ``(c, u)`` which contain the columns of the matrices C and U in the GCROT(m,k) algorithm. For details, see [2]_. The list given and vectors contained in it are modified in-place. If not given, start from empty matrices. The ``c`` elements in the tuples can be ``None``, in which case the vectors are recomputed via ``c = A u`` on start and orthogonalized as described in [3]_. discard_C : bool, optional Discard the C-vectors at the end. Useful if recycling Krylov subspaces for different linear systems. truncate : {'oldest', 'smallest'}, optional Truncation scheme to use. Drop: oldest vectors, or vectors with smallest singular values using the scheme discussed in [1,2]. See [2]_ for detailed comparison. Default: 'oldest' Returns ------- x : array or matrix The solution found. info : int Provides convergence information: * 0 : successful exit * >0 : convergence to tolerance not achieved, number of iterations References ---------- .. [1] E. de Sturler, ''Truncation strategies for optimal Krylov subspace methods'', SIAM J. Numer. Anal. 36, 864 (1999). .. [2] J.E. Hicken and D.W. Zingg, ''A simplified and flexible variant of GCROT for solving nonsymmetric linear systems'', SIAM J. Sci. Comput. 32, 172 (2010). .. [3] M.L. Parks, E. de Sturler, G. Mackey, D.D. Johnson, S. Maiti, ''Recycling Krylov subspaces for sequences of linear systems'', SIAM J. Sci. Comput. 28, 1651 (2006). """ A,M,x,b,postprocess = make_system(A,M,x0,b) if not np.isfinite(b).all(): raise ValueError("RHS must contain only finite numbers") if truncate not in ('oldest', 'smallest'): raise ValueError("Invalid value for 'truncate': %r" % (truncate,)) if atol is None: warnings.warn("scipy.sparse.linalg.gcrotmk called without specifying `atol`. " "The default value will change in the future. To preserve " "current behavior, set ``atol=tol``.", category=DeprecationWarning, stacklevel=2) atol = tol matvec = A.matvec psolve = M.matvec if CU is None: CU = [] if k is None: k = m axpy, dot, scal = None, None, None r = b - matvec(x) axpy, dot, scal, nrm2 = get_blas_funcs(['axpy', 'dot', 'scal', 'nrm2'], (x, r)) b_norm = nrm2(b) if discard_C: CU[:] = [(None, u) for c, u in CU] # Reorthogonalize old vectors if CU: # Sort already existing vectors to the front CU.sort(key=lambda cu: cu[0] is not None) # Fill-in missing ones C = np.empty((A.shape[0], len(CU)), dtype=r.dtype, order='F') us = [] j = 0 while CU: # More memory-efficient: throw away old vectors as we go c, u = CU.pop(0) if c is None: c = matvec(u) C[:,j] = c j += 1 us.append(u) # Orthogonalize Q, R, P = qr(C, overwrite_a=True, mode='economic', pivoting=True) del C # C := Q cs = list(Q.T) # U := U P R^-1, back-substitution new_us = [] for j in xrange(len(cs)): u = us[P[j]] for i in xrange(j): u = axpy(us[P[i]], u, u.shape[0], -R[i,j]) if abs(R[j,j]) < 1e-12 * abs(R[0,0]): # discard rest of the vectors break u = scal(1.0/R[j,j], u) new_us.append(u) # Form the new CU lists CU[:] = list(zip(cs, new_us))[::-1] if CU: axpy, dot = get_blas_funcs(['axpy', 'dot'], (r,)) # Solve first the projection operation with respect to the CU # vectors. This corresponds to modifying the initial guess to # be # # x' = x + U y # y = argmin_y || b - A (x + U y) ||^2 # # The solution is y = C^H (b - A x) for c, u in CU: yc = dot(c, r) x = axpy(u, x, x.shape[0], yc) r = axpy(c, r, r.shape[0], -yc) # GCROT main iteration for j_outer in xrange(maxiter): # -- callback if callback is not None: callback(x) beta = nrm2(r) # -- check stopping condition beta_tol = max(atol, tol * b_norm) if beta <= beta_tol and (j_outer > 0 or CU): # recompute residual to avoid rounding error r = b - matvec(x) beta = nrm2(r) if beta <= beta_tol: j_outer = -1 break ml = m + max(k - len(CU), 0) cs = [c for c, u in CU] try: Q, R, B, vs, zs, y, pres = _fgmres(matvec, r/beta, ml, rpsolve=psolve, atol=max(atol, tol*b_norm)/beta, cs=cs) y *= beta except LinAlgError: # Floating point over/underflow, non-finite result from # matmul etc. -- report failure. break # # At this point, # # [A U, A Z] = [C, V] G; G = [ I B ] # [ 0 H ] # # where [C, V] has orthonormal columns, and r = beta v_0. Moreover, # # || b - A (x + Z y + U q) ||_2 = || r - C B y - V H y - C q ||_2 = min! # # from which y = argmin_y || beta e_1 - H y ||_2, and q = -B y # # # GCROT(m,k) update # # Define new outer vectors # ux := (Z - U B) y ux = zs[0]*y[0] for z, yc in zip(zs[1:], y[1:]): ux = axpy(z, ux, ux.shape[0], yc) # ux += z*yc by = B.dot(y) for cu, byc in zip(CU, by): c, u = cu ux = axpy(u, ux, ux.shape[0], -byc) # ux -= u*byc # cx := V H y hy = Q.dot(R.dot(y)) cx = vs[0] * hy[0] for v, hyc in zip(vs[1:], hy[1:]): cx = axpy(v, cx, cx.shape[0], hyc) # cx += v*hyc # Normalize cx, maintaining cx = A ux # This new cx is orthogonal to the previous C, by construction try: alpha = 1/nrm2(cx) if not np.isfinite(alpha): raise FloatingPointError() except (FloatingPointError, ZeroDivisionError): # Cannot update, so skip it continue cx = scal(alpha, cx) ux = scal(alpha, ux) # Update residual and solution gamma = dot(cx, r) r = axpy(cx, r, r.shape[0], -gamma) # r -= gamma*cx x = axpy(ux, x, x.shape[0], gamma) # x += gamma*ux # Truncate CU if truncate == 'oldest': while len(CU) >= k and CU: del CU[0] elif truncate == 'smallest': if len(CU) >= k and CU: # cf. [1,2] D = solve(R[:-1,:].T, B.T).T W, sigma, V = svd(D) # C := C W[:,:k-1], U := U W[:,:k-1] new_CU = [] for j, w in enumerate(W[:,:k-1].T): c, u = CU[0] c = c * w[0] u = u * w[0] for cup, wp in zip(CU[1:], w[1:]): cp, up = cup c = axpy(cp, c, c.shape[0], wp) u = axpy(up, u, u.shape[0], wp) # Reorthogonalize at the same time; not necessary # in exact arithmetic, but floating point error # tends to accumulate here for cp, up in new_CU: alpha = dot(cp, c) c = axpy(cp, c, c.shape[0], -alpha) u = axpy(up, u, u.shape[0], -alpha) alpha = nrm2(c) c = scal(1.0/alpha, c) u = scal(1.0/alpha, u) new_CU.append((c, u)) CU[:] = new_CU # Add new vector to CU CU.append((cx, ux)) # Include the solution vector to the span CU.append((None, x.copy())) if discard_C: CU[:] = [(None, uz) for cz, uz in CU] return postprocess(x), j_outer + 1
def gmres(A, b, x0=None, tol=1e-5, restrt=None, maxiter=None, xtype=None, M=None, callback=None, residuals=None): ''' Generalized Minimum Residual Method (GMRES) GMRES iteratively refines the initial solution guess to the system Ax = b For robustness, Householder reflections are used to orthonormalize the Krylov Space Givens Rotations are used to provide the residual norm each iteration Parameters ---------- A : array, matrix or sparse matrix n x n, linear system to solve b : array n x 1, right hand side x0 : array n x 1, initial guess default is a vector of zeros tol : float convergence tolerance restrt : int number of restarts total iterations = restrt*maxiter maxiter : int maximum number of allowed inner iterations xtype : type dtype for the solution M : matrix-like n x n, inverted preconditioner, i.e. solve M A x = b. For preconditioning with a mat-vec routine, set A.psolve = func, where func := M y callback : function callback( ||resid||_2 ) is called each iteration, residuals : {None, empty-list} If empty-list, residuals holds the residual norm history, including the initial residual, upon completion Returns ------- (xNew, info) xNew -- an updated guess to the solution of Ax = b info -- halting status of gmres 0 : successful exit >0 : convergence to tolerance not achieved, return iteration count instead. This value is precisely the order of the Krylov space. <0 : numerical breakdown, or illegal input Notes ----- Examples -------- >>>from pyamg.krylov import * >>>from scipy import rand >>>import pyamg >>>A = pyamg.poisson((50,50)) >>>b = rand(A.shape[0],) >>>(x,flag) = gmres(A,b) >>>print pyamg.util.linalg.norm(b - A*x) References ---------- Yousef Saad, "Iterative Methods for Sparse Linear Systems, Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003 ''' # Convert inputs to linear system, with error checking A,M,x,b,postprocess = make_system(A,M,x0,b,xtype) dimen = A.shape[0] # Choose type xtype = upcast(A.dtype, x.dtype, b.dtype, M.dtype) # We assume henceforth that shape=(n,) for all arrays b = ravel(array(b,xtype)) x = ravel(array(x,xtype)) # Should norm(r) be kept if residuals == []: keep_r = True else: keep_r = False # check number of iterations if restrt == None: restrt = 1 elif restrt < 1: raise ValueError('Number of restarts must be positive') if maxiter == None: maxiter = int(max(ceil(dimen/restrt))) elif maxiter < 1: raise ValueError('Number of iterations must be positive') elif maxiter > dimen: warn('maximimum allowed inner iterations (maxiter) are the number of degress of freedom') maxiter = dimen # Scale tol by normb normb = linalg.norm(b) if normb == 0: pass # if callback != None: # callback(0.0) # # return (postprocess(zeros((dimen,)), dtype=xtype),0) else: tol = tol*normb # Is this a one dimensional matrix? if dimen == 1: entry = ravel(A*array([1.0], dtype=xtype)) return (postprocess(b/entry), 0) # Prep for method r = b - ravel(A*x) normr = linalg.norm(r) if keep_r: residuals.append(normr) # Is initial guess sufficient? if normr <= tol: if callback != None: callback(norm(r)) return (postprocess(x), 0) #Apply preconditioner r = ravel(M*r) normr = linalg.norm(r) # Check for nan, inf if any(isnan(r)) or any(isinf(r)): warn('inf or nan after application of preconditioner') return(postprocess(x), -1) # Use separate variable to track iterations. If convergence fails, we cannot # simply report niter = (outer-1)*maxiter + inner. Numerical error could cause # the inner loop to halt before reaching maxiter while the actual ||r|| > tol. niter = 0 # Begin GMRES for outer in range(restrt): # Calculate vector w, which defines the Householder reflector # Take shortcut in calculating, # w = r + sign(r[1])*||r||_2*e_1 w = r beta = mysign(w[0])*normr w[0] += beta w = w / linalg.norm(w) # Preallocate for Krylov vectors, Householder reflectors and Hessenberg matrix # Space required is O(dimen*maxiter) H = zeros( (maxiter, maxiter), dtype=xtype) # upper Hessenberg matrix (actually made upper tri with Given's Rotations) W = zeros( (dimen, maxiter), dtype=xtype) # Householder reflectors W[:,0] = w # Multiply r with (I - 2*w*w.T), i.e. apply the Householder reflector # This is the RHS vector for the problem in the Krylov Space g = zeros((dimen,), dtype=xtype) g[0] = -beta for inner in range(maxiter): # Calcute Krylov vector in two steps # (1) Calculate v = P_j = (I - 2*w*w.T)v, where k = inner v = -2.0*conjugate(w[inner])*w v[inner] += 1.0 # (2) Calculate the rest, v = P_1*P_2*P_3...P_{j-1}*ej. for j in range(inner-1,-1,-1): v = v - 2.0*dot(conjugate(W[:,j]), v)*W[:,j] # Calculate new search direction v = ravel(A*v) #Apply preconditioner v = ravel(M*v) # Check for nan, inf if any(isnan(v)) or any(isinf(v)): warn('inf or nan after application of preconditioner') return(postprocess(x), -1) # Factor in all Householder orthogonal reflections on new search direction for j in range(inner+1): v = v - 2.0*dot(conjugate(W[:,j]), v)*W[:,j] # Calculate next Householder reflector, w # w = v[inner+1:] + sign(v[inner+1])*||v[inner+1:]||_2*e_{inner+1) # Note that if maxiter = dimen, then this is unnecessary for the last inner # iteration, when inner = dimen-1. Here we do not need to calculate a # Householder reflector or Given's rotation because nnz(v) is already the # desired length, i.e. we do not need to zero anything out. if inner != dimen-1: w = zeros((dimen,), dtype=xtype) vslice = v[inner+1:] alpha = linalg.norm(vslice) if alpha != 0: alpha = mysign(vslice[0])*alpha # We do not need the final reflector for future calculations if inner < (maxiter-1): w[inner+1:] = vslice w[inner+1] += alpha w = w / linalg.norm(w) W[:,inner+1] = w # Apply new reflector to v # v = v - 2.0*w*(w.T*v) v[inner+1] = -alpha v[inner+2:] = 0.0 # Apply all previous Given's Rotations to v if inner == 0: # Q will store the cumulative effect of all Given's Rotations Q = scipy.sparse.eye(dimen, dimen, format='csr', dtype=xtype) # Declare initial Qj, which will be the current Given's Rotation rowptr = hstack( (array([0, 2, 4],int), arange(5,dimen+3,dtype=int)) ) colindices = hstack( (array([0, 1, 0, 1],int), arange(2, dimen,dtype=int)) ) data = ones((dimen+2,), dtype=xtype) Qj = csr_matrix( (data, colindices, rowptr), shape=(dimen,dimen), dtype=xtype) else: # Could avoid building a global Given's Rotation, by storing # and applying each 2x2 matrix individually. # But that would require looping, the bane of wonderful Python Q = Qj*Q v = Q*v # Calculate Qj, the next Given's rotation, where j = inner # Note that if maxiter = dimen, then this is unnecessary for the last inner # iteration, when inner = dimen-1. Here we do not need to calculate a # Householder reflector or Given's rotation because nnz(v) is already the # desired length, i.e. we do not need to zero anything out. if inner != dimen-1: if v[inner+1] != 0: # Calculate terms for complex 2x2 Given's Rotation # Note that abs(x) takes the complex modulus h1 = v[inner]; h2 = v[inner+1]; h1_mag = abs(h1); h2_mag = abs(h2); if h1_mag < h2_mag: mu = h1/h2 tau = conjugate(mu)/abs(mu) else: mu = h2/h1 tau = mu/abs(mu) denom = sqrt( h1_mag**2 + h2_mag**2 ) c = h1_mag/denom; s = h2_mag*tau/denom; Qblock = array([[c, conjugate(s)], [-s, c]], dtype=xtype) # Modify Qj in csr-format so that it represents the current # global Given's Rotation equivalent to Qblock if inner != 0: Qj.data[inner-1] = 1.0 Qj.indices[inner-1] = inner-1 Qj.indptr[inner-1] = inner-1 Qj.data[inner:inner+4] = ravel(Qblock) Qj.indices[inner:inner+4] = [inner, inner+1, inner, inner+1] Qj.indptr[inner:inner+3] = [inner, inner+2, inner+4] # Apply Given's Rotation to g, # the RHS for the linear system in the Krylov Subspace. # Note that this dot does a matrix multiply, not an actual # dot product where a conjugate transpose is taken g[inner:inner+2] = dot(Qblock, g[inner:inner+2]) # Apply effect of Given's Rotation to v v[inner] = dot(Qblock[0,:], v[inner:inner+2]) v[inner+1] = 0.0 # Write to upper Hessenberg Matrix, # the LHS for the linear system in the Krylov Subspace H[:,inner] = v[0:maxiter] # Don't update normr if last inner iteration, because # normr is calculated directly after this loop ends. if inner < maxiter-1: normr = abs(g[inner+1]) if normr < tol: break # Allow user access to residual if callback != None: callback( normr ) if keep_r: residuals.append(normr) niter += 1 # end inner loop, back to outer loop # Find best update to x in Krylov Space, V. Solve inner+1 x inner+1 system. # Apparently this is the best way to solve a triangular # system in the magical world of scipy piv = arange(inner+1) y = lu_solve((H[0:(inner+1),0:(inner+1)], piv), g[0:(inner+1)], trans=0) # Use Horner like Scheme to map solution, y, back to original space. # Note that we do not use the last reflector. update = zeros(x.shape, dtype=xtype) for j in range(inner,-1,-1): update[j] += y[j] # Apply j-th reflector, (I - 2.0*w_j*w_j.T)*upadate update = update - 2.0*dot(conjugate(W[:,j]), update)*W[:,j] x = x + update r = b - ravel(A*x) #Apply preconditioner r = ravel(M*r) normr = linalg.norm(r) # Check for nan, inf if any(isnan(r)) or any(isinf(r)): warn('inf or nan after application of preconditioner') return(postprocess(x), -1) # Allow user access to residual if callback != None: callback( normr ) if keep_r: residuals.append(normr) # Has GMRES stagnated? indices = (x != 0) if indices.any(): change = max(abs( update[indices] / x[indices] )) if change < 1e-12: # No change, halt return (postprocess(x), -1) # test for convergence if normr < tol: return (postprocess(x),0) # end outer loop return (postprocess(x), niter)
def idrs(A, b, x0=None, tol=1e-5, s=4, maxiter=None, M=None, callback=None): """ Use Induced Dimension Reduction method [IDR(s)] to solve A x = b Parameters ---------- A : {sparse matrix, dense matrix, LinearOperator} The real or complex N-by-N matrix of the linear system. b : {array, matrix} Right hand side of the linear system. Has shape (N,) or (N,1). x0 : {array, matrix} Starting guess for the solution. tol : float, optional Tolerance to achieve. The algorithm terminates when either the relative or the absolute residual is below `tol`. s : integer, optional specifies the dimension of the shadow space. Normally, a higher s gives faster convergence, but also makes the method more expensive. Default is 4. maxiter : integer, optional Maximum number of iterations. Iteration will stop after maxiter steps even if the specified tolerance has not been achieved. M : {sparse matrix, dense matrix, LinearOperator}, optional Preconditioner for A. The preconditioner should approximate the inverse of A. Effective preconditioning dramatically improves the rate of convergence, which implies that fewer iterations are needed to reach a given error tolerance. callback : function, optional User-supplied function to call after each iteration. It is called as callback(xk), where xk is the current solution vector. Returns ------- x : array or matrix The converged solution. info : integer Provides convergence information: - 0 : successful exit - >0 : convergence to tolerance not achieved, number of iterations - <0 : illegal input or breakdown References ---------- .. [1] P. Sonneveld and M. B. van Gijzen SIAM J. Sci. Comput. Vol. 31, No. 2, pp. 1035--1062, (2008). .. [2] M. B. van Gijzen and P. Sonneveld ACM Trans. Math. Software,, Vol. 38, No. 1, pp. 5:1-5:19, (2011). .. [3] This file is a translation of the following MATLAB implementation: http://ta.twi.tudelft.nl/nw/users/gijzen/idrs.m """ A, M, x, b, postprocess = make_system(A, M, x0, b) n = len(b) if maxiter is None: maxiter = n * 10 matvec = A.matvec psolve = M.matvec xtype = x.dtype axpy, dot = get_blas_funcs(['axpy', 'dot'], dtype=xtype) np.random.seed(0) P = np.random.randn(s, n) bnrm = np.linalg.norm(b) info = 0 if callback is not None: callback(x) # Check for zero rhs: if bnrm == 0.0: # Solution is null-vector return postprocess(np.zeros(n, dtype=xtype)), 0 # Compute initial residual: r = b - matvec(x) rnrm = np.linalg.norm(r) # Relative tolerance tolb = tol * np.linalg.norm(b) if rnrm < tolb: # Initial guess is a good enough solution return postprocess(x), 0 # Initialization angle = 0.7 G = np.zeros((n, s), dtype=xtype) U = np.zeros((n, s), dtype=xtype) Ms = np.eye(s, dtype=xtype) om = 1.0 iter_ = 0 # Main iteration loop, build G-spaces: while rnrm >= tolb and iter_ < maxiter: # New right-hand size for small system: f = P.dot(r) for k in range(0, s): # Solve small system and make v orthogonal to P: c = np.linalg.solve(Ms[k:s, k:s], f[k:s]) v = r - G[:, k:s].dot(c) # Preconditioning: v = psolve(v) # Compute new U(:,k) and G(:,k), G(:,k) is in space G_j U[:, k] = axpy(v, U[:, k:s].dot(c), None, om) # Matrix-vector product G[:, k] = matvec(U[:, k]) # Bi-Orthogonalize the new basis vectors: for i in range(0, k): alpha = dot(P[i, :], G[:, k]) / Ms[i, i] G[:, k] = axpy(G[:, i], G[:, k], None, -alpha) U[:, k] = axpy(U[:, i], U[:, k], None, -alpha) # New column of M = P'*G (first k-1 entries are zero) for i in range(k, s): Ms[i, k] = dot(P[i, :], G[:, k]) if Ms[k, k] == 0.0: # Breakdown return postprocess(x), -1 # Make r orthogonal to g_i, i = 1..k beta = f[k] / Ms[k, k] x = axpy(U[:, k], x, None, beta) r = axpy(G[:, k], r, None, -beta) rnrm = np.linalg.norm(r) if callback is not None: callback(x) iter_ += 1 if rnrm < tolb or iter_ >= maxiter: break # New f = P'*r (first k components are zero) if k < s - 1: f[k + 1:s] = axpy(Ms[k + 1:s, k], f[k + 1:s], None, -beta) # Now we have sufficient vectors in G_j to compute residual in G_j+1 # Note: r is already perpendicular to P so v = r if rnrm < tolb or iter_ >= maxiter: break # Preconditioning: v = psolve(r) # Matrix-vector product t = matvec(v) # Computation of a new omega nr = np.linalg.norm(r) nt = np.linalg.norm(t) ts = dot(t, r) rho = abs(ts / (nt * nr)) om = ts / (nt * nt) if rho < angle: om = om * angle / rho # New vector in G_j+1 x = axpy(v, x, None, om) r = axpy(t, r, None, -om) rnrm = np.linalg.norm(r) if callback is not None: callback(x) iter_ += 1 if rnrm >= tolb: info = iter_ return postprocess(x), info