def msGmres(A,
b,
shifts,
m=200,
x0=None,
tol=1e-5,
maxiter=None,
M=None,
callback=None):
"""
Multi-Shift Gmres(m)
"""
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype=None)
n = len(b)
if maxiter is None:
maxiter = n * 10
matvec = A.matvec
psolve = M.matvec
xtype = x.dtype
axpy, dot, norm = get_blas_funcs(['axpy', 'dot', 'nrm2'], dtype=xtype)
# Relative tolerance
tolb = tol * norm(b)
# Compute initial residual:
r = b - matvec(x)
rnrm2 = norm(r)
# We should call the callback func
# Initialization
V = np.zeros((n, m + 1), dtype=xtype)
H = np.zeros((m + 1, m), dtype=xtype)
cs = np.zeros(m + 1, dtype=xtype)
sn = np.zeros(m + 1, dtype=xtype)
e1 = np.zeros(m + 1, dtype=xtype)
e1[0] = 1.0
# Begin iteration
#for _iter in range(0, maxit):
V[:, 0] = r / rnrm2
s = rnrm2 * e1
for i in range(0, m):
# Construct orthonormal basis using Gram-Schmidt
w = psolve(matvec(V[:, i]))
for k in range(0, i + 1):
H[k, i] = dot(V[:, k], w)
#w = w - H[k, i]*V[:, k]
axpy(V[:, k], w, None, -H[k, i])
H[i + 1, i] = norm(w)
V[:, i + 1] = w / H[i + 1, i]
# Multi-Shift with \bar{H}
# We should call the callback func
return V, H
def conjugate_gradient(A,
b,
x0=None,
tol=1e-5,
norm=np.linalg.norm,
maxiter=None,
callback=None,
init_seed=None):
dtype = A.dtype
A, B, x, b, postprocess = make_system(A, None, x0, b)
n = len(b)
if maxiter is None:
maxiter = n * 10
r = b - A.dot(x)
rnrm = norm(r)
p = r.copy()
if callback is not None:
callback(x)
tolr = tol * norm(b)
if rnrm < tolr:
return postprocess(x)[:, None]
for i in range(maxiter):
r_norm = r.T.dot(r)
Ap = A.dot(p)
alpha = r_norm / p.T.dot(Ap)
x += alpha * p
if callback is not None:
callback(x)
r -= alpha * Ap
rnrm = norm(r)
if rnrm < tolr:
break
beta = r.T.dot(r)
p = r + p * (beta / r_norm)
return postprocess(x)[:, None]
def bicgstab(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None,
callback=None, residuals=None):
'''Biconjugate Gradient Algorithm with Stabilization
Solves the linear system Ax = b. Left preconditioning is supported.
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by ||r_0||_2
maxiter : int
maximum number of allowed iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A A.H x = M b.
callback : function
User-supplied function is called after each iteration as
callback(xk), where xk is the current solution vector
residuals : list
residuals has the residual norm history,
including the initial residual, appended to it
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of bicgstab
== ======================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead.
<0 numerical breakdown, or illegal input
== ======================================
Notes
-----
The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
Examples
--------
>>> from pyamg.krylov.bicgstab import bicgstab
>>> from pyamg.util.linalg import norm
>>> import numpy
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = numpy.ones((A.shape[0],))
>>> (x,flag) = bicgstab(A,b, maxiter=2, tol=1e-8)
>>> print norm(b - A*x)
4.68163045309
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 231-234, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
# Convert inputs to linear system, with error checking
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype)
##
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always', module='pyamg\.krylov\._bicgstab')
# Check iteration numbers
if maxiter is None:
maxiter = len(x) + 5
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
# Prep for method
r = b - A*x
normr = norm(r)
if residuals is not None:
residuals[:] = [normr]
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol*normb:
return (postprocess(x), 0)
# Scale tol by ||r_0||_2
if normr != 0.0:
tol = tol*normr
# Is this a one dimensional matrix?
if A.shape[0] == 1:
entry = ravel(A*array([1.0], dtype=xtype))
return (postprocess(b/entry), 0)
rstar = r.copy()
p = r.copy()
rrstarOld = inner(rstar.conjugate(), r)
iter = 0
# Begin BiCGStab
while True:
Mp = M*p
AMp = A*Mp
# alpha = (r_j, rstar) / (A*M*p_j, rstar)
alpha = rrstarOld/inner(rstar.conjugate(), AMp)
# s_j = r_j - alpha*A*M*p_j
s = r - alpha*AMp
Ms = M*s
AMs = A*Ms
# omega = (A*M*s_j, s_j)/(A*M*s_j, A*M*s_j)
omega = inner(AMs.conjugate(), s)/inner(AMs.conjugate(), AMs)
# x_{j+1} = x_j + alpha*M*p_j + omega*M*s_j
x = x + alpha*Mp + omega*Ms
# r_{j+1} = s_j - omega*A*M*s
r = s - omega*AMs
# beta_j = (r_{j+1}, rstar)/(r_j, rstar) * (alpha/omega)
rrstarNew = inner(rstar.conjugate(), r)
beta = (rrstarNew / rrstarOld) * (alpha / omega)
rrstarOld = rrstarNew
# p_{j+1} = r_{j+1} + beta*(p_j - omega*A*M*p)
p = r + beta*(p - omega*AMp)
iter += 1
normr = norm(r)
if residuals is not None:
residuals.append(normr)
if callback is not None:
callback(x)
if normr < tol:
return (postprocess(x), 0)
if iter == maxiter:
return (postprocess(x), iter)
def lgmres(A, b, x0=None, tol=1e-5, maxiter=1000, M=None, callback=None,
inner_m=30, outer_k=3, outer_v=None, store_outer_Av=True,
prepend_outer_v=False, res = "both"):
"""
Solve a matrix equation using the LGMRES algorithm.
The LGMRES algorithm [1]_ [2]_ is designed to avoid some problems
in the convergence in restarted GMRES, and often converges in fewer
iterations.
Parameters
----------
A : {sparse matrix, dense matrix, LinearOperator}
The real or complex N-by-N matrix of the linear system.
b : {array, matrix}
Right hand side of the linear system. Has shape (N,) or (N,1).
x0 : {array, matrix}
Starting guess for the solution.
tol : float, optional
Tolerance to achieve. The algorithm terminates when either the relative
or the absolute residual is below `tol`.
res: string, optional
choose between relative, absolute or both (scipy default) to terminate
the algo
maxiter : int, optional
Maximum number of iterations. Iteration will stop after maxiter
steps even if the specified tolerance has not been achieved.
M : {sparse matrix, dense matrix, LinearOperator}, optional
Preconditioner for A. The preconditioner should approximate the
inverse of A. Effective preconditioning dramatically improves the
rate of convergence, which implies that fewer iterations are needed
to reach a given error tolerance.
callback : function, optional
User-supplied function to call after each iteration. It is called
as callback(xk), where xk is the current solution vector.
inner_m : int, optional
Number of inner GMRES iterations per each outer iteration.
outer_k : int, optional
Number of vectors to carry between inner GMRES iterations.
According to [1]_, good values are in the range of 1...3.
However, note that if you want to use the additional vectors to
accelerate solving multiple similar problems, larger values may
be beneficial.
outer_v : list of tuples, optional
List containing tuples ``(v, Av)`` of vectors and corresponding
matrix-vector products, used to augment the Krylov subspace, and
carried between inner GMRES iterations. The element ``Av`` can
be `None` if the matrix-vector product should be re-evaluated.
This parameter is modified in-place by `lgmres`, and can be used
to pass "guess" vectors in and out of the algorithm when solving
similar problems.
store_outer_Av : bool, optional
Whether LGMRES should store also A*v in addition to vectors `v`
in the `outer_v` list. Default is True.
prepend_outer_v : bool, optional
Whether to put outer_v augmentation vectors before Krylov iterates.
In standard LGMRES, prepend_outer_v=False.
Returns
-------
x : array or matrix
The converged solution.
info : int
Provides convergence information:
- 0 : successful exit
- >0 : convergence to tolerance not achieved, number of iterations
- <0 : illegal input or breakdown
Notes
-----
The LGMRES algorithm [1]_ [2]_ is designed to avoid the
slowing of convergence in restarted GMRES, due to alternating
residual vectors. Typically, it often outperforms GMRES(m) of
comparable memory requirements by some measure, or at least is not
much worse.
Another advantage in this algorithm is that you can supply it with
'guess' vectors in the `outer_v` argument that augment the Krylov
subspace. If the solution lies close to the span of these vectors,
the algorithm converges faster. This can be useful if several very
similar matrices need to be inverted one after another, such as in
Newton-Krylov iteration where the Jacobian matrix often changes
little in the nonlinear steps.
References
----------
.. [1] A.H. Baker and E.R. Jessup and T. Manteuffel, "A Technique for
Accelerating the Convergence of Restarted GMRES", SIAM J. Matrix
Anal. Appl. 26, 962 (2005).
.. [2] A.H. Baker, "On Improving the Performance of the Linear Solver
restarted GMRES", PhD thesis, University of Colorado (2003).
Examples
--------
>>> from scipy.sparse import csc_matrix
>>> from scipy.sparse.linalg import lgmres
>>> A = csc_matrix([[3, 2, 0], [1, -1, 0], [0, 5, 1]], dtype=float)
>>> b = np.array([2, 4, -1], dtype=float)
>>> x, exitCode = lgmres(A, b)
>>> print(exitCode) # 0 indicates successful convergence
0
>>> np.allclose(A.dot(x), b)
True
"""
A,M,x,b,postprocess = make_system(A,M,x0,b)
if not np.isfinite(b).all():
raise ValueError("RHS must contain only finite numbers")
matvec = A.matvec
psolve = M.matvec
if outer_v is None:
outer_v = []
axpy, dot, scal = None, None, None
nrm2 = get_blas_funcs('nrm2', [b])
b_norm = nrm2(b)
if b_norm == 0:
b_norm = 1
for k_outer in xrange(maxiter):
r_outer = matvec(x) - b
# -- callback
if callback is not None:
callback(x)
# -- determine input type routines
if axpy is None:
if np.iscomplexobj(r_outer) and not np.iscomplexobj(x):
x = x.astype(r_outer.dtype)
axpy, dot, scal, nrm2 = get_blas_funcs(['axpy', 'dot', 'scal', 'nrm2'],
(x, r_outer))
trtrs = get_lapack_funcs('trtrs', (x, r_outer))
# -- check stopping condition
r_norm = nrm2(r_outer)
if res == "both":
if r_norm <= tol * b_norm or r_norm <= tol:
break
elif res == "absolute":
if r_norm <= tol:
break
elif res == "relative":
if r_norm <= tol * b_norm:
break
else:
raise ValueError("res must be absolute, relative or both")
# -- inner LGMRES iteration
v0 = -psolve(r_outer)
inner_res_0 = nrm2(v0)
if inner_res_0 == 0:
rnorm = nrm2(r_outer)
raise RuntimeError("Preconditioner returned a zero vector; "
"|v| ~ %.1g, |M v| = 0" % rnorm)
v0 = scal(1.0/inner_res_0, v0)
try:
Q, R, B, vs, zs, y = _fgmres(matvec,
v0,
inner_m,
lpsolve=psolve,
atol=tol*b_norm/r_norm,
outer_v=outer_v,
prepend_outer_v=prepend_outer_v)
y *= inner_res_0
if not np.isfinite(y).all():
# Overflow etc. in computation. There's no way to
# recover from this, so we have to bail out.
raise LinAlgError()
except LinAlgError:
# Floating point over/underflow, non-finite result from
# matmul etc. -- report failure.
return postprocess(x), k_outer + 1
# -- GMRES terminated: eval solution
dx = zs[0]*y[0]
for w, yc in zip(zs[1:], y[1:]):
dx = axpy(w, dx, dx.shape[0], yc) # dx += w*yc
# -- Store LGMRES augmentation vectors
nx = nrm2(dx)
if nx > 0:
if store_outer_Av:
q = Q.dot(R.dot(y))
ax = vs[0]*q[0]
for v, qc in zip(vs[1:], q[1:]):
ax = axpy(v, ax, ax.shape[0], qc)
outer_v.append((dx/nx, ax/nx))
else:
outer_v.append((dx/nx, None))
# -- Retain only a finite number of augmentation vectors
while len(outer_v) > outer_k:
del outer_v[0]
# -- Apply step
x += dx
else:
# didn't converge ...
return postprocess(x), maxiter
return postprocess(x), 0
def steepest_descent(A,
b,
x0=None,
tol=1e-5,
maxiter=None,
xtype=None,
M=None,
callback=None,
residuals=None):
'''Steepest descent algorithm
Solves the linear system Ax = b. Left preconditioning is supported.
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by the
preconditioner norm of r_0, or ||r_0||_M.
maxiter : int
maximum number of allowed iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A x = M b.
callback : function
User-supplied function is called after each iteration as
callback(xk), where xk is the current solution vector
residuals : list
residuals contains the residual norm history,
including the initial residual. The preconditioner norm
is used, instead of the Euclidean norm.
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of cg
== =======================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead.
<0 numerical breakdown, or illegal input
== =======================================
Notes
-----
The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
The residual in the preconditioner norm is both used for halting and
returned in the residuals list.
Examples
--------
>>> from pyamg.krylov import steepest_descent
>>> from pyamg.util.linalg import norm
>>> import numpy
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = numpy.ones((A.shape[0],))
>>> (x,flag) = steepest_descent(A,b, maxiter=2, tol=1e-8)
>>> print norm(b - A*x)
7.89436429704
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 137--142, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype=None)
n = len(b)
##
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always',
module='pyamg\.krylov\._steepest_descent')
# determine maxiter
if maxiter is None:
maxiter = int(len(b))
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
# setup method
r = b - A * x
z = M * r
rz = inner(r.conjugate(), z)
# use preconditioner norm
normr = sqrt(rz)
if residuals is not None:
residuals[:] = [normr] #initial residual
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol * normb:
return (postprocess(x), 0)
# Scale tol by ||r_0||_M
if normr != 0.0:
tol = tol * normr
# How often should r be recomputed
recompute_r = 50
iter = 0
while True:
iter = iter + 1
q = A * z
zAz = inner(z.conjugate(), q) # check curvature of A
if zAz < 0.0:
warn(
"\nIndefinite matrix detected in steepest descent, aborting\n")
return (postprocess(x), -1)
alpha = rz / zAz # step size
x = x + alpha * z
if mod(iter, recompute_r) and iter > 0:
r = b - A * x
else:
r = r - alpha * q
z = M * r
rz = inner(r.conjugate(), z)
if rz < 0.0: # check curvature of M
warn(
"\nIndefinite preconditioner detected in steepest descent, aborting\n"
)
return (postprocess(x), -1)
normr = sqrt(rz) # use preconditioner norm
if residuals is not None:
residuals.append(normr)
if callback is not None:
callback(x)
if normr < tol:
return (postprocess(x), 0)
elif rz == 0.0:
# important to test after testing normr < tol. rz == 0.0 is an
# indicator of convergence when r = 0.0
warn(
"\nSingular preconditioner detected in steepest descent, ceasing iterations\n"
)
return (postprocess(x), -1)
if iter == maxiter:
return (postprocess(x), iter)
def gmres_householder(A,
b,
x0=None,
tol=1e-5,
restrt=None,
maxiter=None,
xtype=None,
M=None,
callback=None,
residuals=None):
'''
Generalized Minimum Residual Method (GMRES)
GMRES iteratively refines the initial solution guess to the
system Ax = b
Householder reflections are used for orthogonalization
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n, 1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by the norm
of the initial preconditioned residual
restrt : {None, int}
- if int, restrt is max number of inner iterations
and maxiter is the max number of outer iterations
- if None, do not restart GMRES, and max number of inner iterations
is maxiter
maxiter : {None, int}
- if restrt is None, maxiter is the max number of inner iterations
and GMRES does not restart
- if restrt is int, maxiter is the max number of outer iterations,
and restrt is the max number of inner iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A x = M b.
callback : function
User-supplied function is called after each iteration as
callback( ||rk||_2 ), where rk is the current preconditioned residual
vector
residuals : list
residuals contains the preconditioned residual norm history,
including the initial residual.
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of gmres
== =============================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead. This value
is precisely the order of the Krylov space.
<0 numerical breakdown, or illegal input
== =============================================
Notes
-----
- The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
- For robustness, Householder reflections are used to orthonormalize
the Krylov Space
Givens Rotations are used to provide the residual norm each iteration
Examples
--------
>>> from pyamg.krylov import gmres
>>> from pyamg.util.linalg import norm
>>> import numpy as np
>>> from pyamg.gallery import poisson
>>> A = poisson((10, 10))
>>> b = np.ones((A.shape[0],))
>>> (x, flag) = gmres(A, b, maxiter=2, tol=1e-8, orthog='householder')
>>> print norm(b - A*x)
6.5428213057
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
# Convert inputs to linear system, with error checking
A, M, x, b, postprocess = make_system(A, M, x0, b)
dimen = A.shape[0]
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always',
module='pyamg\.krylov\._gmres_householder')
# Choose type
if not hasattr(A, 'dtype'):
Atype = upcast(x.dtype, b.dtype)
else:
Atype = A.dtype
if not hasattr(M, 'dtype'):
Mtype = upcast(x.dtype, b.dtype)
else:
Mtype = M.dtype
xtype = upcast(Atype, x.dtype, b.dtype, Mtype)
if restrt is not None:
restrt = int(restrt)
if maxiter is not None:
maxiter = int(maxiter)
# Should norm(r) be kept
if residuals == []:
keep_r = True
else:
keep_r = False
# Set number of outer and inner iterations
if restrt:
if maxiter:
max_outer = maxiter
else:
max_outer = 1
if restrt > dimen:
warn('Setting number of inner iterations (restrt) to maximum \
allowed, which is A.shape[0] ')
restrt = dimen
max_inner = restrt
else:
max_outer = 1
if maxiter > dimen:
warn('Setting number of inner iterations (maxiter) to maximum \
allowed, which is A.shape[0] ')
maxiter = dimen
elif maxiter is None:
maxiter = min(dimen, 40)
max_inner = maxiter
# Get fast access to underlying LAPACK routine
[lartg] = get_lapack_funcs(['lartg'], [x])
# Is this a one dimensional matrix?
if dimen == 1:
entry = ravel(A * array([1.0], dtype=xtype))
return (postprocess(b / entry), 0)
# Prep for method
r = b - ravel(A * x)
# Apply preconditioner
r = ravel(M * r)
normr = norm(r)
if keep_r:
residuals.append(normr)
# Check for nan, inf
# if isnan(r).any() or isinf(r).any():
# warn('inf or nan after application of preconditioner')
# return(postprocess(x), -1)
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol * normb:
if callback is not None:
callback(norm(r))
return (postprocess(x), 0)
# Scale tol by ||r_0||_2, we use the preconditioned residual
# because this is left preconditioned GMRES.
if normr != 0.0:
tol = tol * normr
# Use separate variable to track iterations. If convergence fails, we
# cannot simply report niter = (outer-1)*max_outer + inner. Numerical
# error could cause the inner loop to halt while the actual ||r|| > tol.
niter = 0
# Begin GMRES
for outer in range(max_outer):
# Calculate vector w, which defines the Householder reflector
# Take shortcut in calculating,
# w = r + sign(r[1])*||r||_2*e_1
w = r
beta = mysign(w[0]) * normr
w[0] = w[0] + beta
w[:] = w / norm(w)
# Preallocate for Krylov vectors, Householder reflectors and
# Hessenberg matrix
# Space required is O(dimen*max_inner)
# Givens Rotations
Q = zeros((4 * max_inner, ), dtype=xtype)
# upper Hessenberg matrix (made upper tri with Givens Rotations)
H = zeros((max_inner, max_inner), dtype=xtype)
# Householder reflectors
W = zeros((max_inner + 1, dimen), dtype=xtype)
W[0, :] = w
# Multiply r with (I - 2*w*w.T), i.e. apply the Householder reflector
# This is the RHS vector for the problem in the Krylov Space
g = zeros((dimen, ), dtype=xtype)
g[0] = -beta
for inner in range(max_inner):
# Calculate Krylov vector in two steps
# (1) Calculate v = P_j = (I - 2*w*w.T)v, where k = inner
v = -2.0 * conjugate(w[inner]) * w
v[inner] = v[inner] + 1.0
# (2) Calculate the rest, v = P_1*P_2*P_3...P_{j-1}*ej.
# for j in range(inner-1,-1,-1):
# v -= 2.0*dot(conjugate(W[j,:]), v)*W[j,:]
amg_core.apply_householders(v, ravel(W), dimen, inner - 1, -1, -1)
# Calculate new search direction
v = ravel(A * v)
# Apply preconditioner
v = ravel(M * v)
# Check for nan, inf
# if isnan(v).any() or isinf(v).any():
# warn('inf or nan after application of preconditioner')
# return(postprocess(x), -1)
# Factor in all Householder orthogonal reflections on new search
# direction
# for j in range(inner+1):
# v -= 2.0*dot(conjugate(W[j,:]), v)*W[j,:]
amg_core.apply_householders(v, ravel(W), dimen, 0, inner + 1, 1)
# Calculate next Householder reflector, w
# w = v[inner+1:] + sign(v[inner+1])*||v[inner+1:]||_2*e_{inner+1)
# Note that if max_inner = dimen, then this is unnecessary for the
# last inner iteration, when inner = dimen-1. Here we do not need
# to calculate a Householder reflector or Givens rotation because
# nnz(v) is already the desired length, i.e. we do not need to
# zero anything out.
if inner != dimen - 1:
if inner < (max_inner - 1):
w = W[inner + 1, :]
vslice = v[inner + 1:]
alpha = norm(vslice)
if alpha != 0:
alpha = mysign(vslice[0]) * alpha
# do not need the final reflector for future calculations
if inner < (max_inner - 1):
w[inner + 1:] = vslice
w[inner + 1] += alpha
w[:] = w / norm(w)
# Apply new reflector to v
# v = v - 2.0*w*(w.T*v)
v[inner + 1] = -alpha
v[inner + 2:] = 0.0
if inner > 0:
# Apply all previous Givens Rotations to v
amg_core.apply_givens(Q, v, dimen, inner)
# Calculate the next Givens rotation, where j = inner Note that if
# max_inner = dimen, then this is unnecessary for the last inner
# iteration, when inner = dimen-1. Here we do not need to
# calculate a Householder reflector or Givens rotation because
# nnz(v) is already the desired length, i.e. we do not need to zero
# anything out.
if inner != dimen - 1:
if v[inner + 1] != 0:
[c, s, r] = lartg(v[inner], v[inner + 1])
Qblock = array([[c, s], [-conjugate(s), c]], dtype=xtype)
Q[(inner * 4):((inner + 1) * 4)] = ravel(Qblock).copy()
# Apply Givens Rotation to g, the RHS for the linear system
# in the Krylov Subspace. Note that this dot does a matrix
# multiply, not an actual dot product where a conjugate
# transpose is taken
g[inner:inner + 2] = dot(Qblock, g[inner:inner + 2])
# Apply effect of Givens Rotation to v
v[inner] = dot(Qblock[0, :], v[inner:inner + 2])
v[inner + 1] = 0.0
# Write to upper Hessenberg Matrix,
# the LHS for the linear system in the Krylov Subspace
H[:, inner] = v[0:max_inner]
niter += 1
# Don't update normr if last inner iteration, because
# normr is calculated directly after this loop ends.
if inner < max_inner - 1:
normr = abs(g[inner + 1])
if normr < tol:
break
# Allow user access to residual
if callback is not None:
callback(normr)
if keep_r:
residuals.append(normr)
# end inner loop, back to outer loop
# Find best update to x in Krylov Space, V. Solve inner+1 x inner+1
# system. Apparently this is the best way to solve a triangular system
# in the magical world of scipy
# piv = arange(inner+1)
# y = lu_solve((H[0:(inner+1), 0:(inner+1)], piv), g[0:(inner+1)],
# trans=0)
y = sp.linalg.solve(H[0:(inner + 1), 0:(inner + 1)], g[0:(inner + 1)])
# Use Horner like Scheme to map solution, y, back to original space.
# Note that we do not use the last reflector.
update = zeros(x.shape, dtype=xtype)
# for j in range(inner,-1,-1):
# update[j] += y[j]
# # Apply j-th reflector, (I - 2.0*w_j*w_j.T)*upadate
# update -= 2.0*dot(conjugate(W[j,:]), update)*W[j,:]
amg_core.householder_hornerscheme(update, ravel(W), ravel(y), dimen,
inner, -1, -1)
x[:] = x + update
r = b - ravel(A * x)
# Apply preconditioner
r = ravel(M * r)
normr = norm(r)
# Check for nan, inf
# if isnan(r).any() or isinf(r).any():
# warn('inf or nan after application of preconditioner')
# return(postprocess(x), -1)
# Allow user access to residual
if callback is not None:
callback(normr)
if keep_r:
residuals.append(normr)
# Has GMRES stagnated?
indices = (x != 0)
if indices.any():
change = max(abs(update[indices] / x[indices]))
if change < 1e-12:
# No change, halt
return (postprocess(x), -1)
# test for convergence
if normr < tol:
return (postprocess(x), 0)
# end outer loop
return (postprocess(x), niter)
def gmres_mgs(A, b, x0=None, tol=1e-5, restrt=None, maxiter=None, xtype=None,
M=None, callback=None, residuals=None, reorth=False):
'''
Generalized Minimum Residual Method (GMRES)
GMRES iteratively refines the initial solution guess to the system
Ax = b
Modified Gram-Schmidt version
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by the norm
of the initial preconditioned residual
restrt : {None, int}
- if int, restrt is max number of inner iterations
and maxiter is the max number of outer iterations
- if None, do not restart GMRES, and max number of inner iterations
is maxiter
maxiter : {None, int}
- if restrt is None, maxiter is the max number of inner iterations
and GMRES does not restart
- if restrt is int, maxiter is the max number of outer iterations,
and restrt is the max number of inner iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A x = M b.
callback : function
User-supplied function is called after each iteration as
callback( ||rk||_2 ), where rk is the current preconditioned residual
vector
residuals : list
residuals contains the preconditioned residual norm history,
including the initial residual.
reorth : boolean
If True, then a check is made whether to re-orthogonalize the Krylov
space each GMRES iteration
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of gmres
== =============================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead. This value
is precisely the order of the Krylov space.
<0 numerical breakdown, or illegal input
== =============================================
Notes
-----
- The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
- For robustness, modified Gram-Schmidt is used to orthogonalize the
Krylov Space Givens Rotations are used to provide the residual norm
each iteration
Examples
--------
>>> from pyamg.krylov import gmres
>>> from pyamg.util.linalg import norm
>>> import numpy as np
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = np.ones((A.shape[0],))
>>> (x,flag) = gmres(A,b, maxiter=2, tol=1e-8, orthog='mgs')
>>> print norm(b - A*x)
>>> 6.5428213057
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003
http://www-users.cs.umn.edu/~saad/books.html
.. [2] C. T. Kelley, http://www4.ncsu.edu/~ctk/matlab_roots.html
'''
# Convert inputs to linear system, with error checking
A, M, x, b, postprocess = make_system(A, M, x0, b)
dimen = A.shape[0]
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always', module='pyamg\.krylov\._gmres_mgs')
# Choose type
if not hasattr(A, 'dtype'):
Atype = upcast(x.dtype, b.dtype)
else:
Atype = A.dtype
if not hasattr(M, 'dtype'):
Mtype = upcast(x.dtype, b.dtype)
else:
Mtype = M.dtype
xtype = upcast(Atype, x.dtype, b.dtype, Mtype)
if restrt is not None:
restrt = int(restrt)
if maxiter is not None:
maxiter = int(maxiter)
# Get fast access to underlying BLAS routines
# dotc is the conjugate dot, dotu does no conjugation
[lartg] = get_lapack_funcs(['lartg'], [x] )
if iscomplexobj(zeros((1,), dtype=xtype)):
[axpy, dotu, dotc, scal] =\
get_blas_funcs(['axpy', 'dotu', 'dotc', 'scal'], [x])
else:
# real type
[axpy, dotu, dotc, scal] =\
get_blas_funcs(['axpy', 'dot', 'dot', 'scal'], [x])
# Make full use of direct access to BLAS by defining own norm
def norm(z):
return sqrt(real(dotc(z, z)))
# Should norm(r) be kept
if residuals == []:
keep_r = True
else:
keep_r = False
# Set number of outer and inner iterations
if restrt:
if maxiter:
max_outer = maxiter
else:
max_outer = 1
if restrt > dimen:
warn('Setting number of inner iterations (restrt) to maximum\
allowed, which is A.shape[0] ')
restrt = dimen
max_inner = restrt
else:
max_outer = 1
if maxiter > dimen:
warn('Setting number of inner iterations (maxiter) to maximum\
allowed, which is A.shape[0] ')
maxiter = dimen
elif maxiter is None:
maxiter = min(dimen, 40)
max_inner = maxiter
# Is this a one dimensional matrix?
if dimen == 1:
entry = ravel(A*array([1.0], dtype=xtype))
return (postprocess(b/entry), 0)
# Prep for method
r = b - ravel(A*x)
# Apply preconditioner
r = ravel(M*r)
normr = norm(r)
if keep_r:
residuals.append(normr)
# Check for nan, inf
# if isnan(r).any() or isinf(r).any():
# warn('inf or nan after application of preconditioner')
# return(postprocess(x), -1)
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol*normb:
if callback is not None:
callback(norm(r))
return (postprocess(x), 0)
# Scale tol by ||r_0||_2, we use the preconditioned residual
# because this is left preconditioned GMRES.
if normr != 0.0:
tol = tol*normr
# Use separate variable to track iterations. If convergence fails, we
# cannot simply report niter = (outer-1)*max_outer + inner. Numerical
# error could cause the inner loop to halt while the actual ||r|| > tol.
niter = 0
# Begin GMRES
for outer in range(max_outer):
# Preallocate for Givens Rotations, Hessenberg matrix and Krylov Space
# Space required is O(dimen*max_inner).
# NOTE: We are dealing with row-major matrices, so we traverse in a
# row-major fashion,
# i.e., H and V's transpose is what we store.
Q = [] # Givens Rotations
# Upper Hessenberg matrix, which is then
# converted to upper tri with Givens Rots
H = zeros((max_inner+1, max_inner+1), dtype=xtype)
V = zeros((max_inner+1, dimen), dtype=xtype) # Krylov Space
# vs store the pointers to each column of V.
# This saves a considerable amount of time.
vs = []
# v = r/normr
V[0, :] = scal(1.0/normr, r)
vs.append(V[0, :])
# This is the RHS vector for the problem in the Krylov Space
g = zeros((dimen,), dtype=xtype)
g[0] = normr
for inner in range(max_inner):
# New Search Direction
v = V[inner+1, :]
v[:] = ravel(M*(A*vs[-1]))
vs.append(v)
normv_old = norm(v)
# Check for nan, inf
# if isnan(V[inner+1, :]).any() or isinf(V[inner+1, :]).any():
# warn('inf or nan after application of preconditioner')
# return(postprocess(x), -1)
# Modified Gram Schmidt
for k in range(inner+1):
vk = vs[k]
alpha = dotc(vk, v)
H[inner, k] = alpha
v[:] = axpy(vk, v, dimen, -alpha)
normv = norm(v)
H[inner, inner+1] = normv
# Re-orthogonalize
if (reorth is True) and (normv_old == normv_old + 0.001*normv):
for k in range(inner+1):
vk = vs[k]
alpha = dotc(vk, v)
H[inner, k] = H[inner, k] + alpha
v[:] = axpy(vk, v, dimen, -alpha)
# Check for breakdown
if H[inner, inner+1] != 0.0:
v[:] = scal(1.0/H[inner, inner+1], v)
# Apply previous Givens rotations to H
if inner > 0:
apply_givens(Q, H[inner, :], inner)
# Calculate and apply next complex-valued Givens Rotation
# ==> Note that if max_inner = dimen, then this is unnecessary
# for the last inner
# iteration, when inner = dimen-1.
if inner != dimen-1:
if H[inner, inner+1] != 0:
[c, s, r] = lartg(H[inner, inner], H[inner, inner+1])
Qblock = array([[c, s], [-conjugate(s), c]], dtype=xtype)
Q.append(Qblock)
# Apply Givens Rotation to g,
# the RHS for the linear system in the Krylov Subspace.
g[inner:inner+2] = sp.dot(Qblock, g[inner:inner+2])
# Apply effect of Givens Rotation to H
H[inner, inner] = dotu(Qblock[0, :],
H[inner, inner:inner+2])
H[inner, inner+1] = 0.0
niter += 1
# Don't update normr if last inner iteration, because
# normr is calculated directly after this loop ends.
if inner < max_inner-1:
normr = abs(g[inner+1])
if normr < tol:
break
# Allow user access to residual
if callback is not None:
callback(normr)
if keep_r:
residuals.append(normr)
# end inner loop, back to outer loop
# Find best update to x in Krylov Space V. Solve inner x inner system.
y = sp.linalg.solve(H[0:inner+1, 0:inner+1].T, g[0:inner+1])
update = ravel(sp.mat(V[:inner+1, :]).T*y.reshape(-1, 1))
x = x + update
r = b - ravel(A*x)
# Apply preconditioner
r = ravel(M*r)
normr = norm(r)
# Check for nan, inf
# if isnan(r).any() or isinf(r).any():
# warn('inf or nan after application of preconditioner')
# return(postprocess(x), -1)
# Allow user access to residual
if callback is not None:
callback(normr)
if keep_r:
residuals.append(normr)
# Has GMRES stagnated?
indices = (x != 0)
if indices.any():
change = max(abs(update[indices] / x[indices]))
if change < 1e-12:
# No change, halt
return (postprocess(x), -1)
# test for convergence
if normr < tol:
return (postprocess(x), 0)
# end outer loop
return (postprocess(x), niter)
def gcrotmk(A,
b,
x0=None,
tol=1e-5,
maxiter=1000,
M=None,
callback=None,
m=20,
k=None,
CU=None,
discard_C=False,
truncate='oldest'):
"""
Solve a matrix equation using flexible GCROT(m,k) algorithm.
Parameters
----------
A : {sparse matrix, dense matrix, LinearOperator}
The real or complex N-by-N matrix of the linear system.
b : {array, matrix}
Right hand side of the linear system. Has shape (N,) or (N,1).
x0 : {array, matrix}
Starting guess for the solution.
tol : float, optional
Tolerance to achieve. The algorithm terminates when either the relative
or the absolute residual is below `tol`.
maxiter : int, optional
Maximum number of iterations. Iteration will stop after maxiter
steps even if the specified tolerance has not been achieved.
M : {sparse matrix, dense matrix, LinearOperator}, optional
Preconditioner for A. The preconditioner should approximate the
inverse of A. gcrotmk is a 'flexible' algorithm and the preconditioner
can vary from iteration to iteration. Effective preconditioning
dramatically improves the rate of convergence, which implies that
fewer iterations are needed to reach a given error tolerance.
callback : function, optional
User-supplied function to call after each iteration. It is called
as callback(xk), where xk is the current solution vector.
m : int, optional
Number of inner FGMRES iterations per each outer iteration.
Default: 20
k : int, optional
Number of vectors to carry between inner FGMRES iterations.
According to [2]_, good values are around m.
Default: m
CU : list of tuples, optional
List of tuples ``(c, u)`` which contain the columns of the matrices
C and U in the GCROT(m,k) algorithm. For details, see [2]_.
The list given and vectors contained in it are modified in-place.
If not given, start from empty matrices. The ``c`` elements in the
tuples can be ``None``, in which case the vectors are recomputed
via ``c = A u`` on start and orthogonalized as described in [3]_.
discard_C : bool, optional
Discard the C-vectors at the end. Useful if recycling Krylov subspaces
for different linear systems.
truncate : {'oldest', 'smallest'}, optional
Truncation scheme to use. Drop: oldest vectors, or vectors with
smallest singular values using the scheme discussed in [1,2].
See [2]_ for detailed comparison.
Default: 'oldest'
Returns
-------
x : array or matrix
The solution found.
info : int
Provides convergence information:
* 0 : successful exit
* >0 : convergence to tolerance not achieved, number of iterations
References
----------
.. [1] E. de Sturler, ''Truncation strategies for optimal Krylov subspace
methods'', SIAM J. Numer. Anal. 36, 864 (1999).
.. [2] J.E. Hicken and D.W. Zingg, ''A simplified and flexible variant
of GCROT for solving nonsymmetric linear systems'',
SIAM J. Sci. Comput. 32, 172 (2010).
.. [3] M.L. Parks, E. de Sturler, G. Mackey, D.D. Johnson, S. Maiti,
''Recycling Krylov subspaces for sequences of linear systems'',
SIAM J. Sci. Comput. 28, 1651 (2006).
"""
A, M, x, b, postprocess = make_system(A, M, x0, b)
if not np.isfinite(b).all():
raise ValueError("RHS must contain only finite numbers")
if truncate not in ('oldest', 'smallest'):
raise ValueError("Invalid value for 'truncate': %r" % (truncate, ))
matvec = A.matvec
psolve = M.matvec
if CU is None:
CU = []
if k is None:
k = m
axpy, dot, scal = None, None, None
r = b - matvec(x)
axpy, dot, scal, nrm2 = get_blas_funcs(['axpy', 'dot', 'scal', 'nrm2'],
(x, r))
b_norm = nrm2(b)
if b_norm == 0:
b_norm = 1
if discard_C:
CU[:] = [(None, u) for c, u in CU]
# Reorthogonalize old vectors
if CU:
# Sort already existing vectors to the front
CU.sort(key=lambda cu: cu[0] is not None)
# Fill-in missing ones
C = np.empty((A.shape[0], len(CU)), dtype=r.dtype, order='F')
us = []
j = 0
while CU:
# More memory-efficient: throw away old vectors as we go
c, u = CU.pop(0)
if c is None:
c = matvec(u)
C[:, j] = c
j += 1
us.append(u)
# Orthogonalize
Q, R, P = qr(C, overwrite_a=True, mode='economic', pivoting=True)
del C
# C := Q
cs = list(Q.T)
# U := U P R^-1, back-substitution
new_us = []
for j in xrange(len(cs)):
u = us[P[j]]
for i in xrange(j):
u = axpy(us[P[i]], u, u.shape[0], -R[i, j])
if abs(R[j, j]) < 1e-12 * abs(R[0, 0]):
# discard rest of the vectors
break
u = scal(1.0 / R[j, j], u)
new_us.append(u)
# Form the new CU lists
CU[:] = list(zip(cs, new_us))[::-1]
if CU:
axpy, dot = get_blas_funcs(['axpy', 'dot'], (r, ))
# Solve first the projection operation with respect to the CU
# vectors. This corresponds to modifying the initial guess to
# be
#
# x' = x + U y
# y = argmin_y || b - A (x + U y) ||^2
#
# The solution is y = C^H (b - A x)
for c, u in CU:
yc = dot(c, r)
x = axpy(u, x, x.shape[0], yc)
r = axpy(c, r, r.shape[0], -yc)
# GCROT main iteration
for j_outer in xrange(maxiter):
# -- callback
if callback is not None:
callback(x)
beta = nrm2(r)
# -- check stopping condition
if beta <= max(tol, tol * b_norm):
j_outer = -1
break
ml = m + max(k - len(CU), 0)
cs = [c for c, u in CU]
try:
Q, R, B, vs, zs, y = _fgmres(matvec,
r / beta,
ml,
rpsolve=psolve,
atol=tol * b_norm / beta,
cs=cs)
y *= beta
except LinAlgError:
# Floating point over/underflow, non-finite result from
# matmul etc. -- report failure.
break
#
# At this point,
#
# [A U, A Z] = [C, V] G; G = [ I B ]
# [ 0 H ]
#
# where [C, V] has orthonormal columns, and r = beta v_0. Moreover,
#
# || b - A (x + Z y + U q) ||_2 = || r - C B y - V H y - C q ||_2 = min!
#
# from which y = argmin_y || beta e_1 - H y ||_2, and q = -B y
#
#
# GCROT(m,k) update
#
# Define new outer vectors
# ux := (Z - U B) y
ux = zs[0] * y[0]
for z, yc in zip(zs[1:], y[1:]):
ux = axpy(z, ux, ux.shape[0], yc) # ux += z*yc
by = B.dot(y)
for cu, byc in zip(CU, by):
c, u = cu
ux = axpy(u, ux, ux.shape[0], -byc) # ux -= u*byc
# cx := V H y
hy = Q.dot(R.dot(y))
cx = vs[0] * hy[0]
for v, hyc in zip(vs[1:], hy[1:]):
cx = axpy(v, cx, cx.shape[0], hyc) # cx += v*hyc
# Normalize cx, maintaining cx = A ux
# This new cx is orthogonal to the previous C, by construction
try:
alpha = 1 / nrm2(cx)
if not np.isfinite(alpha):
raise FloatingPointError()
except (FloatingPointError, ZeroDivisionError):
# Cannot update, so skip it
continue
cx = scal(alpha, cx)
ux = scal(alpha, ux)
# Update residual and solution
gamma = dot(cx, r)
r = axpy(cx, r, r.shape[0], -gamma) # r -= gamma*cx
x = axpy(ux, x, x.shape[0], gamma) # x += gamma*ux
# Truncate CU
if truncate == 'oldest':
while len(CU) >= k and CU:
del CU[0]
elif truncate == 'smallest':
if len(CU) >= k and CU:
# cf. [1,2]
D = solve(R[:-1, :].T, B.T).T
W, sigma, V = svd(D)
# C := C W[:,:k-1], U := U W[:,:k-1]
new_CU = []
for j, w in enumerate(W[:, :k - 1].T):
c, u = CU[0]
c = c * w[0]
u = u * w[0]
for cup, wp in zip(CU[1:], w[1:]):
cp, up = cup
c = axpy(cp, c, c.shape[0], wp)
u = axpy(up, u, u.shape[0], wp)
# Reorthogonalize at the same time; not necessary
# in exact arithmetic, but floating point error
# tends to accumulate here
for cp, up in new_CU:
alpha = dot(cp, c)
c = axpy(cp, c, c.shape[0], -alpha)
u = axpy(up, u, u.shape[0], -alpha)
alpha = nrm2(c)
c = scal(1.0 / alpha, c)
u = scal(1.0 / alpha, u)
new_CU.append((c, u))
CU[:] = new_CU
# Add new vector to CU
CU.append((cx, ux))
# Include the solution vector to the span
CU.append((None, x.copy()))
if discard_C:
CU[:] = [(None, uz) for cz, uz in CU]
return postprocess(x), j_outer + 1
def bicgstab(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None,
callback=None, residuals=None):
'''Biconjugate Gradient Algorithm with Stabilization
Solves the linear system Ax = b. Left preconditioning is supported.
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by ||r_0||_2
maxiter : int
maximum number of allowed iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A A.H x = M b.
callback : function
User-supplied function is called after each iteration as
callback(xk), where xk is the current solution vector
residuals : list
residuals has the residual norm history,
including the initial residual, appended to it
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of bicgstab
== ======================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead.
<0 numerical breakdown, or illegal input
== ======================================
Notes
-----
The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
Examples
--------
>>> from pyamg.krylov.bicgstab import bicgstab
>>> from pyamg.util.linalg import norm
>>> import numpy as np
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = np.ones((A.shape[0],))
>>> (x,flag) = bicgstab(A,b, maxiter=2, tol=1e-8)
>>> print norm(b - A*x)
4.68163045309
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 231-234, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
# Convert inputs to linear system, with error checking
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype)
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always', module='pyamg\.krylov\._bicgstab')
# Check iteration numbers
if maxiter is None:
maxiter = len(x) + 5
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
# Prep for method
r = b - A*x
normr = norm(r)
if residuals is not None:
residuals[:] = [normr]
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol*normb:
return (postprocess(x), 0)
# Scale tol by ||r_0||_2
if normr != 0.0:
tol = tol*normr
# Is this a one dimensional matrix?
if A.shape[0] == 1:
entry = np.ravel(A*np.array([1.0], dtype=xtype))
return (postprocess(b/entry), 0)
rstar = r.copy()
p = r.copy()
rrstarOld = np.inner(rstar.conjugate(), r)
iter = 0
# Begin BiCGStab
while True:
Mp = M*p
AMp = A*Mp
# alpha = (r_j, rstar) / (A*M*p_j, rstar)
alpha = rrstarOld/np.inner(rstar.conjugate(), AMp)
# s_j = r_j - alpha*A*M*p_j
s = r - alpha*AMp
Ms = M*s
AMs = A*Ms
# omega = (A*M*s_j, s_j)/(A*M*s_j, A*M*s_j)
omega = np.inner(AMs.conjugate(), s)/np.inner(AMs.conjugate(), AMs)
# x_{j+1} = x_j + alpha*M*p_j + omega*M*s_j
x = x + alpha*Mp + omega*Ms
# r_{j+1} = s_j - omega*A*M*s
r = s - omega*AMs
# beta_j = (r_{j+1}, rstar)/(r_j, rstar) * (alpha/omega)
rrstarNew = np.inner(rstar.conjugate(), r)
beta = (rrstarNew / rrstarOld) * (alpha / omega)
rrstarOld = rrstarNew
# p_{j+1} = r_{j+1} + beta*(p_j - omega*A*M*p)
p = r + beta*(p - omega*AMp)
iter += 1
normr = norm(r)
if residuals is not None:
residuals.append(normr)
if callback is not None:
callback(x)
if normr < tol:
return (postprocess(x), 0)
if iter == maxiter:
return (postprocess(x), iter)
def cgnr(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None,
callback=None, residuals=None):
'''Conjugate Gradient, Normal Residual algorithm
Applies CG to the normal equations, A.H A x = b. Left preconditioning
is supported. Note that unless A is well-conditioned, the use of
CGNR is inadvisable
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by ||r_0||_2
maxiter : int
maximum number of allowed iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A.H A x = b.
callback : function
User-supplied function is called after each iteration as
callback(xk), where xk is the current solution vector
residuals : list
residuals has the residual norm history,
including the initial residual, appended to it
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of cgnr
== =======================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead.
<0 numerical breakdown, or illegal input
== =======================================
Notes
-----
The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
Examples
--------
>>> from pyamg.krylov.cgnr import cgnr
>>> from pyamg.util.linalg import norm
>>> import numpy as np
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = np.ones((A.shape[0],))
>>> (x,flag) = cgnr(A,b, maxiter=2, tol=1e-8)
>>> print norm(b - A*x)
9.3910201849
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 276-7, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
# Store the conjugate transpose explicitly as it will be used much later on
if isspmatrix(A):
AH = A.H
else:
# TODO avoid doing this since A may be a different sparse type
AH = aslinearoperator(asmatrix(A).H)
# Convert inputs to linear system, with error checking
A, M, x, b, postprocess = make_system(A, M, x0, b)
dimen = A.shape[0]
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always', module='pyamg\.krylov\._cgnr')
# Choose type
if not hasattr(A, 'dtype'):
Atype = upcast(x.dtype, b.dtype)
else:
Atype = A.dtype
if not hasattr(M, 'dtype'):
Mtype = upcast(x.dtype, b.dtype)
else:
Mtype = M.dtype
xtype = upcast(Atype, x.dtype, b.dtype, Mtype)
# Should norm(r) be kept
if residuals == []:
keep_r = True
else:
keep_r = False
# How often should r be recomputed
recompute_r = 8
# Check iteration numbers. CGNR suffers from loss of orthogonality quite
# easily, so we arbitrarily let the method go up to 130% over the
# theoretically necessary limit of maxiter=dimen
if maxiter is None:
maxiter = int(ceil(1.3*dimen)) + 2
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
elif maxiter > (1.3*dimen):
warn('maximum allowed inner iterations (maxiter) are the 130% times \
the number of dofs')
maxiter = int(ceil(1.3*dimen)) + 2
# Prep for method
r = b - A*x
rhat = AH*r
normr = norm(r)
if keep_r:
residuals.append(normr)
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol*normb:
if callback is not None:
callback(x)
return (postprocess(x), 0)
# Scale tol by ||r_0||_2
if normr != 0.0:
tol = tol*normr
# Begin CGNR
# Apply preconditioner and calculate initial search direction
z = M*rhat
p = z.copy()
old_zr = inner(z.conjugate(), rhat)
for iter in range(maxiter):
# w_j = A p_j
w = A*p
# alpha = (z_j, rhat_j) / (w_j, w_j)
alpha = old_zr / inner(w.conjugate(), w)
# x_{j+1} = x_j + alpha*p_j
x += alpha*p
# r_{j+1} = r_j - alpha*w_j
if mod(iter, recompute_r) and iter > 0:
r -= alpha*w
else:
r = b - A*x
# rhat_{j+1} = A.H*r_{j+1}
rhat = AH*r
# z_{j+1} = M*r_{j+1}
z = M*rhat
# beta = (z_{j+1}, rhat_{j+1}) / (z_j, rhat_j)
new_zr = inner(z.conjugate(), rhat)
beta = new_zr / old_zr
old_zr = new_zr
# p_{j+1} = A.H*z_{j+1} + beta*p_j
p *= beta
p += z
# Allow user access to residual
if callback is not None:
callback(x)
# test for convergence
normr = norm(r)
if keep_r:
residuals.append(normr)
if normr < tol:
return (postprocess(x), 0)
# end loop
return (postprocess(x), iter+1)
def gmres(A,
b,
x0=None,
tol=1e-5,
restrt=None,
maxiter=None,
xtype=None,
M=None,
callback=None,
residuals=None):
'''
Generalized Minimum Residual Method (GMRES)
GMRES iteratively refines the initial solution guess to the system Ax = b
For robustness, Householder reflections are used to orthonormalize the Krylov Space
Givens Rotations are used to provide the residual norm each iteration
Parameters
----------
A : array, matrix or sparse matrix
n x n, linear system to solve
b : array
n x 1, right hand side
x0 : array
n x 1, initial guess
default is a vector of zeros
tol : float
convergence tolerance
restrt : int
number of restarts
total iterations = restrt*maxiter
maxiter : int
maximum number of allowed inner iterations
xtype : type
dtype for the solution
M : matrix-like
n x n, inverted preconditioner, i.e. solve M A x = b.
For preconditioning with a mat-vec routine, set
A.psolve = func, where func := M y
callback : function
callback( ||resid||_2 ) is called each iteration,
residuals : {None, empty-list}
If empty-list, residuals holds the residual norm history,
including the initial residual, upon completion
Returns
-------
(xNew, info)
xNew -- an updated guess to the solution of Ax = b
info -- halting status of gmres
0 : successful exit
>0 : convergence to tolerance not achieved,
return iteration count instead. This value
is precisely the order of the Krylov space.
<0 : numerical breakdown, or illegal input
Notes
-----
Examples
--------
>>>from pyamg.krylov import *
>>>from scipy import rand
>>>import pyamg
>>>A = pyamg.poisson((50,50))
>>>b = rand(A.shape[0],)
>>>(x,flag) = gmres(A,b)
>>>print pyamg.util.linalg.norm(b - A*x)
References
----------
Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003
'''
# Convert inputs to linear system, with error checking
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype)
dimen = A.shape[0]
# Choose type
xtype = upcast(A.dtype, x.dtype, b.dtype, M.dtype)
# We assume henceforth that shape=(n,) for all arrays
b = ravel(array(b, xtype))
x = ravel(array(x, xtype))
# Should norm(r) be kept
if residuals == []:
keep_r = True
else:
keep_r = False
# check number of iterations
if restrt == None:
restrt = 1
elif restrt < 1:
raise ValueError('Number of restarts must be positive')
if maxiter == None:
maxiter = int(max(ceil(dimen / restrt)))
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
elif maxiter > dimen:
warn(
'maximimum allowed inner iterations (maxiter) are the number of degress of freedom'
)
maxiter = dimen
# Scale tol by normb
normb = linalg.norm(b)
if normb == 0:
pass
# if callback != None:
# callback(0.0)
#
# return (postprocess(zeros((dimen,)), dtype=xtype),0)
else:
tol = tol * normb
# Is this a one dimensional matrix?
if dimen == 1:
entry = ravel(A * array([1.0], dtype=xtype))
return (postprocess(b / entry), 0)
# Prep for method
r = b - ravel(A * x)
normr = linalg.norm(r)
if keep_r:
residuals.append(normr)
# Is initial guess sufficient?
if normr <= tol:
if callback != None:
callback(norm(r))
return (postprocess(x), 0)
#Apply preconditioner
r = ravel(M * r)
normr = linalg.norm(r)
# Check for nan, inf
if any(isnan(r)) or any(isinf(r)):
warn('inf or nan after application of preconditioner')
return (postprocess(x), -1)
# Use separate variable to track iterations. If convergence fails, we cannot
# simply report niter = (outer-1)*maxiter + inner. Numerical error could cause
# the inner loop to halt before reaching maxiter while the actual ||r|| > tol.
niter = 0
# Begin GMRES
for outer in range(restrt):
# Calculate vector w, which defines the Householder reflector
# Take shortcut in calculating,
# w = r + sign(r[1])*||r||_2*e_1
w = r
beta = mysign(w[0]) * normr
w[0] += beta
w = w / linalg.norm(w)
# Preallocate for Krylov vectors, Householder reflectors and Hessenberg matrix
# Space required is O(dimen*maxiter)
H = zeros(
(maxiter, maxiter), dtype=xtype
) # upper Hessenberg matrix (actually made upper tri with Given's Rotations)
W = zeros((dimen, maxiter), dtype=xtype) # Householder reflectors
W[:, 0] = w
# Multiply r with (I - 2*w*w.T), i.e. apply the Householder reflector
# This is the RHS vector for the problem in the Krylov Space
g = zeros((dimen, ), dtype=xtype)
g[0] = -beta
for inner in range(maxiter):
# Calcute Krylov vector in two steps
# (1) Calculate v = P_j = (I - 2*w*w.T)v, where k = inner
v = -2.0 * conjugate(w[inner]) * w
v[inner] += 1.0
# (2) Calculate the rest, v = P_1*P_2*P_3...P_{j-1}*ej.
for j in range(inner - 1, -1, -1):
v = v - 2.0 * dot(conjugate(W[:, j]), v) * W[:, j]
# Calculate new search direction
v = ravel(A * v)
#Apply preconditioner
v = ravel(M * v)
# Check for nan, inf
if any(isnan(v)) or any(isinf(v)):
warn('inf or nan after application of preconditioner')
return (postprocess(x), -1)
# Factor in all Householder orthogonal reflections on new search direction
for j in range(inner + 1):
v = v - 2.0 * dot(conjugate(W[:, j]), v) * W[:, j]
# Calculate next Householder reflector, w
# w = v[inner+1:] + sign(v[inner+1])*||v[inner+1:]||_2*e_{inner+1)
# Note that if maxiter = dimen, then this is unnecessary for the last inner
# iteration, when inner = dimen-1. Here we do not need to calculate a
# Householder reflector or Given's rotation because nnz(v) is already the
# desired length, i.e. we do not need to zero anything out.
if inner != dimen - 1:
w = zeros((dimen, ), dtype=xtype)
vslice = v[inner + 1:]
alpha = linalg.norm(vslice)
if alpha != 0:
alpha = mysign(vslice[0]) * alpha
# We do not need the final reflector for future calculations
if inner < (maxiter - 1):
w[inner + 1:] = vslice
w[inner + 1] += alpha
w = w / linalg.norm(w)
W[:, inner + 1] = w
# Apply new reflector to v
# v = v - 2.0*w*(w.T*v)
v[inner + 1] = -alpha
v[inner + 2:] = 0.0
# Apply all previous Given's Rotations to v
if inner == 0:
# Q will store the cumulative effect of all Given's Rotations
Q = scipy.sparse.eye(dimen, dimen, format='csr', dtype=xtype)
# Declare initial Qj, which will be the current Given's Rotation
rowptr = hstack((array([0, 2, 4],
int), arange(5, dimen + 3, dtype=int)))
colindices = hstack((array([0, 1, 0, 1],
int), arange(2, dimen, dtype=int)))
data = ones((dimen + 2, ), dtype=xtype)
Qj = csr_matrix((data, colindices, rowptr),
shape=(dimen, dimen),
dtype=xtype)
else:
# Could avoid building a global Given's Rotation, by storing
# and applying each 2x2 matrix individually.
# But that would require looping, the bane of wonderful Python
Q = Qj * Q
v = Q * v
# Calculate Qj, the next Given's rotation, where j = inner
# Note that if maxiter = dimen, then this is unnecessary for the last inner
# iteration, when inner = dimen-1. Here we do not need to calculate a
# Householder reflector or Given's rotation because nnz(v) is already the
# desired length, i.e. we do not need to zero anything out.
if inner != dimen - 1:
if v[inner + 1] != 0:
# Calculate terms for complex 2x2 Given's Rotation
# Note that abs(x) takes the complex modulus
h1 = v[inner]
h2 = v[inner + 1]
h1_mag = abs(h1)
h2_mag = abs(h2)
if h1_mag < h2_mag:
mu = h1 / h2
tau = conjugate(mu) / abs(mu)
else:
mu = h2 / h1
tau = mu / abs(mu)
denom = sqrt(h1_mag**2 + h2_mag**2)
c = h1_mag / denom
s = h2_mag * tau / denom
Qblock = array([[c, conjugate(s)], [-s, c]], dtype=xtype)
# Modify Qj in csr-format so that it represents the current
# global Given's Rotation equivalent to Qblock
if inner != 0:
Qj.data[inner - 1] = 1.0
Qj.indices[inner - 1] = inner - 1
Qj.indptr[inner - 1] = inner - 1
Qj.data[inner:inner + 4] = ravel(Qblock)
Qj.indices[inner:inner +
4] = [inner, inner + 1, inner, inner + 1]
Qj.indptr[inner:inner + 3] = [inner, inner + 2, inner + 4]
# Apply Given's Rotation to g,
# the RHS for the linear system in the Krylov Subspace.
# Note that this dot does a matrix multiply, not an actual
# dot product where a conjugate transpose is taken
g[inner:inner + 2] = dot(Qblock, g[inner:inner + 2])
# Apply effect of Given's Rotation to v
v[inner] = dot(Qblock[0, :], v[inner:inner + 2])
v[inner + 1] = 0.0
# Write to upper Hessenberg Matrix,
# the LHS for the linear system in the Krylov Subspace
H[:, inner] = v[0:maxiter]
# Don't update normr if last inner iteration, because
# normr is calculated directly after this loop ends.
if inner < maxiter - 1:
normr = abs(g[inner + 1])
if normr < tol:
break
# Allow user access to residual
if callback != None:
callback(normr)
if keep_r:
residuals.append(normr)
niter += 1
# end inner loop, back to outer loop
# Find best update to x in Krylov Space, V. Solve inner+1 x inner+1 system.
# Apparently this is the best way to solve a triangular
# system in the magical world of scipy
piv = arange(inner + 1)
y = lu_solve((H[0:(inner + 1), 0:(inner + 1)], piv),
g[0:(inner + 1)],
trans=0)
# Use Horner like Scheme to map solution, y, back to original space.
# Note that we do not use the last reflector.
update = zeros(x.shape, dtype=xtype)
for j in range(inner, -1, -1):
update[j] += y[j]
# Apply j-th reflector, (I - 2.0*w_j*w_j.T)*upadate
update = update - 2.0 * dot(conjugate(W[:, j]), update) * W[:, j]
x = x + update
r = b - ravel(A * x)
#Apply preconditioner
r = ravel(M * r)
normr = linalg.norm(r)
# Check for nan, inf
if any(isnan(r)) or any(isinf(r)):
warn('inf or nan after application of preconditioner')
return (postprocess(x), -1)
# Allow user access to residual
if callback != None:
callback(normr)
if keep_r:
residuals.append(normr)
# Has GMRES stagnated?
indices = (x != 0)
if indices.any():
change = max(abs(update[indices] / x[indices]))
if change < 1e-12:
# No change, halt
return (postprocess(x), -1)
# test for convergence
if normr < tol:
return (postprocess(x), 0)
# end outer loop
return (postprocess(x), niter)
def presid_gmres(A,
b,
verbose,
x0=None,
tol=1e-05,
restart=None,
maxiter=None,
M=None):
callback = gmres_counter(verbose)
A, M, x, b, postprocess = make_system(A, M, x0, b)
n = len(b)
if maxiter is None:
maxiter = n * 10
if restart is None:
restart = 20
restart = min(restart, n)
matvec = A.matvec
psolve = M.matvec
ltr = _type_conv[x.dtype.char]
revcom = getattr(_iterative, ltr + 'gmresrevcom')
bnrm2 = np.linalg.norm(b)
Mb_nrm2 = np.linalg.norm(psolve(b))
get_residual = lambda: np.linalg.norm(matvec(x) - b)
atol = tol
if bnrm2 == 0:
return postprocess(b), 0
# Tolerance passed to GMRESREVCOM applies to the inner iteration
# and deals with the left-preconditioned residual.
ptol_max_factor = 1.0
ptol = Mb_nrm2 * min(ptol_max_factor, atol / bnrm2)
resid = np.nan
presid = np.nan
ndx1 = 1
ndx2 = -1
# Use _aligned_zeros to work around a f2py bug in Numpy 1.9.1
work = _aligned_zeros((6 + restart) * n, dtype=x.dtype)
work2 = _aligned_zeros((restart + 1) * (2 * restart + 2), dtype=x.dtype)
ijob = 1
info = 0
ftflag = True
iter_ = maxiter
old_ijob = ijob
first_pass = True
resid_ready = False
iter_num = 1
while True:
### begin my modifications
if presid / bnrm2 < atol:
resid = presid / bnrm2
info = 1
if info: ptol = 10000
### end my modifications
x, iter_, presid, info, ndx1, ndx2, sclr1, sclr2, ijob = \
revcom(b, x, restart, work, work2, iter_, presid, info, ndx1, ndx2, ijob, ptol)
slice1 = slice(ndx1 - 1, ndx1 - 1 + n)
slice2 = slice(ndx2 - 1, ndx2 - 1 + n)
if (ijob == -1): # gmres success, update last residual
if resid_ready and callback is not None:
callback(presid / bnrm2)
resid_ready = False
break
elif (ijob == 1):
work[slice2] *= sclr2
work[slice2] += sclr1 * matvec(x)
elif (ijob == 2):
work[slice1] = psolve(work[slice2])
if not first_pass and old_ijob == 3:
resid_ready = True
first_pass = False
elif (ijob == 3):
work[slice2] *= sclr2
work[slice2] += sclr1 * matvec(work[slice1])
if resid_ready and callback is not None:
callback(presid / bnrm2)
resid_ready = False
iter_num = iter_num + 1
elif (ijob == 4):
if ftflag:
info = -1
ftflag = False
resid, info = _stoptest(work[slice1], atol)
# Inner loop tolerance control
if info or presid > ptol:
ptol_max_factor = min(1.0, 1.5 * ptol_max_factor)
else:
# Inner loop tolerance OK, but outer loop not.
ptol_max_factor = max(1e-16, 0.25 * ptol_max_factor)
if resid != 0:
ptol = presid * min(ptol_max_factor, atol / resid)
else:
ptol = presid * ptol_max_factor
old_ijob = ijob
ijob = 2
if iter_num > maxiter:
info = maxiter
break
if info >= 0 and not (resid <= atol):
# info isn't set appropriately otherwise
info = maxiter
return postprocess(x), info, mydict['resnorms']
def idrs(A,
b,
x0=None,
tol=1e-5,
norm=np.linalg.norm,
s=4,
maxiter=None,
callback=None,
init_seed=None):
"""
References
----------
.. [1] P. Sonneveld and M. B. van Gijzen
SIAM J. Sci. Comput. Vol. 31, No. 2, pp. 1035--1062, (2008).
.. [2] M. B. van Gijzen and P. Sonneveld
ACM Trans. Math. Software,, Vol. 38, No. 1, pp. 5:1-5:19, (2011).
"""
dtype = A.dtype
A, _, x, b, postprocess = make_system(A, None, x0, b)
n = len(b)
if maxiter is None:
maxiter = n * 10
if callback is not None:
callback(x)
r = b - A.dot(x)
np.random.seed(init_seed)
P = np.random.randn(n, s)
P[:, 0] = r # For comparison with BiCGStab
P = orth(P).T
rnrm = norm(r)
# Relative tolerance
tolr = tol * norm(b)
if rnrm < tolr:
# Initial guess is a good enough solution
return postprocess(x)
# Initialization
dR = np.zeros((n, s), dtype=dtype)
dX = np.zeros_like(dR)
M = np.zeros((s, s), dtype=dtype)
for i in range(s):
v = A.dot(r)
om = np.dot(v, r) / np.dot(v, v)
dX[:, i] = om * r
dR[:, i] = -om * v
x += dX[:, i]
if callback is not None:
callback(x)
r += dR[:, i]
M[:, i] = P.dot(dR[:, i])
iter_ = s
current = 0
m = P.dot(r)
while rnrm > tolr and iter_ < maxiter:
for k in range(s + 1):
c = np.linalg.solve(M, m)
q = -dR.dot(c)
v = r + q
if k == 0:
t = A.dot(v)
om = np.dot(t, v) / np.dot(v, v)
dR[:, current] = q - np.dot(om, t)
dX[:, current] = -dX.dot(c) + om * v
else:
dX[:, current] = -dX.dot(c) + om * v
dR[:, current] = -A.dot(dX[:, current])
x += dX[:, current]
if callback is not None:
callback(x)
r += dR[:, current]
iter_ += 1
dm = P.dot(dR[:, current])
M[:, current] = dm
m += dm
current = (current + 1) % s
rnrm = norm(r)
return postprocess(x)[:, None]
def minres(A,
b,
x0=None,
shift=0.0,
tol=1e-5,
maxiter=None,
M=None,
callback=None,
show=False,
check=False):
A, M, x, b, postprocess = make_system(A, M, x0, b)
matvec = A.matvec
psolve = M.matvec
first = 'Enter minres. '
last = 'Exit minres. '
n = A.shape[0]
if maxiter is None:
maxiter = 5 * n
msg = [
' beta2 = 0. If M = I, b and x are eigenvectors ', # -1
' beta1 = 0. The exact solution is x = 0 ', # 0
' A solution to Ax = b was found, given rtol ', # 1
' A least-squares solution was found, given rtol ', # 2
' Reasonable accuracy achieved, given eps ', # 3
' x has converged to an eigenvector ', # 4
' acond has exceeded 0.1/eps ', # 5
' The iteration limit was reached ', # 6
' A does not define a symmetric matrix ', # 7
' M does not define a symmetric matrix ', # 8
' M does not define a pos-def preconditioner '
] # 9
if show:
print(first + 'Solution of symmetric Ax = b')
print(first + 'n = %3g shift = %23.14e' % (n, shift))
print(first + 'itnlim = %3g rtol = %11.2e' % (maxiter, tol))
print()
istop = 0
itn = 0
Anorm = 0
Acond = 0
rnorm = 0
ynorm = 0
xtype = x.dtype
eps = finfo(xtype).eps
x = zeros(n, dtype=xtype)
# Set up y and v for the first Lanczos vector v1.
# y = beta1 P' v1, where P = C**(-1).
# v is really P' v1.
y = b
r1 = b
y = psolve(b)
beta1 = inner(b, y)
if beta1 < 0:
raise ValueError('indefinite preconditioner')
elif beta1 == 0:
return (postprocess(x), 0)
beta1 = sqrt(beta1)
if check:
# are these too strict?
# see if A is symmetric
w = matvec(y)
r2 = matvec(w)
s = inner(w, w)
t = inner(y, r2)
z = abs(s - t)
epsa = (s + eps) * eps**(1.0 / 3.0)
if z > epsa:
raise ValueError('non-symmetric matrix')
# see if M is symmetric
r2 = psolve(y)
s = inner(y, y)
t = inner(r1, r2)
z = abs(s - t)
epsa = (s + eps) * eps**(1.0 / 3.0)
if z > epsa:
raise ValueError('non-symmetric preconditioner')
# Initialize other quantities
oldb = 0
beta = beta1
dbar = 0
epsln = 0
qrnorm = beta1
phibar = beta1
rhs1 = beta1
rhs2 = 0
tnorm2 = 0
ynorm2 = 0
cs = -1
sn = 0
w = zeros(n, dtype=xtype)
w2 = zeros(n, dtype=xtype)
r2 = r1
if show:
print()
print()
print(
' Itn x(1) Compatible LS norm(A) cond(A) gbar/|A|'
)
while itn < maxiter:
itn += 1
s = 1.0 / beta
v = s * y
y = matvec(v)
y = y - shift * v
if itn >= 2:
y = y - (beta / oldb) * r1
alfa = inner(v, y)
y = y - (alfa / beta) * r2
r1 = r2
r2 = y
y = psolve(r2)
oldb = beta
beta = inner(r2, y)
if beta < 0:
raise ValueError('non-symmetric matrix')
beta = sqrt(beta)
tnorm2 += alfa**2 + oldb**2 + beta**2
if itn == 1:
if beta / beta1 <= 10 * eps:
istop = -1 # Terminate later
# tnorm2 = alfa**2 ??
gmax = abs(alfa)
gmin = gmax
# Apply previous rotation Qk-1 to get
# [deltak epslnk+1] = [cs sn][dbark 0 ]
# [gbar k dbar k+1] [sn -cs][alfak betak+1].
oldeps = epsln
delta = cs * dbar + sn * alfa # delta1 = 0 deltak
gbar = sn * dbar - cs * alfa # gbar 1 = alfa1 gbar k
epsln = sn * beta # epsln2 = 0 epslnk+1
dbar = -cs * beta # dbar 2 = beta2 dbar k+1
root = norm([gbar, dbar])
Arnorm = phibar * root
# Compute the next plane rotation Qk
gamma = norm([gbar, beta]) # gammak
gamma = max(gamma, eps)
cs = gbar / gamma # ck
sn = beta / gamma # sk
phi = cs * phibar # phik
phibar = sn * phibar # phibark+1
# Update x.
denom = 1.0 / gamma
w1 = w2
w2 = w
w = (v - oldeps * w1 - delta * w2) * denom
x = x + phi * w
# Go round again.
gmax = max(gmax, gamma)
gmin = min(gmin, gamma)
z = rhs1 / gamma
ynorm2 = z**2 + ynorm2
rhs1 = rhs2 - delta * z
rhs2 = -epsln * z
# Estimate various norms and test for convergence.
Anorm = sqrt(tnorm2)
ynorm = sqrt(ynorm2)
epsa = Anorm * eps
epsx = Anorm * ynorm * eps
epsr = Anorm * ynorm * tol
diag = gbar
if diag == 0:
diag = epsa
qrnorm = phibar
rnorm = qrnorm
test1 = rnorm / (Anorm * ynorm) # ||r|| / (||A|| ||x||)
test2 = root / Anorm # ||Ar|| / (||A|| ||r||)
# Estimate cond(A).
# In this version we look at the diagonals of R in the
# factorization of the lower Hessenberg matrix, Q * H = R,
# where H is the tridiagonal matrix from Lanczos with one
# extra row, beta(k+1) e_k^T.
Acond = gmax / (gmin + eps)
# See if any of the stopping criteria are satisfied.
# In rare cases, istop is already -1 from above (Abar = const*I).
if istop == 0:
t1 = 1 + test1 # These tests work if tol < eps
t2 = 1 + test2
if t2 <= 1:
istop = 2
if t1 <= 1:
istop = 1
if itn >= maxiter:
istop = 6
if Acond >= 0.1 / eps:
istop = 4
if epsx >= beta:
istop = 3
# if rnorm <= epsx : istop = 2
# if rnorm <= epsr : istop = 1
if test2 <= tol:
istop = 2
if test1 <= tol:
istop = 1
# See if it is time to print something.
prnt = False
if n <= 40:
prnt = True
if itn <= 10:
prnt = True
if itn >= maxiter - 10:
prnt = True
if itn % 10 == 0:
prnt = True
if qrnorm <= 10 * epsx:
prnt = True
if qrnorm <= 10 * epsr:
prnt = True
if Acond <= 1e-2 / eps:
prnt = True
if istop != 0:
prnt = True
if show and prnt:
str1 = '%6g %12.5e %10.3e' % (itn, x[0], test1)
str2 = ' %10.3e' % (test2, )
str3 = ' %8.1e %8.1e %8.1e' % (Anorm, Acond, gbar / Anorm)
print(str1 + str2 + str3)
if itn % 10 == 0:
print()
if callback is not None:
callback(x)
if istop in [1, 2]:
pass
#break # TODO check this
if show:
print()
print(last + ' istop = %3g itn =%5g' % (istop, itn))
print(last + ' Anorm = %12.4e Acond = %12.4e' %
(Anorm, Acond))
print(last + ' rnorm = %12.4e ynorm = %12.4e' %
(rnorm, ynorm))
print(last + ' Arnorm = %12.4e' % (Arnorm, ))
print(last + msg[istop + 1])
if istop == 6:
info = maxiter
else:
info = 0
return (postprocess(x), info)
def minimal_residual(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None,
callback=None, residuals=None):
'''Minimal residual (MR) algorithm
Solves the linear system Ax = b. Left preconditioning is supported.
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by the
preconditioner norm of r_0, or ||r_0||_M.
maxiter : int
maximum number of allowed iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A x = M b.
callback : function
User-supplied function is called after each iteration as
callback(xk), where xk is the current solution vector
residuals : list
residuals contains the residual norm history,
including the initial residual. The preconditioner norm
is used, instead of the Euclidean norm.
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of cg
== =======================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead.
<0 numerical breakdown, or illegal input
== =======================================
Notes
-----
The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
The residual in the preconditioner norm is both used for halting and
returned in the residuals list.
Examples
--------
>>> from pyamg.krylov import minimal_residual
>>> from pyamg.util.linalg import norm
>>> import numpy
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = numpy.ones((A.shape[0],))
>>> (x,flag) = minimal_residual(A,b, maxiter=2, tol=1e-8)
>>> print norm(b - A*x)
7.26369350856
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 137--142, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
A,M,x,b,postprocess = make_system(A,M,x0,b,xtype=None)
n = len(b)
##
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always', module='pyamg\.krylov\._minimal_residual')
# determine maxiter
if maxiter is None:
maxiter = int(len(b))
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
# setup method
r = M*(b - A*x)
normr = norm(r)
# store initial residual
if residuals is not None:
residuals[:] = [normr]
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol*normb:
return (postprocess(x), 0)
# Scale tol by ||r_0||_M
if normr != 0.0:
tol = tol*normr
# How often should r be recomputed
recompute_r = 50
iter = 0
while True:
iter = iter+1
p = M*(A*r)
rMAr = inner(p.conjugate(), r) # check curvature of M^-1 A
if rMAr < 0.0:
warn("\nIndefinite matrix detected in minimal residual, aborting\n")
return (postprocess(x), -1)
alpha = rMAr / inner(p.conjugate(), p)
x = x + alpha*r
if mod(iter, recompute_r) and iter > 0:
r = M*(b - A*x)
else:
r = r - alpha*p
normr = norm(r)
if residuals is not None:
residuals.append(normr)
if callback is not None:
callback(x)
if normr < tol:
return (postprocess(x), 0)
if iter == maxiter:
return (postprocess(x), iter)
def fgmres(A, b, x0=None, tol=1e-5, restrt=None, maxiter=None, xtype=None,
M=None, callback=None, residuals=None):
'''Flexible Generalized Minimum Residual Method (fGMRES)
fGMRES iteratively refines the initial solution guess to the
system Ax = b. fGMRES is flexible in the sense that the right
preconditioner (M) can vary from iteration to iteration.
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by ||r_0||_2
restrt : {None, int}
- if int, restrt is max number of inner iterations
and maxiter is the max number of outer iterations
- if None, do not restart GMRES, and max number of inner iterations is
maxiter
maxiter : {None, int}
- if restrt is None, maxiter is the max number of inner iterations
and GMRES does not restart
- if restrt is int, maxiter is the max number of outer iterations,
and restrt is the max number of inner iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve A M x = M b.
M need not be stationary for fgmres
callback : function
User-supplied function is called after each iteration as
callback( ||rk||_2 ), where rk is the current residual vector
residuals : list
residuals has the residual norm history,
including the initial residual, appended to it
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of gmres
== =============================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead. This value
is precisely the order of the Krylov space.
<0 numerical breakdown, or illegal input
== =============================================
Notes
-----
- The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
- fGMRES allows for non-stationary preconditioners, as opposed to GMRES
- For robustness, Householder reflections are used to orthonormalize
the Krylov Space
Givens Rotations are used to provide the residual norm each iteration
Flexibility implies that the right preconditioner, M or A.psolve, can
vary from iteration to iteration
Examples
--------
>>> from pyamg.krylov.fgmres import fgmres
>>> from pyamg.util.linalg import norm
>>> import numpy as np
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = np.ones((A.shape[0],))
>>> (x,flag) = fgmres(A,b, maxiter=2, tol=1e-8)
>>> print norm(b - A*x)
6.5428213057
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
# Convert inputs to linear system, with error checking
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype)
dimen = A.shape[0]
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always', module='pyamg\.krylov\._fgmres')
# Choose type
if not hasattr(A, 'dtype'):
Atype = upcast(x.dtype, b.dtype)
else:
Atype = A.dtype
if not hasattr(M, 'dtype'):
Mtype = upcast(x.dtype, b.dtype)
else:
Mtype = M.dtype
xtype = upcast(Atype, x.dtype, b.dtype, Mtype)
# Should norm(r) be kept
if residuals == []:
keep_r = True
else:
keep_r = False
# Set number of outer and inner iterations
if restrt:
if maxiter:
max_outer = maxiter
else:
max_outer = 1
if restrt > dimen:
warn('Setting number of inner iterations (restrt) to maximum \
allowed, which is A.shape[0] ')
restrt = dimen
max_inner = restrt
else:
max_outer = 1
if maxiter > dimen:
warn('Setting number of inner iterations (maxiter) to maximum \
allowed, which is A.shape[0] ')
maxiter = dimen
elif maxiter is None:
maxiter = min(dimen, 40)
max_inner = maxiter
# Get fast access to underlying BLAS routines
[rotg] = get_blas_funcs(['rotg'], [x])
# Is this a one dimensional matrix?
if dimen == 1:
entry = ravel(A*array([1.0], dtype=xtype))
return (postprocess(b/entry), 0)
# Prep for method
r = b - ravel(A*x)
normr = norm(r)
if keep_r:
residuals.append(normr)
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol*normb:
if callback is not None:
callback(norm(r))
return (postprocess(x), 0)
# Scale tol by ||r_0||_2, we don't use the preconditioned
# residual because this is right preconditioned GMRES.
if normr != 0.0:
tol = tol*normr
# Use separate variable to track iterations. If convergence fails,
# we cannot simply report niter = (outer-1)*max_outer + inner. Numerical
# error could cause the inner loop to halt while the actual ||r|| > tol.
niter = 0
# Begin fGMRES
for outer in range(max_outer):
# Calculate vector w, which defines the Householder reflector
# Take shortcut in calculating,
# w = r + sign(r[1])*||r||_2*e_1
w = r
beta = mysign(w[0])*normr
w[0] += beta
w /= norm(w)
# Preallocate for Krylov vectors, Householder reflectors and Hessenberg
# matrix
# Space required is O(dimen*max_inner)
# Givens Rotations
Q = zeros((4*max_inner,), dtype=xtype)
# upper Hessenberg matrix (made upper tri with Givens Rotations)
H = zeros((max_inner, max_inner), dtype=xtype)
W = zeros((max_inner, dimen), dtype=xtype) # Householder reflectors
# For fGMRES, preconditioned vectors must be stored
# No Horner-like scheme exists that allow us to avoid this
Z = zeros((dimen, max_inner), dtype=xtype)
W[0, :] = w
# Multiply r with (I - 2*w*w.T), i.e. apply the Householder reflector
# This is the RHS vector for the problem in the Krylov Space
g = zeros((dimen,), dtype=xtype)
g[0] = -beta
for inner in range(max_inner):
# Calculate Krylov vector in two steps
# (1) Calculate v = P_j = (I - 2*w*w.T)v, where k = inner
v = -2.0*conjugate(w[inner])*w
v[inner] += 1.0
# (2) Calculate the rest, v = P_1*P_2*P_3...P_{j-1}*ej.
# for j in range(inner-1,-1,-1):
# v = v - 2.0*dot(conjugate(W[j,:]), v)*W[j,:]
amg_core.apply_householders(v, ravel(W), dimen, inner-1, -1, -1)
# Apply preconditioner
v = ravel(M*v)
# Check for nan, inf
# if isnan(v).any() or isinf(v).any():
# warn('inf or nan after application of preconditioner')
# return(postprocess(x), -1)
Z[:, inner] = v
# Calculate new search direction
v = ravel(A*v)
# Factor in all Householder orthogonal reflections on new search
# direction
# for j in range(inner+1):
# v = v - 2.0*dot(conjugate(W[j,:]), v)*W[j,:]
amg_core.apply_householders(v, ravel(W), dimen, 0, inner+1, 1)
# Calculate next Householder reflector, w
# w = v[inner+1:] + sign(v[inner+1])*||v[inner+1:]||_2*e_{inner+1)
# Note that if max_inner = dimen, then this is unnecessary for
# the last inner iteration, when inner = dimen-1. Here we do
# not need to calculate a Householder reflector or Givens
# rotation because nnz(v) is already the desired length,
# i.e. we do not need to zero anything out.
if inner != dimen-1:
if inner < (max_inner-1):
w = W[inner+1, :]
vslice = v[inner+1:]
alpha = norm(vslice)
if alpha != 0:
alpha = mysign(vslice[0])*alpha
# do not need the final reflector for future calculations
if inner < (max_inner-1):
w[inner+1:] = vslice
w[inner+1] += alpha
w /= norm(w)
# Apply new reflector to v
# v = v - 2.0*w*(w.T*v)
v[inner+1] = -alpha
v[inner+2:] = 0.0
if inner > 0:
# Apply all previous Givens Rotations to v
amg_core.apply_givens(Q, v, dimen, inner)
# Calculate the next Givens rotation, where j = inner Note that if
# max_inner = dimen, then this is unnecessary for the last inner
# iteration, when inner = dimen-1. Here we do not need to
# calculate a Householder reflector or Givens rotation because
# nnz(v) is already the desired length, i.e. we do not need to zero
# anything out.
if inner != dimen-1:
if v[inner+1] != 0:
[c, s] = rotg(v[inner], v[inner+1])
Qblock = array([[c, s], [-conjugate(s), c]], dtype=xtype)
Q[(inner*4): ((inner+1)*4)] = ravel(Qblock).copy()
# Apply Givens Rotation to g, the RHS for the linear system
# in the Krylov Subspace. Note that this dot does a matrix
# multiply, not an actual dot product where a conjugate
# transpose is taken
g[inner:inner+2] = dot(Qblock, g[inner:inner+2])
# Apply effect of Givens Rotation to v
v[inner] = dot(Qblock[0, :], v[inner:inner+2])
v[inner+1] = 0.0
# Write to upper Hessenberg Matrix,
# the LHS for the linear system in the Krylov Subspace
H[:, inner] = v[0:max_inner]
# Don't update normr if last inner iteration, because
# normr is calculated directly after this loop ends.
if inner < max_inner-1:
normr = abs(g[inner+1])
if normr < tol:
break
# Allow user access to residual
if callback is not None:
callback(normr)
if keep_r:
residuals.append(normr)
niter += 1
# end inner loop, back to outer loop
# Find best update to x in Krylov Space, V. Solve inner+1 x inner+1
# system. Apparently this is the best way to solve a triangular system
# in the magical world of scipy
# piv = arange(inner+1)
# y = lu_solve((H[0:(inner+1),0:(inner+1)], piv),
# g[0:(inner+1)], trans=0)
y = sp.linalg.solve(H[0:(inner+1), 0:(inner+1)], g[0:(inner+1)])
# No Horner like scheme exists because the preconditioner can change
# each iteration # Hence, we must store each preconditioned vector
update = dot(Z[:, 0:inner+1], y)
x = x + update
r = b - ravel(A*x)
normr = norm(r)
# Allow user access to residual
if callback is not None:
callback(normr)
if keep_r:
residuals.append(normr)
# Has fGMRES stagnated?
indices = (x != 0)
if indices.any():
change = max(abs(update[indices] / x[indices]))
if change < 1e-12:
# No change, halt
return (postprocess(x), -1)
# test for convergence
if normr < tol:
return (postprocess(x), 0)
# end outer loop
return (postprocess(x), niter)
def cg(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None, callback=None, residuals=None):
'''Conjugate Gradient algorithm
Solves the linear system Ax = b. Left preconditioning is supported.
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by ||b||
maxiter : int
maximum number of allowed iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A A.H x = b.
callback : function
User-supplied funtion is called after each iteration as
callback(xk), where xk is the current solution vector
residuals : list
residuals has the residual norm history,
including the initial residual, appended to it
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of cg
== =======================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead.
<0 numerical breakdown, or illegal input
== =======================================
Notes
-----
The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
Examples
--------
>>> from pyamg.krylov.cg import cg
>>> from pyamg.util.linalg import norm
>>> import numpy
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = numpy.ones((A.shape[0],))
>>> (x,flag) = cg(A,b, maxiter=2, tol=1e-8)
>>> print norm(b - A*x)
10.9370700187
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 262-67, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
A,M,x,b,postprocess = make_system(A,M,x0,b,xtype=None)
n = len(b)
# Determine maxiter
if maxiter is None:
maxiter = int(1.3*len(b)) + 2
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
# Scale tol by normb
normb = norm(b)
if normb != 0:
tol = tol*normb
# setup method
r = b - A*x
z = M*r
p = z.copy()
rz = inner(conjugate(r), z)
normr = norm(r)
if residuals is not None:
residuals[:] = [normr] #initial residual
if normr < tol:
return (postprocess(x), 0)
iter = 0
while True:
Ap = A*p
rz_old = rz
alpha = rz/inner(conjugate(Ap), p) # 3 (step # in Saad's pseudocode)
x += alpha * p # 4
r -= alpha * Ap # 5
z = M*r # 6
rz = inner(conjugate(r), z)
beta = rz/rz_old # 7
p *= beta # 8
p += z
iter += 1
normr = norm(r)
if residuals is not None:
residuals.append(normr)
if callback is not None:
callback(x)
if normr < tol:
return (postprocess(x), 0)
if iter == maxiter:
return (postprocess(x), iter)
def cgne(A,
b,
x0=None,
tol=1e-5,
maxiter=None,
xtype=None,
M=None,
callback=None,
residuals=None):
'''Conjugate Gradient, Normal Error algorithm
Applies CG to the normal equations, A.H A x = b. Left preconditioning
is supported. Note that unless A is well-conditioned, the use of
CGNE is inadvisable
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by ||r_0||_2
maxiter : int
maximum number of allowed iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A A.H x = M b.
callback : function
User-supplied function is called after each iteration as
callback(xk), where xk is the current solution vector
residuals : list
residuals has the residual norm history,
including the initial residual, appended to it
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of cgne
== =======================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead.
<0 numerical breakdown, or illegal input
== =======================================
Notes
-----
- The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
Examples
--------
>>> from pyamg.krylov.cgne import cgne
>>> from pyamg.util.linalg import norm
>>> import numpy as np
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = np.ones((A.shape[0],))
>>> (x,flag) = cgne(A,b, maxiter=2, tol=1e-8)
>>> print norm(b - A*x)
46.1547104367
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 276-7, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
# Store the conjugate transpose explicitly as it will be used much later on
if isspmatrix(A):
AH = A.H
else:
# TODO avoid doing this since A may be a different sparse type
AH = aslinearoperator(np.asmatrix(A).H)
# Convert inputs to linear system, with error checking
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype)
dimen = A.shape[0]
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always', module='pyamg\.krylov\._cgne')
# Choose type
if not hasattr(A, 'dtype'):
Atype = upcast(x.dtype, b.dtype)
else:
Atype = A.dtype
if not hasattr(M, 'dtype'):
Mtype = upcast(x.dtype, b.dtype)
else:
Mtype = M.dtype
xtype = upcast(Atype, x.dtype, b.dtype, Mtype)
# Should norm(r) be kept
if residuals == []:
keep_r = True
else:
keep_r = False
# How often should r be recomputed
recompute_r = 8
# Check iteration numbers. CGNE suffers from loss of orthogonality quite
# easily, so we arbitrarily let the method go up to 130% over the
# theoretically necessary limit of maxiter=dimen
if maxiter is None:
maxiter = int(np.ceil(1.3 * dimen)) + 2
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
elif maxiter > (1.3 * dimen):
warn('maximum allowed inner iterations (maxiter) are the 130% times \
the number of dofs')
maxiter = int(np.ceil(1.3 * dimen)) + 2
# Prep for method
r = b - A * x
normr = norm(r)
if keep_r:
residuals.append(normr)
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol * normb:
if callback is not None:
callback(x)
return (postprocess(x), 0)
# Scale tol by ||r_0||_2
if normr != 0.0:
tol = tol * normr
# Begin CGNE
# Apply preconditioner and calculate initial search direction
z = M * r
p = AH * z
old_zr = np.inner(z.conjugate(), r)
for iter in range(maxiter):
# alpha = (z_j, r_j) / (p_j, p_j)
alpha = old_zr / np.inner(p.conjugate(), p)
# x_{j+1} = x_j + alpha*p_j
x += alpha * p
# r_{j+1} = r_j - alpha*w_j, where w_j = A*p_j
if np.mod(iter, recompute_r) and iter > 0:
r -= alpha * (A * p)
else:
r = b - A * x
# z_{j+1} = M*r_{j+1}
z = M * r
# beta = (z_{j+1}, r_{j+1}) / (z_j, r_j)
new_zr = np.inner(z.conjugate(), r)
beta = new_zr / old_zr
old_zr = new_zr
# p_{j+1} = A.H*z_{j+1} + beta*p_j
p *= beta
p += AH * z
# Allow user access to residual
if callback is not None:
callback(x)
# test for convergence
normr = norm(r)
if keep_r:
residuals.append(normr)
if normr < tol:
return (postprocess(x), 0)
# end loop
return (postprocess(x), iter + 1)
def idrs_biortho(A,
b,
x0=None,
tol=1e-5,
norm=np.linalg.norm,
s=4,
maxiter=None,
callback=None,
init_seed=None):
"""
References
----------
.. [1] P. Sonneveld and M. B. van Gijzen
SIAM J. Sci. Comput. Vol. 31, No. 2, pp. 1035--1062, (2008).
.. [2] M. B. van Gijzen and P. Sonneveld
ACM Trans. Math. Software,, Vol. 38, No. 1, pp. 5:1-5:19, (2011).
"""
dtype = A.dtype
A, B, x, b, postprocess = make_system(A, None, x0, b)
n = len(b)
if maxiter is None:
maxiter = n * 10
if callback is not None:
callback(x)
r = b - A.dot(x)
rnrm = norm(r)
np.random.seed(init_seed)
P = np.random.randn(n, s)
P[:, 0] = r # For comparison with BiCdRStab
P = orth(P).T
if callback is not None:
callback(x)
tolr = tol * norm(b)
if rnrm < tolr:
return postprocess(x)
dR = np.zeros((n, s), dtype=dtype)
dX = np.zeros((n, s), dtype=dtype)
M = np.eye(s, dtype=dtype)
om = 1.0
iter_ = 0
while rnrm >= tolr and iter_ < maxiter:
m = P.dot(r)
for k in range(0, s):
c = np.linalg.solve(M[k:s, k:s], m[k:s])
v = r - dR[:, k:s].dot(c)
v = B.dot(v)
dX[:, k] = dX[:, k:s].dot(c) + om * v
dR[:, k] = A.dot(dX[:, k])
for i in range(0, k):
alpha = np.dot(P[i, :], dR[:, k]) / M[i, i]
dR[:, k] = dR[:, k] - alpha * dR[:, i]
dX[:, k] = dX[:, k] - alpha * dX[:, i]
M[k:s, k] = np.dot(P[k:s, :], dR[:, k])
beta = m[k] / M[k, k]
r -= beta * dR[:, k]
rnrm = np.linalg.norm(r)
x += beta * dX[:, k]
if callback is not None:
callback(x)
iter_ += 1
if k + 1 < s:
m[k + 1:s] -= beta * M[k + 1:s, k]
if rnrm < tolr or iter_ >= maxiter:
break
v = B.dot(r)
t = A.dot(v)
om = np.dot(t, r) / np.dot(t, t)
x += om * v
if callback is not None:
callback(x)
r -= om * t
rnrm = norm(r)
iter_ += 1
return postprocess(x)[:, None]
def cr(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None,
callback=None, residuals=None):
'''Conjugate Residual algorithm
Solves the linear system Ax = b. Left preconditioning is supported.
The matrix A must be Hermitian symmetric (but not necessarily definite).
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by the
preconditioner norm of r_0, or ||r_0||_M.
maxiter : int
maximum number of allowed iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A x = M b.
callback : function
User-supplied function is called after each iteration as
callback(xk), where xk is the current solution vector
residuals : list
residuals contains the residual norm history,
including the initial residual. The preconditioner norm
is used, instead of the Euclidean norm.
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of cr
== =======================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead.
<0 numerical breakdown, or illegal input
== =======================================
Notes
-----
The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
The 2-norm of the preconditioned residual is used both for halting and
returned in the residuals list.
Examples
--------
>>> from pyamg.krylov.cr import cr
>>> from pyamg.util.linalg import norm
>>> import numpy as np
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = np.ones((A.shape[0],))
>>> (x,flag) = cr(A,b, maxiter=2, tol=1e-8)
>>> print norm(b - A*x)
10.9370700187
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 262-67, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype=None)
# n = len(b)
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always', module='pyamg\.krylov\._cr')
# determine maxiter
if maxiter is None:
maxiter = int(1.3*len(b)) + 2
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
# choose tolerance for numerically zero values
# t = A.dtype.char
# eps = np.finfo(np.float).eps
# feps = np.finfo(np.single).eps
# geps = np.finfo(np.longfloat).eps
# _array_precision = {'f': 0, 'd': 1, 'g': 2, 'F': 0, 'D': 1, 'G': 2}
# numerically_zero = {0: feps*1e3, 1: eps*1e6,
# 2: geps*1e6}[_array_precision[t]]
# setup method
r = b - A*x
z = M*r
p = z.copy()
zz = inner(z.conjugate(), z)
# use preconditioner norm
normr = sqrt(zz)
if residuals is not None:
residuals[:] = [normr] # initial residual
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol*normb:
return (postprocess(x), 0)
# Scale tol by ||r_0||_M
if normr != 0.0:
tol = tol*normr
# How often should r be recomputed
recompute_r = 8
iter = 0
Az = A*z
rAz = inner(r.conjugate(), Az)
Ap = A*p
while True:
rAz_old = rAz
alpha = rAz / inner(Ap.conjugate(), Ap) # 3
x += alpha * p # 4
if mod(iter, recompute_r) and iter > 0: # 5
r -= alpha * Ap
else:
r = b - A*x
z = M*r
Az = A*z
rAz = inner(r.conjugate(), Az)
beta = rAz/rAz_old # 6
p *= beta # 7
p += z
Ap *= beta # 8
Ap += Az
iter += 1
zz = inner(z.conjugate(), z)
normr = sqrt(zz) # use preconditioner norm
if residuals is not None:
residuals.append(normr)
if callback is not None:
callback(x)
if normr < tol:
return (postprocess(x), 0)
elif zz == 0.0:
# important to test after testing normr < tol. rz == 0.0 is an
# indicator of convergence when r = 0.0
warn("\nSingular preconditioner detected in CR, ceasing \
iterations\n")
return (postprocess(x), -1)
if iter == maxiter:
return (postprocess(x), iter)
def msGmres(A,
b,
shifts,
m=200,
x0=None,
tol=1e-5,
maxiter=None,
M=None,
callback=None):
"""
Multi-Shift Gmres(m)
"""
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype=None)
n = len(b)
if maxiter is None:
maxiter = n * 10
matvec = A.matvec
psolve = M.matvec
xtype = x.dtype
# Initialization
# BLAS functions
axpy, dot, norm, rotmat = get_blas_funcs(['axpy', 'dot', 'nrm2', 'rotg'],
dtype=xtype)
# Relative tolerance
tolb = tol * norm(b)
# Compute initial residual:
r = b - matvec(x)
rnrm2 = norm(r)
nshifts = len(shifts)
X = np.zeros((n, nshifts), dtype=xtype)
for j in xrange(0, nshifts):
X[:, j] = np.array(x)
errors = np.zeros(nshifts, dtype=xtype)
V = np.zeros((n, m + 1), dtype=xtype)
H = np.zeros((m + 1, m), dtype=xtype)
# Compute residuals
converged = True
for j in xrange(0, nshifts):
rj = r + shifts[j] * X[:, j]
errors[j] = norm(rj)
converged = converged and (errors[j] < tolb)
if callback is not None:
callback(errors)
if converged:
return V, H
# Begin iteration
# for _iter in range(0, maxit):
V[:, 0] = r / rnrm2
for i in xrange(0, m):
# Construct orthonormal basis using Gram-Schmidt
w = psolve(matvec(V[:, i]))
for k in range(0, i + 1):
H[k, i] = dot(V[:, k], w)
axpy(V[:, k], w, None, -H[k, i])
H[i + 1, i] = norm(w)
V[:, i + 1] = w / H[i + 1, i]
# Multi-Shift part
for j in xrange(0, nshifts):
# Hj = H - w_j I
Hj = np.array(H)
for k in xrange(0, m):
Hj[k, k] -= shifts[j]
# s = |r_j| e_1
s = np.zeros(m + 1, dtype=xtype)
s[0] = errors[j]
# Apply Givens rotation to H_j[:m+1, :m] and s
# Solve Triangular system with scipy.linalg.solve_triangular
# Update X
# Residuals? ¯\_(ツ)_/¯
return V, H
def cr(A, b, x0=None, tol=1e-5, maxiter=None, xtype=None, M=None,
callback=None, residuals=None):
'''Conjugate Residual algorithm
Solves the linear system Ax = b. Left preconditioning is supported.
The matrix A must be Hermitian symmetric (but not necessarily definite).
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by the
preconditioner norm of r_0, or ||r_0||_M.
maxiter : int
maximum number of allowed iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A x = M b.
callback : function
User-supplied function is called after each iteration as
callback(xk), where xk is the current solution vector
residuals : list
residuals contains the residual norm history,
including the initial residual. The preconditioner norm
is used, instead of the Euclidean norm.
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of cr
== =======================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead.
<0 numerical breakdown, or illegal input
== =======================================
Notes
-----
The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
The 2-norm of the preconditioned residual is used both for halting and
returned in the residuals list.
Examples
--------
>>> from pyamg.krylov.cr import cr
>>> from pyamg.util.linalg import norm
>>> import numpy
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = numpy.ones((A.shape[0],))
>>> (x,flag) = cr(A,b, maxiter=2, tol=1e-8)
>>> print norm(b - A*x)
10.9370700187
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 262-67, 2003
http://www-users.cs.umn.edu/~saad/books.html
'''
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype=None)
n = len(b)
##
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always', module='pyamg\.krylov\._cr')
# determine maxiter
if maxiter is None:
maxiter = int(1.3*len(b)) + 2
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
# choose tolerance for numerically zero values
t = A.dtype.char
eps = numpy.finfo(numpy.float).eps
feps = numpy.finfo(numpy.single).eps
geps = numpy.finfo(numpy.longfloat).eps
_array_precision = {'f': 0, 'd': 1, 'g': 2, 'F': 0, 'D': 1, 'G': 2}
numerically_zero = {0: feps*1e3, 1: eps*1e6,
2: geps*1e6}[_array_precision[t]]
# setup method
r = b - A*x
z = M*r
p = z.copy()
zz = inner(z.conjugate(), z)
# use preconditioner norm
normr = sqrt(zz)
if residuals is not None:
residuals[:] = [normr] # initial residual
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol*normb:
return (postprocess(x), 0)
# Scale tol by ||r_0||_M
if normr != 0.0:
tol = tol*normr
# How often should r be recomputed
recompute_r = 8
iter = 0
Az = A*z
rAz = inner(r.conjugate(), Az)
Ap = A*p
while True:
rAz_old = rAz
alpha = rAz / inner(Ap.conjugate(), Ap) # 3
x += alpha * p # 4
if mod(iter, recompute_r) and iter > 0: # 5
r -= alpha * Ap
else:
r = b - A*x
z = M*r
Az = A*z
rAz = inner(r.conjugate(), Az)
beta = rAz/rAz_old # 6
p *= beta # 7
p += z
Ap *= beta # 8
Ap += Az
iter += 1
zz = inner(z.conjugate(), z)
normr = sqrt(zz) # use preconditioner norm
if residuals is not None:
residuals.append(normr)
if callback is not None:
callback(x)
if normr < tol:
return (postprocess(x), 0)
elif zz == 0.0:
# important to test after testing normr < tol. rz == 0.0 is an
# indicator of convergence when r = 0.0
warn("\nSingular preconditioner detected in CR, ceasing \
iterations\n")
return (postprocess(x), -1)
if iter == maxiter:
return (postprocess(x), iter)
def idrs(A, b, x0=None, tol=1e-5, s=4, maxiter=None, xtype=None,
M=None, callback=None):
"""
Use Induced Dimension Reduction method [IDR(s)] to solve A x = b
Parameters
----------
A : {sparse matrix, dense matrix, LinearOperator}
The real or complex N-by-N matrix of the linear system.
b : {array, matrix}
Right hand side of the linear system. Has shape (N,) or (N,1).
x0 : {array, matrix}
Starting guess for the solution.
tol : float, optional
Tolerance to achieve. The algorithm terminates when either the relative
or the absolute residual is below `tol`.
s : integer, optional
specifies the dimension of the shadow space. Normally, a higher
s gives faster convergence, but also makes the method more expensive.
Default is 4.
maxiter : integer, optional
Maximum number of iterations. Iteration will stop after maxiter
steps even if the specified tolerance has not been achieved.
xtype : {'f','d','F','D'}
This parameter is deprecated -- avoid using it.
M : {sparse matrix, dense matrix, LinearOperator}, optional
Preconditioner for A. The preconditioner should approximate the
inverse of A. Effective preconditioning dramatically improves the
rate of convergence, which implies that fewer iterations are needed
to reach a given error tolerance.
callback : function, optional
User-supplied function to call after each iteration. It is called
as callback(xk), where xk is the current solution vector.
Returns
-------
x : array or matrix
The converged solution.
info : integer
Provides convergence information:
- 0 : successful exit
- >0 : convergence to tolerance not achieved, number of iterations
- <0 : illegal input or breakdown
References
----------
.. [1] P. Sonneveld and M. B. van Gijzen
SIAM J. Sci. Comput. Vol. 31, No. 2, pp. 1035--1062, (2008).
.. [2] M. B. van Gijzen and P. Sonneveld
ACM Trans. Math. Software,, Vol. 38, No. 1, pp. 5:1-5:19, (2011).
.. [3] This file is a translation of the following MATLAB implementation:
http://ta.twi.tudelft.nl/nw/users/gijzen/idrs.m
"""
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype)
n = len(b)
if maxiter is None:
maxiter = n*10
matvec = A.matvec
psolve = M.matvec
xtype = x.dtype
axpy, dot, scal = get_blas_funcs(['axpy', 'dot', 'scal'], dtype=xtype)
np.random.seed(0)
P = np.random.rand(s, n)
bnrm = np.linalg.norm(b)
info = 0
# Check for zero rhs:
if bnrm == 0.0:
# Solution is null-vector
return postprocess(np.zeros(n, dtype=xtype)), 0
# Compute initial residual:
r = b - matvec(x)
rnrm = np.linalg.norm(r)
# Relative tolerance
tolb = tol*np.linalg.norm(b)
if rnrm < tolb:
# Initial guess is a good enough solution
return postprocess(x), 0
# Initialization
angle = 0.7
G = np.zeros((n, s), dtype=xtype)
U = np.zeros((n, s), dtype=xtype)
Ms = np.eye(s, dtype=xtype)
om = 1.0
iter_ = 0
# Main iteration loop, build G-spaces:
while rnrm >= tolb and iter_ < maxiter:
# New right-hand size for small system:
f = P.dot(r)
for k in xrange(s):
# Solve small system and make v orthogonal to P:
c = np.linalg.solve(Ms[k:s, k:s], f[k:s])
v = r - G[:, k:s].dot(c)
# Preconditioning:
v = psolve(v)
# Compute new U(:,k) and G(:,k), G(:,k) is in space G_j
U[:, k] = axpy(v, U[:, k:s].dot(c), None, om)
# Matrix-vector product
G[:, k] = matvec(U[:, k])
# Bi-Orthogonalize the new basis vectors:
for i in xrange(0, k):
alpha = dot(P[i, :], G[:, k]) / Ms[i, i]
G[:, k] = axpy(G[:, i], G[:, k], None, -alpha)
U[:, k] = axpy(U[:, i], U[:, k], None, -alpha)
# New column of M = P'*G (first k-1 entries are zero)
for i in xrange(k, s):
Ms[i, k] = dot(P[i, :], G[:, k])
if Ms[k, k] == 0.0:
# Breakdown
return postprocess(x), -1
# Make r orthogonal to q_i, i = 1..k
beta = f[k] / Ms[k, k]
x = axpy(U[:, k], x, None, beta)
r = axpy(G[:, k], r, None, -beta)
rnrm = np.linalg.norm(r)
if callback is not None:
callback(x)
iter_ += 1
if rnrm < tolb or iter_ >= maxiter:
break
# New f = P'*r (first k components are zero)
if k < s - 1:
f[k + 1:s] = axpy(Ms[k + 1:s, k], f[k + 1:s], None, -beta)
# Now we have sufficient vectors in G_j to compute residual in G_j+1
# Note: r is already perpendicular to P so v = r
if rnrm < tolb or iter_ >= maxiter:
break
# Preconditioning:
v = psolve(r)
# Matrix-vector product
t = matvec(v)
# Computation of a new omega
ns = np.linalg.norm(r)
nt = np.linalg.norm(t)
ts = dot(t, r)
rho = abs(ts / (nt * ns))
om = ts / (nt * nt)
if rho < angle:
om = om * angle / rho
# New vector in G_j+1
x = axpy(v, x, None, om)
r = axpy(t, r, None, -om)
rnrm = np.linalg.norm(r)
if callback is not None:
callback(x)
iter_ += 1
if rnrm >= tolb:
info = iter_
return postprocess(x), info
def gmres_mgs(A,
b,
x0=None,
tol=1e-5,
restrt=None,
maxiter=None,
xtype=None,
M=None,
callback=None,
residuals=None,
reorth=False):
'''
Generalized Minimum Residual Method (GMRES)
GMRES iteratively refines the initial solution guess to the system
Ax = b
Modified Gram-Schmidt version
Parameters
----------
A : {array, matrix, sparse matrix, LinearOperator}
n x n, linear system to solve
b : {array, matrix}
right hand side, shape is (n,) or (n,1)
x0 : {array, matrix}
initial guess, default is a vector of zeros
tol : float
relative convergence tolerance, i.e. tol is scaled by the norm
of the initial preconditioned residual
restrt : {None, int}
- if int, restrt is max number of inner iterations
and maxiter is the max number of outer iterations
- if None, do not restart GMRES, and max number of inner iterations
is maxiter
maxiter : {None, int}
- if restrt is None, maxiter is the max number of inner iterations
and GMRES does not restart
- if restrt is int, maxiter is the max number of outer iterations,
and restrt is the max number of inner iterations
xtype : type
dtype for the solution, default is automatic type detection
M : {array, matrix, sparse matrix, LinearOperator}
n x n, inverted preconditioner, i.e. solve M A x = M b.
callback : function
User-supplied function is called after each iteration as
callback(xk), where xk is the current solution vector
residuals : list
residuals contains the preconditioned residual norm history,
including the initial residual.
reorth : boolean
If True, then a check is made whether to re-orthogonalize the Krylov
space each GMRES iteration
Returns
-------
(xNew, info)
xNew : an updated guess to the solution of Ax = b
info : halting status of gmres
== =============================================
0 successful exit
>0 convergence to tolerance not achieved,
return iteration count instead. This value
is precisely the order of the Krylov space.
<0 numerical breakdown, or illegal input
== =============================================
Notes
-----
- The LinearOperator class is in scipy.sparse.linalg.interface.
Use this class if you prefer to define A or M as a mat-vec routine
as opposed to explicitly constructing the matrix. A.psolve(..) is
still supported as a legacy.
- For robustness, modified Gram-Schmidt is used to orthogonalize the
Krylov Space Givens Rotations are used to provide the residual norm
each iteration
Examples
--------
>>> from pyamg.krylov import gmres
>>> from pyamg.util.linalg import norm
>>> import numpy as np
>>> from pyamg.gallery import poisson
>>> A = poisson((10,10))
>>> b = np.ones((A.shape[0],))
>>> (x,flag) = gmres(A,b, maxiter=2, tol=1e-8, orthog='mgs')
>>> print norm(b - A*x)
>>> 6.5428213057
References
----------
.. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003
http://www-users.cs.umn.edu/~saad/books.html
.. [2] C. T. Kelley, http://www4.ncsu.edu/~ctk/matlab_roots.html
'''
# Convert inputs to linear system, with error checking
A, M, x, b, postprocess = make_system(A, M, x0, b, xtype)
dimen = A.shape[0]
# Ensure that warnings are always reissued from this function
import warnings
warnings.filterwarnings('always', module='pyamg\.krylov\._gmres_mgs')
# Choose type
if not hasattr(A, 'dtype'):
Atype = upcast(x.dtype, b.dtype)
else:
Atype = A.dtype
if not hasattr(M, 'dtype'):
Mtype = upcast(x.dtype, b.dtype)
else:
Mtype = M.dtype
xtype = upcast(Atype, x.dtype, b.dtype, Mtype)
if restrt is not None:
restrt = int(restrt)
if maxiter is not None:
maxiter = int(maxiter)
# Get fast access to underlying BLAS routines
# dotc is the conjugate dot, dotu does no conjugation
[lartg] = get_lapack_funcs(['lartg'], [x])
if np.iscomplexobj(np.zeros((1, ), dtype=xtype)):
[axpy, dotu, dotc, scal] =\
get_blas_funcs(['axpy', 'dotu', 'dotc', 'scal'], [x])
else:
# real type
[axpy, dotu, dotc, scal] =\
get_blas_funcs(['axpy', 'dot', 'dot', 'scal'], [x])
# Make full use of direct access to BLAS by defining own norm
def norm(z):
return np.sqrt(np.real(dotc(z, z)))
# Should norm(r) be kept
if residuals == []:
keep_r = True
else:
keep_r = False
# Set number of outer and inner iterations
if restrt:
if maxiter:
max_outer = maxiter
else:
max_outer = 1
if restrt > dimen:
warn('Setting number of inner iterations (restrt) to maximum\
allowed, which is A.shape[0] ')
restrt = dimen
max_inner = restrt
else:
max_outer = 1
if maxiter > dimen:
warn('Setting number of inner iterations (maxiter) to maximum\
allowed, which is A.shape[0] ')
maxiter = dimen
elif maxiter is None:
maxiter = min(dimen, 40)
max_inner = maxiter
# Is this a one dimensional matrix?
if dimen == 1:
entry = np.ravel(A * np.array([1.0], dtype=xtype))
return (postprocess(b / entry), 0)
# Prep for method
r = b - np.ravel(A * x)
# Apply preconditioner
r = np.ravel(M * r)
normr = norm(r)
if keep_r:
residuals.append(normr)
# Check for nan, inf
# if isnan(r).any() or isinf(r).any():
# warn('inf or nan after application of preconditioner')
# return(postprocess(x), -1)
# Check initial guess ( scaling by b, if b != 0,
# must account for case when norm(b) is very small)
normb = norm(b)
if normb == 0.0:
normb = 1.0
if normr < tol * normb:
return (postprocess(x), 0)
# Scale tol by ||r_0||_2, we use the preconditioned residual
# because this is left preconditioned GMRES.
if normr != 0.0:
tol = tol * normr
# Use separate variable to track iterations. If convergence fails, we
# cannot simply report niter = (outer-1)*max_outer + inner. Numerical
# error could cause the inner loop to halt while the actual ||r|| > tol.
niter = 0
# Begin GMRES
for outer in range(max_outer):
# Preallocate for Givens Rotations, Hessenberg matrix and Krylov Space
# Space required is O(dimen*max_inner).
# NOTE: We are dealing with row-major matrices, so we traverse in a
# row-major fashion,
# i.e., H and V's transpose is what we store.
Q = [] # Givens Rotations
# Upper Hessenberg matrix, which is then
# converted to upper tri with Givens Rots
H = np.zeros((max_inner + 1, max_inner + 1), dtype=xtype)
V = np.zeros((max_inner + 1, dimen), dtype=xtype) # Krylov Space
# vs store the pointers to each column of V.
# This saves a considerable amount of time.
vs = []
# v = r/normr
V[0, :] = scal(1.0 / normr, r)
vs.append(V[0, :])
# This is the RHS vector for the problem in the Krylov Space
g = np.zeros((dimen, ), dtype=xtype)
g[0] = normr
for inner in range(max_inner):
# New Search Direction
v = V[inner + 1, :]
v[:] = np.ravel(M * (A * vs[-1]))
vs.append(v)
normv_old = norm(v)
# Check for nan, inf
# if isnan(V[inner+1, :]).any() or isinf(V[inner+1, :]).any():
# warn('inf or nan after application of preconditioner')
# return(postprocess(x), -1)
# Modified Gram Schmidt
for k in range(inner + 1):
vk = vs[k]
alpha = dotc(vk, v)
H[inner, k] = alpha
v[:] = axpy(vk, v, dimen, -alpha)
normv = norm(v)
H[inner, inner + 1] = normv
# Re-orthogonalize
if (reorth is True) and (normv_old == normv_old + 0.001 * normv):
for k in range(inner + 1):
vk = vs[k]
alpha = dotc(vk, v)
H[inner, k] = H[inner, k] + alpha
v[:] = axpy(vk, v, dimen, -alpha)
# Check for breakdown
if H[inner, inner + 1] != 0.0:
v[:] = scal(1.0 / H[inner, inner + 1], v)
# Apply previous Givens rotations to H
if inner > 0:
apply_givens(Q, H[inner, :], inner)
# Calculate and apply next complex-valued Givens Rotation
# ==> Note that if max_inner = dimen, then this is unnecessary
# for the last inner
# iteration, when inner = dimen-1.
if inner != dimen - 1:
if H[inner, inner + 1] != 0:
[c, s, r] = lartg(H[inner, inner], H[inner, inner + 1])
Qblock = np.array([[c, s], [-np.conjugate(s), c]],
dtype=xtype)
Q.append(Qblock)
# Apply Givens Rotation to g,
# the RHS for the linear system in the Krylov Subspace.
g[inner:inner + 2] = np.dot(Qblock, g[inner:inner + 2])
# Apply effect of Givens Rotation to H
H[inner, inner] = dotu(Qblock[0, :], H[inner,
inner:inner + 2])
H[inner, inner + 1] = 0.0
niter += 1
# Don't update normr if last inner iteration, because
# normr is calculated directly after this loop ends.
if inner < max_inner - 1:
normr = np.abs(g[inner + 1])
if normr < tol:
break
# Allow user access to the iterates
if callback is not None:
callback(x)
if keep_r:
residuals.append(normr)
# end inner loop, back to outer loop
# Find best update to x in Krylov Space V. Solve inner x inner system.
y = sp.linalg.solve(H[0:inner + 1, 0:inner + 1].T, g[0:inner + 1])
update = np.ravel(np.mat(V[:inner + 1, :]).T * y.reshape(-1, 1))
x = x + update
r = b - np.ravel(A * x)
# Apply preconditioner
r = np.ravel(M * r)
normr = norm(r)
# Check for nan, inf
# if isnan(r).any() or isinf(r).any():
# warn('inf or nan after application of preconditioner')
# return(postprocess(x), -1)
# Allow user access to the iterates
if callback is not None:
callback(x)
if keep_r:
residuals.append(normr)
# Has GMRES stagnated?
indices = (x != 0)
if indices.any():
change = np.max(np.abs(update[indices] / x[indices]))
if change < 1e-12:
# No change, halt
return (postprocess(x), -1)
# test for convergence
if normr < tol:
return (postprocess(x), 0)
# end outer loop
return (postprocess(x), niter)
def gcrotmk(A, b, x0=None, tol=1e-5, maxiter=1000, M=None, callback=None,
m=20, k=None, CU=None, discard_C=False, truncate='oldest',
atol=None):
"""
Solve a matrix equation using flexible GCROT(m,k) algorithm.
Parameters
----------
A : {sparse matrix, dense matrix, LinearOperator}
The real or complex N-by-N matrix of the linear system.
b : {array, matrix}
Right hand side of the linear system. Has shape (N,) or (N,1).
x0 : {array, matrix}
Starting guess for the solution.
tol, atol : float, optional
Tolerances for convergence, ``norm(residual) <= max(tol*norm(b), atol)``.
The default for ``atol`` is `tol`.
.. warning::
The default value for `atol` will be changed in a future release.
For future compatibility, specify `atol` explicitly.
maxiter : int, optional
Maximum number of iterations. Iteration will stop after maxiter
steps even if the specified tolerance has not been achieved.
M : {sparse matrix, dense matrix, LinearOperator}, optional
Preconditioner for A. The preconditioner should approximate the
inverse of A. gcrotmk is a 'flexible' algorithm and the preconditioner
can vary from iteration to iteration. Effective preconditioning
dramatically improves the rate of convergence, which implies that
fewer iterations are needed to reach a given error tolerance.
callback : function, optional
User-supplied function to call after each iteration. It is called
as callback(xk), where xk is the current solution vector.
m : int, optional
Number of inner FGMRES iterations per each outer iteration.
Default: 20
k : int, optional
Number of vectors to carry between inner FGMRES iterations.
According to [2]_, good values are around m.
Default: m
CU : list of tuples, optional
List of tuples ``(c, u)`` which contain the columns of the matrices
C and U in the GCROT(m,k) algorithm. For details, see [2]_.
The list given and vectors contained in it are modified in-place.
If not given, start from empty matrices. The ``c`` elements in the
tuples can be ``None``, in which case the vectors are recomputed
via ``c = A u`` on start and orthogonalized as described in [3]_.
discard_C : bool, optional
Discard the C-vectors at the end. Useful if recycling Krylov subspaces
for different linear systems.
truncate : {'oldest', 'smallest'}, optional
Truncation scheme to use. Drop: oldest vectors, or vectors with
smallest singular values using the scheme discussed in [1,2].
See [2]_ for detailed comparison.
Default: 'oldest'
Returns
-------
x : array or matrix
The solution found.
info : int
Provides convergence information:
* 0 : successful exit
* >0 : convergence to tolerance not achieved, number of iterations
References
----------
.. [1] E. de Sturler, ''Truncation strategies for optimal Krylov subspace
methods'', SIAM J. Numer. Anal. 36, 864 (1999).
.. [2] J.E. Hicken and D.W. Zingg, ''A simplified and flexible variant
of GCROT for solving nonsymmetric linear systems'',
SIAM J. Sci. Comput. 32, 172 (2010).
.. [3] M.L. Parks, E. de Sturler, G. Mackey, D.D. Johnson, S. Maiti,
''Recycling Krylov subspaces for sequences of linear systems'',
SIAM J. Sci. Comput. 28, 1651 (2006).
"""
A,M,x,b,postprocess = make_system(A,M,x0,b)
if not np.isfinite(b).all():
raise ValueError("RHS must contain only finite numbers")
if truncate not in ('oldest', 'smallest'):
raise ValueError("Invalid value for 'truncate': %r" % (truncate,))
if atol is None:
warnings.warn("scipy.sparse.linalg.gcrotmk called without specifying `atol`. "
"The default value will change in the future. To preserve "
"current behavior, set ``atol=tol``.",
category=DeprecationWarning, stacklevel=2)
atol = tol
matvec = A.matvec
psolve = M.matvec
if CU is None:
CU = []
if k is None:
k = m
axpy, dot, scal = None, None, None
r = b - matvec(x)
axpy, dot, scal, nrm2 = get_blas_funcs(['axpy', 'dot', 'scal', 'nrm2'], (x, r))
b_norm = nrm2(b)
if discard_C:
CU[:] = [(None, u) for c, u in CU]
# Reorthogonalize old vectors
if CU:
# Sort already existing vectors to the front
CU.sort(key=lambda cu: cu[0] is not None)
# Fill-in missing ones
C = np.empty((A.shape[0], len(CU)), dtype=r.dtype, order='F')
us = []
j = 0
while CU:
# More memory-efficient: throw away old vectors as we go
c, u = CU.pop(0)
if c is None:
c = matvec(u)
C[:,j] = c
j += 1
us.append(u)
# Orthogonalize
Q, R, P = qr(C, overwrite_a=True, mode='economic', pivoting=True)
del C
# C := Q
cs = list(Q.T)
# U := U P R^-1, back-substitution
new_us = []
for j in xrange(len(cs)):
u = us[P[j]]
for i in xrange(j):
u = axpy(us[P[i]], u, u.shape[0], -R[i,j])
if abs(R[j,j]) < 1e-12 * abs(R[0,0]):
# discard rest of the vectors
break
u = scal(1.0/R[j,j], u)
new_us.append(u)
# Form the new CU lists
CU[:] = list(zip(cs, new_us))[::-1]
if CU:
axpy, dot = get_blas_funcs(['axpy', 'dot'], (r,))
# Solve first the projection operation with respect to the CU
# vectors. This corresponds to modifying the initial guess to
# be
#
# x' = x + U y
# y = argmin_y || b - A (x + U y) ||^2
#
# The solution is y = C^H (b - A x)
for c, u in CU:
yc = dot(c, r)
x = axpy(u, x, x.shape[0], yc)
r = axpy(c, r, r.shape[0], -yc)
# GCROT main iteration
for j_outer in xrange(maxiter):
# -- callback
if callback is not None:
callback(x)
beta = nrm2(r)
# -- check stopping condition
beta_tol = max(atol, tol * b_norm)
if beta <= beta_tol and (j_outer > 0 or CU):
# recompute residual to avoid rounding error
r = b - matvec(x)
beta = nrm2(r)
if beta <= beta_tol:
j_outer = -1
break
ml = m + max(k - len(CU), 0)
cs = [c for c, u in CU]
try:
Q, R, B, vs, zs, y, pres = _fgmres(matvec,
r/beta,
ml,
rpsolve=psolve,
atol=max(atol, tol*b_norm)/beta,
cs=cs)
y *= beta
except LinAlgError:
# Floating point over/underflow, non-finite result from
# matmul etc. -- report failure.
break
#
# At this point,
#
# [A U, A Z] = [C, V] G; G = [ I B ]
# [ 0 H ]
#
# where [C, V] has orthonormal columns, and r = beta v_0. Moreover,
#
# || b - A (x + Z y + U q) ||_2 = || r - C B y - V H y - C q ||_2 = min!
#
# from which y = argmin_y || beta e_1 - H y ||_2, and q = -B y
#
#
# GCROT(m,k) update
#
# Define new outer vectors
# ux := (Z - U B) y
ux = zs[0]*y[0]
for z, yc in zip(zs[1:], y[1:]):
ux = axpy(z, ux, ux.shape[0], yc) # ux += z*yc
by = B.dot(y)
for cu, byc in zip(CU, by):
c, u = cu
ux = axpy(u, ux, ux.shape[0], -byc) # ux -= u*byc
# cx := V H y
hy = Q.dot(R.dot(y))
cx = vs[0] * hy[0]
for v, hyc in zip(vs[1:], hy[1:]):
cx = axpy(v, cx, cx.shape[0], hyc) # cx += v*hyc
# Normalize cx, maintaining cx = A ux
# This new cx is orthogonal to the previous C, by construction
try:
alpha = 1/nrm2(cx)
if not np.isfinite(alpha):
raise FloatingPointError()
except (FloatingPointError, ZeroDivisionError):
# Cannot update, so skip it
continue
cx = scal(alpha, cx)
ux = scal(alpha, ux)
# Update residual and solution
gamma = dot(cx, r)
r = axpy(cx, r, r.shape[0], -gamma) # r -= gamma*cx
x = axpy(ux, x, x.shape[0], gamma) # x += gamma*ux
# Truncate CU
if truncate == 'oldest':
while len(CU) >= k and CU:
del CU[0]
elif truncate == 'smallest':
if len(CU) >= k and CU:
# cf. [1,2]
D = solve(R[:-1,:].T, B.T).T
W, sigma, V = svd(D)
# C := C W[:,:k-1], U := U W[:,:k-1]
new_CU = []
for j, w in enumerate(W[:,:k-1].T):
c, u = CU[0]
c = c * w[0]
u = u * w[0]
for cup, wp in zip(CU[1:], w[1:]):
cp, up = cup
c = axpy(cp, c, c.shape[0], wp)
u = axpy(up, u, u.shape[0], wp)
# Reorthogonalize at the same time; not necessary
# in exact arithmetic, but floating point error
# tends to accumulate here
for cp, up in new_CU:
alpha = dot(cp, c)
c = axpy(cp, c, c.shape[0], -alpha)
u = axpy(up, u, u.shape[0], -alpha)
alpha = nrm2(c)
c = scal(1.0/alpha, c)
u = scal(1.0/alpha, u)
new_CU.append((c, u))
CU[:] = new_CU
# Add new vector to CU
CU.append((cx, ux))
# Include the solution vector to the span
CU.append((None, x.copy()))
if discard_C:
CU[:] = [(None, uz) for cz, uz in CU]
return postprocess(x), j_outer + 1
def gmres(A, b, x0=None, tol=1e-5, restrt=None, maxiter=None, xtype=None, M=None, callback=None, residuals=None):
'''
Generalized Minimum Residual Method (GMRES)
GMRES iteratively refines the initial solution guess to the system Ax = b
For robustness, Householder reflections are used to orthonormalize the Krylov Space
Givens Rotations are used to provide the residual norm each iteration
Parameters
----------
A : array, matrix or sparse matrix
n x n, linear system to solve
b : array
n x 1, right hand side
x0 : array
n x 1, initial guess
default is a vector of zeros
tol : float
convergence tolerance
restrt : int
number of restarts
total iterations = restrt*maxiter
maxiter : int
maximum number of allowed inner iterations
xtype : type
dtype for the solution
M : matrix-like
n x n, inverted preconditioner, i.e. solve M A x = b.
For preconditioning with a mat-vec routine, set
A.psolve = func, where func := M y
callback : function
callback( ||resid||_2 ) is called each iteration,
residuals : {None, empty-list}
If empty-list, residuals holds the residual norm history,
including the initial residual, upon completion
Returns
-------
(xNew, info)
xNew -- an updated guess to the solution of Ax = b
info -- halting status of gmres
0 : successful exit
>0 : convergence to tolerance not achieved,
return iteration count instead. This value
is precisely the order of the Krylov space.
<0 : numerical breakdown, or illegal input
Notes
-----
Examples
--------
>>>from pyamg.krylov import *
>>>from scipy import rand
>>>import pyamg
>>>A = pyamg.poisson((50,50))
>>>b = rand(A.shape[0],)
>>>(x,flag) = gmres(A,b)
>>>print pyamg.util.linalg.norm(b - A*x)
References
----------
Yousef Saad, "Iterative Methods for Sparse Linear Systems,
Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003
'''
# Convert inputs to linear system, with error checking
A,M,x,b,postprocess = make_system(A,M,x0,b,xtype)
dimen = A.shape[0]
# Choose type
xtype = upcast(A.dtype, x.dtype, b.dtype, M.dtype)
# We assume henceforth that shape=(n,) for all arrays
b = ravel(array(b,xtype))
x = ravel(array(x,xtype))
# Should norm(r) be kept
if residuals == []:
keep_r = True
else:
keep_r = False
# check number of iterations
if restrt == None:
restrt = 1
elif restrt < 1:
raise ValueError('Number of restarts must be positive')
if maxiter == None:
maxiter = int(max(ceil(dimen/restrt)))
elif maxiter < 1:
raise ValueError('Number of iterations must be positive')
elif maxiter > dimen:
warn('maximimum allowed inner iterations (maxiter) are the number of degress of freedom')
maxiter = dimen
# Scale tol by normb
normb = linalg.norm(b)
if normb == 0:
pass
# if callback != None:
# callback(0.0)
#
# return (postprocess(zeros((dimen,)), dtype=xtype),0)
else:
tol = tol*normb
# Is this a one dimensional matrix?
if dimen == 1:
entry = ravel(A*array([1.0], dtype=xtype))
return (postprocess(b/entry), 0)
# Prep for method
r = b - ravel(A*x)
normr = linalg.norm(r)
if keep_r:
residuals.append(normr)
# Is initial guess sufficient?
if normr <= tol:
if callback != None:
callback(norm(r))
return (postprocess(x), 0)
#Apply preconditioner
r = ravel(M*r)
normr = linalg.norm(r)
# Check for nan, inf
if any(isnan(r)) or any(isinf(r)):
warn('inf or nan after application of preconditioner')
return(postprocess(x), -1)
# Use separate variable to track iterations. If convergence fails, we cannot
# simply report niter = (outer-1)*maxiter + inner. Numerical error could cause
# the inner loop to halt before reaching maxiter while the actual ||r|| > tol.
niter = 0
# Begin GMRES
for outer in range(restrt):
# Calculate vector w, which defines the Householder reflector
# Take shortcut in calculating,
# w = r + sign(r[1])*||r||_2*e_1
w = r
beta = mysign(w[0])*normr
w[0] += beta
w = w / linalg.norm(w)
# Preallocate for Krylov vectors, Householder reflectors and Hessenberg matrix
# Space required is O(dimen*maxiter)
H = zeros( (maxiter, maxiter), dtype=xtype) # upper Hessenberg matrix (actually made upper tri with Given's Rotations)
W = zeros( (dimen, maxiter), dtype=xtype) # Householder reflectors
W[:,0] = w
# Multiply r with (I - 2*w*w.T), i.e. apply the Householder reflector
# This is the RHS vector for the problem in the Krylov Space
g = zeros((dimen,), dtype=xtype)
g[0] = -beta
for inner in range(maxiter):
# Calcute Krylov vector in two steps
# (1) Calculate v = P_j = (I - 2*w*w.T)v, where k = inner
v = -2.0*conjugate(w[inner])*w
v[inner] += 1.0
# (2) Calculate the rest, v = P_1*P_2*P_3...P_{j-1}*ej.
for j in range(inner-1,-1,-1):
v = v - 2.0*dot(conjugate(W[:,j]), v)*W[:,j]
# Calculate new search direction
v = ravel(A*v)
#Apply preconditioner
v = ravel(M*v)
# Check for nan, inf
if any(isnan(v)) or any(isinf(v)):
warn('inf or nan after application of preconditioner')
return(postprocess(x), -1)
# Factor in all Householder orthogonal reflections on new search direction
for j in range(inner+1):
v = v - 2.0*dot(conjugate(W[:,j]), v)*W[:,j]
# Calculate next Householder reflector, w
# w = v[inner+1:] + sign(v[inner+1])*||v[inner+1:]||_2*e_{inner+1)
# Note that if maxiter = dimen, then this is unnecessary for the last inner
# iteration, when inner = dimen-1. Here we do not need to calculate a
# Householder reflector or Given's rotation because nnz(v) is already the
# desired length, i.e. we do not need to zero anything out.
if inner != dimen-1:
w = zeros((dimen,), dtype=xtype)
vslice = v[inner+1:]
alpha = linalg.norm(vslice)
if alpha != 0:
alpha = mysign(vslice[0])*alpha
# We do not need the final reflector for future calculations
if inner < (maxiter-1):
w[inner+1:] = vslice
w[inner+1] += alpha
w = w / linalg.norm(w)
W[:,inner+1] = w
# Apply new reflector to v
# v = v - 2.0*w*(w.T*v)
v[inner+1] = -alpha
v[inner+2:] = 0.0
# Apply all previous Given's Rotations to v
if inner == 0:
# Q will store the cumulative effect of all Given's Rotations
Q = scipy.sparse.eye(dimen, dimen, format='csr', dtype=xtype)
# Declare initial Qj, which will be the current Given's Rotation
rowptr = hstack( (array([0, 2, 4],int), arange(5,dimen+3,dtype=int)) )
colindices = hstack( (array([0, 1, 0, 1],int), arange(2, dimen,dtype=int)) )
data = ones((dimen+2,), dtype=xtype)
Qj = csr_matrix( (data, colindices, rowptr), shape=(dimen,dimen), dtype=xtype)
else:
# Could avoid building a global Given's Rotation, by storing
# and applying each 2x2 matrix individually.
# But that would require looping, the bane of wonderful Python
Q = Qj*Q
v = Q*v
# Calculate Qj, the next Given's rotation, where j = inner
# Note that if maxiter = dimen, then this is unnecessary for the last inner
# iteration, when inner = dimen-1. Here we do not need to calculate a
# Householder reflector or Given's rotation because nnz(v) is already the
# desired length, i.e. we do not need to zero anything out.
if inner != dimen-1:
if v[inner+1] != 0:
# Calculate terms for complex 2x2 Given's Rotation
# Note that abs(x) takes the complex modulus
h1 = v[inner]; h2 = v[inner+1];
h1_mag = abs(h1); h2_mag = abs(h2);
if h1_mag < h2_mag:
mu = h1/h2
tau = conjugate(mu)/abs(mu)
else:
mu = h2/h1
tau = mu/abs(mu)
denom = sqrt( h1_mag**2 + h2_mag**2 )
c = h1_mag/denom; s = h2_mag*tau/denom;
Qblock = array([[c, conjugate(s)], [-s, c]], dtype=xtype)
# Modify Qj in csr-format so that it represents the current
# global Given's Rotation equivalent to Qblock
if inner != 0:
Qj.data[inner-1] = 1.0
Qj.indices[inner-1] = inner-1
Qj.indptr[inner-1] = inner-1
Qj.data[inner:inner+4] = ravel(Qblock)
Qj.indices[inner:inner+4] = [inner, inner+1, inner, inner+1]
Qj.indptr[inner:inner+3] = [inner, inner+2, inner+4]
# Apply Given's Rotation to g,
# the RHS for the linear system in the Krylov Subspace.
# Note that this dot does a matrix multiply, not an actual
# dot product where a conjugate transpose is taken
g[inner:inner+2] = dot(Qblock, g[inner:inner+2])
# Apply effect of Given's Rotation to v
v[inner] = dot(Qblock[0,:], v[inner:inner+2])
v[inner+1] = 0.0
# Write to upper Hessenberg Matrix,
# the LHS for the linear system in the Krylov Subspace
H[:,inner] = v[0:maxiter]
# Don't update normr if last inner iteration, because
# normr is calculated directly after this loop ends.
if inner < maxiter-1:
normr = abs(g[inner+1])
if normr < tol:
break
# Allow user access to residual
if callback != None:
callback( normr )
if keep_r:
residuals.append(normr)
niter += 1
# end inner loop, back to outer loop
# Find best update to x in Krylov Space, V. Solve inner+1 x inner+1 system.
# Apparently this is the best way to solve a triangular
# system in the magical world of scipy
piv = arange(inner+1)
y = lu_solve((H[0:(inner+1),0:(inner+1)], piv), g[0:(inner+1)], trans=0)
# Use Horner like Scheme to map solution, y, back to original space.
# Note that we do not use the last reflector.
update = zeros(x.shape, dtype=xtype)
for j in range(inner,-1,-1):
update[j] += y[j]
# Apply j-th reflector, (I - 2.0*w_j*w_j.T)*upadate
update = update - 2.0*dot(conjugate(W[:,j]), update)*W[:,j]
x = x + update
r = b - ravel(A*x)
#Apply preconditioner
r = ravel(M*r)
normr = linalg.norm(r)
# Check for nan, inf
if any(isnan(r)) or any(isinf(r)):
warn('inf or nan after application of preconditioner')
return(postprocess(x), -1)
# Allow user access to residual
if callback != None:
callback( normr )
if keep_r:
residuals.append(normr)
# Has GMRES stagnated?
indices = (x != 0)
if indices.any():
change = max(abs( update[indices] / x[indices] ))
if change < 1e-12:
# No change, halt
return (postprocess(x), -1)
# test for convergence
if normr < tol:
return (postprocess(x),0)
# end outer loop
return (postprocess(x), niter)
def idrs(A, b, x0=None, tol=1e-5, s=4, maxiter=None, M=None, callback=None):
"""
Use Induced Dimension Reduction method [IDR(s)] to solve A x = b
Parameters
----------
A : {sparse matrix, dense matrix, LinearOperator}
The real or complex N-by-N matrix of the linear system.
b : {array, matrix}
Right hand side of the linear system. Has shape (N,) or (N,1).
x0 : {array, matrix}
Starting guess for the solution.
tol : float, optional
Tolerance to achieve. The algorithm terminates when either the relative
or the absolute residual is below `tol`.
s : integer, optional
specifies the dimension of the shadow space. Normally, a higher
s gives faster convergence, but also makes the method more expensive.
Default is 4.
maxiter : integer, optional
Maximum number of iterations. Iteration will stop after maxiter
steps even if the specified tolerance has not been achieved.
M : {sparse matrix, dense matrix, LinearOperator}, optional
Preconditioner for A. The preconditioner should approximate the
inverse of A. Effective preconditioning dramatically improves the
rate of convergence, which implies that fewer iterations are needed
to reach a given error tolerance.
callback : function, optional
User-supplied function to call after each iteration. It is called
as callback(xk), where xk is the current solution vector.
Returns
-------
x : array or matrix
The converged solution.
info : integer
Provides convergence information:
- 0 : successful exit
- >0 : convergence to tolerance not achieved, number of iterations
- <0 : illegal input or breakdown
References
----------
.. [1] P. Sonneveld and M. B. van Gijzen
SIAM J. Sci. Comput. Vol. 31, No. 2, pp. 1035--1062, (2008).
.. [2] M. B. van Gijzen and P. Sonneveld
ACM Trans. Math. Software,, Vol. 38, No. 1, pp. 5:1-5:19, (2011).
.. [3] This file is a translation of the following MATLAB implementation:
http://ta.twi.tudelft.nl/nw/users/gijzen/idrs.m
"""
A, M, x, b, postprocess = make_system(A, M, x0, b)
n = len(b)
if maxiter is None:
maxiter = n * 10
matvec = A.matvec
psolve = M.matvec
xtype = x.dtype
axpy, dot = get_blas_funcs(['axpy', 'dot'], dtype=xtype)
np.random.seed(0)
P = np.random.randn(s, n)
bnrm = np.linalg.norm(b)
info = 0
if callback is not None:
callback(x)
# Check for zero rhs:
if bnrm == 0.0:
# Solution is null-vector
return postprocess(np.zeros(n, dtype=xtype)), 0
# Compute initial residual:
r = b - matvec(x)
rnrm = np.linalg.norm(r)
# Relative tolerance
tolb = tol * np.linalg.norm(b)
if rnrm < tolb:
# Initial guess is a good enough solution
return postprocess(x), 0
# Initialization
angle = 0.7
G = np.zeros((n, s), dtype=xtype)
U = np.zeros((n, s), dtype=xtype)
Ms = np.eye(s, dtype=xtype)
om = 1.0
iter_ = 0
# Main iteration loop, build G-spaces:
while rnrm >= tolb and iter_ < maxiter:
# New right-hand size for small system:
f = P.dot(r)
for k in range(0, s):
# Solve small system and make v orthogonal to P:
c = np.linalg.solve(Ms[k:s, k:s], f[k:s])
v = r - G[:, k:s].dot(c)
# Preconditioning:
v = psolve(v)
# Compute new U(:,k) and G(:,k), G(:,k) is in space G_j
U[:, k] = axpy(v, U[:, k:s].dot(c), None, om)
# Matrix-vector product
G[:, k] = matvec(U[:, k])
# Bi-Orthogonalize the new basis vectors:
for i in range(0, k):
alpha = dot(P[i, :], G[:, k]) / Ms[i, i]
G[:, k] = axpy(G[:, i], G[:, k], None, -alpha)
U[:, k] = axpy(U[:, i], U[:, k], None, -alpha)
# New column of M = P'*G (first k-1 entries are zero)
for i in range(k, s):
Ms[i, k] = dot(P[i, :], G[:, k])
if Ms[k, k] == 0.0:
# Breakdown
return postprocess(x), -1
# Make r orthogonal to g_i, i = 1..k
beta = f[k] / Ms[k, k]
x = axpy(U[:, k], x, None, beta)
r = axpy(G[:, k], r, None, -beta)
rnrm = np.linalg.norm(r)
if callback is not None:
callback(x)
iter_ += 1
if rnrm < tolb or iter_ >= maxiter:
break
# New f = P'*r (first k components are zero)
if k < s - 1:
f[k + 1:s] = axpy(Ms[k + 1:s, k], f[k + 1:s], None, -beta)
# Now we have sufficient vectors in G_j to compute residual in G_j+1
# Note: r is already perpendicular to P so v = r
if rnrm < tolb or iter_ >= maxiter:
break
# Preconditioning:
v = psolve(r)
# Matrix-vector product
t = matvec(v)
# Computation of a new omega
nr = np.linalg.norm(r)
nt = np.linalg.norm(t)
ts = dot(t, r)
rho = abs(ts / (nt * nr))
om = ts / (nt * nt)
if rho < angle:
om = om * angle / rho
# New vector in G_j+1
x = axpy(v, x, None, om)
r = axpy(t, r, None, -om)
rnrm = np.linalg.norm(r)
if callback is not None:
callback(x)
iter_ += 1
if rnrm >= tolb:
info = iter_
return postprocess(x), info
