def update_x_with_TR(self,x,alpha,d):
x_bk = x.copy()
x.axpy(alpha,d)
self.Bx.zero()
self.B_op.mult(x, self.Bx)
x_Bnorm2 = self.Bx.inner(x)
if x_Bnorm2 < self.TR_radius_2:
return False
else:
# Move point to boundary of trust region
self.Bx.zero()
self.B_op.mult(x_bk, self.Bx)
x_Bnorm2 = self.Bx.inner(x_bk)
Bd = Vector()
self.B_op.init_vector(Bd,0)
Bd.zero()
self.B_op.mult(self.d,Bd)
d_Bnorm2 = Bd.inner(d)
d_Bx = Bd.inner(x_bk)
a_tau = alpha*alpha*d_Bnorm2
b_tau_half = alpha* d_Bx
c_tau = x_Bnorm2- self.TR_radius_2
# Solve quadratic for tau
tau = (-b_tau_half + math.sqrt(b_tau_half*b_tau_half - a_tau*c_tau))/a_tau
x.zero()
x.axpy(1,x_bk)
x.axpy(tau*alpha, d)
return True
class Solver2Operator:
def __init__(self,S,mpi_comm=mpi_comm_world(), init_vector = None):
self.S = S
self.tmp = Vector(mpi_comm)
self.my_init_vector = init_vector
if self.my_init_vector is None:
if hasattr(self.S, "init_vector"):
self.my_init_vector = self.S.init_vector
elif hasattr(self.S, "operator"):
self.my_init_vector = self.S.operator().init_vector
elif hasattr(self.S, "get_operator"):
self.my_init_vector = self.S.get_operator().init_vector
def init_vector(self, x, dim):
if self.my_init_vector:
self.my_init_vector(x,dim)
else:
raise NotImplementedError("Solver2Operator.init_vector")
def mult(self,x,y):
self.S.solve(y,x)
def inner(self, x, y):
self.S.solve(self.tmp,y)
return self.tmp.inner(x)
class Solver2Operator:
def __init__(self, S, mpi_comm=mpi_comm_world(), init_vector=None):
self.S = S
self.tmp = Vector(mpi_comm)
self.my_init_vector = init_vector
if self.my_init_vector is None:
if hasattr(self.S, "init_vector"):
self.my_init_vector = self.S.init_vector
elif hasattr(self.S, "operator"):
self.my_init_vector = self.S.operator().init_vector
elif hasattr(self.S, "get_operator"):
self.my_init_vector = self.S.get_operator().init_vector
def init_vector(self, x, dim):
if self.my_init_vector:
self.my_init_vector(x, dim)
else:
raise NotImplementedError("Solver2Operator.init_vector")
def mult(self, x, y):
self.S.solve(y, x)
def inner(self, x, y):
self.S.solve(self.tmp, y)
return self.tmp.inner(x)
class Operator2Solver:
def __init__(self,op, mpi_comm=mpi_comm_world()):
self.op = op
self.tmp = Vector(mpi_comm)
def init_vector(self, x, dim):
if hasattr(self.op, "init_vector"):
self.op.init_vector(x,dim)
else:
raise
def solve(self,y,x):
self.op.mult(x,y)
def inner(self, x, y):
self.op.mult(y,self.tmp)
return self.tmp.inner(x)
class Operator2Solver:
def __init__(self, op, mpi_comm=mpi_comm_world()):
self.op = op
self.tmp = Vector(mpi_comm)
def init_vector(self, x, dim):
if hasattr(self.op, "init_vector"):
self.op.init_vector(x, dim)
else:
raise
def solve(self, y, x):
self.op.mult(x, y)
def inner(self, x, y):
self.op.mult(y, self.tmp)
return self.tmp.inner(x)
class Solver2Operator:
def __init__(self, S):
self.S = S
self.tmp = Vector()
def init_vector(self, x, dim):
if hasattr(self.S, "init_vector"):
self.S.init_vector(x, dim)
elif hasattr(self.S, "operator"):
self.S.operator().init_vector(x, dim)
else:
raise
def mult(self, x, y):
self.S.solve(y, x)
def inner(self, x, y):
self.S.solve(self.tmp, y)
return self.tmp.inner(x)
def inner(self, x, y):
Hx = Vector(self.help.mpi_comm())
self.init_vector(Hx, 0)
self.mult(x, Hx)
return Hx.inner(y)
class CGSolverSteihaug:
"""
Solve the linear system A x = b using preconditioned conjugate gradient ( B preconditioner)
and the Steihaug stopping criterion:
- reason of termination 0: we reached the maximum number of iterations (no convergence)
- reason of termination 1: we reduced the residual up to the given tolerance (convergence)
- reason of termination 2: we reached a negative direction (premature termination due to not spd matrix)
The operator A is set using the method set_operator(A).
A must provide the following two methods:
- A.mult(x,y): y = A*x
- A.init_vector(x, dim): initialize the vector x so that it is compatible with the range (dim = 0) or
the domain (dim = 1) of A.
The preconditioner B is set using the method set_preconditioner(B).
B must provide the following method:
- B.solve(z,r): z is the action of the preconditioner B on the vector r
To solve the linear system A*x = b call solve(x,b). Here x and b are assumed
to be FEniCS::Vector objects
Maximum number of iterations, tolerances, verbosity level etc can be
set using the parameters attributes.
"""
reason = [
"Maximum Number of Iterations Reached",
"Relative/Absolute residual less than tol",
"Reached a negative direction"
]
def __init__(self):
self.parameters = {}
self.parameters["rel_tolerance"] = 1e-9
self.parameters["abs_tolerance"] = 1e-12
self.parameters["max_iter"] = 1000
self.parameters["zero_initial_guess"] = True
self.parameters["print_level"] = 0
self.A = None
self.B = None
self.converged = False
self.iter = 0
self.reasonid = 0
self.final_norm = 0
self.r = Vector()
self.z = Vector()
self.d = Vector()
if dolfin.__version__[2] == '3': self.vrs130 = True
else: self.vrs130 = False
def set_operator(self, A):
self.A = A
if self.vrs130:
self.r = self.A.init_vector130()
self.z = self.A.init_vector130()
self.d = self.A.init_vector130()
else:
self.A.init_vector(self.r, 0)
self.A.init_vector(self.z, 0)
self.A.init_vector(self.d, 0)
def set_preconditioner(self, B):
self.B = B
def solve(self, x, b):
self.iter = 0
self.converged = False
self.reasonid = 0
betanom = 0.0
alpha = 0.0
beta = 0.0
if self.parameters["zero_initial_guess"]:
self.r.zero()
self.r.axpy(1.0, b)
x.zero()
else:
self.A.mult(x, self.r)
self.r *= -1.0
self.r.axpy(1.0, b)
self.z.zero()
self.B.solve(self.z, self.r) #z = B^-1 r
self.d.zero()
self.d.axpy(1., self.z)
#d = z
nom0 = self.d.inner(self.r)
nom = nom0
if self.parameters["print_level"] == 1:
print " Iterartion : ", 0, " (B r, r) = ", nom
rtol2 = nom * self.parameters["rel_tolerance"] * self.parameters[
"rel_tolerance"]
atol2 = self.parameters["abs_tolerance"] * self.parameters[
"abs_tolerance"]
r0 = max(rtol2, atol2)
if nom <= r0:
self.converged = True
self.reasonid = 1
self.final_norm = math.sqrt(nom)
if (self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
return
self.A.mult(self.d, self.z) #z = A d
den = self.z.inner(self.d)
if den <= 0.0:
self.converged = True
self.reasonid = 2
self.final_norm = math.sqrt(nom)
if (self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
return
# start iteration
self.iter = 1
while True:
alpha = nom / den
x.axpy(alpha, self.d) # x = x + alpha d
self.r.axpy(-alpha, self.z) # r = r - alpha A d
self.B.solve(self.z, self.r) # z = B^-1 r
betanom = self.r.inner(self.z)
if self.parameters["print_level"] == 1:
print " Iteration : ", self.iter, " (B r, r) = ", betanom
if betanom < r0:
self.converged = True
self.reasonid = 1
self.final_norm = math.sqrt(betanom)
if (self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
break
self.iter += 1
if self.iter > self.parameters["max_iter"]:
self.converged = False
self.reasonid = 0
self.final_norm = math.sqrt(betanom)
if (self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Not Converged. Final residual norm ", self.final_norm
break
beta = betanom / nom
self.d *= beta
self.d.axpy(1., self.z) #d = z + beta d
self.A.mult(self.d, self.z) # z = A d
den = self.d.inner(self.z)
if den <= 0.0:
self.converged = True
self.reasonid = 2
self.final_norm = math.sqrt(nom)
if (self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
break
nom = betanom
def solve(self, F, u, grad = None, H = None):
if grad is None:
print "Using Automatic Differentiation to compute the gradient"
grad = derivative(F,u)
if H is None:
print "Using Automatic Differentiation to compute the Hessian"
H = derivative(grad, u)
rtol = self.parameters["rel_tolerance"]
atol = self.parameters["abs_tolerance"]
gdu_tol = self.parameters["gdu_tolerance"]
max_iter = self.parameters["max_iter"]
c_armijo = self.parameters["c_armijo"]
max_backtrack = self.parameters["max_backtracking_iter"]
prt_level = self.parameters["print_level"]
cg_coarsest_tol = self.parameters["cg_coarse_tolerance"]
Fn = assemble(F)
gn = assemble(grad)
g0_norm = gn.norm("l2")
gn_norm = g0_norm
tol = max(g0_norm*rtol, atol)
du = Vector()
self.converged = False
self.reason = 0
if prt_level > 0:
print "{0:3} {1:15} {2:15} {3:15} {4:15} {5:15} {6:5}".format(
"It", "Energy", "||g||", "(g,du)", "alpha", "tol_cg", "cg_it")
for self.it in range(max_iter):
Hn = assemble(H)
Hn.init_vector(du,1)
solver = PETScKrylovSolver("cg", "petsc_amg")
solver.set_operator(Hn)
solver.parameters["nonzero_initial_guess"] = False
cg_tol = min(cg_coarsest_tol, math.sqrt( gn_norm/g0_norm) )
solver.parameters["relative_tolerance"] = cg_tol
lin_it = solver.solve(du,gn)
self.total_cg_iter += lin_it
du_gn = -du.inner(gn)
if(-du_gn < gdu_tol):
self.converged=True
self.reason = 3
break
u_backtrack = u.copy(deepcopy=True)
alpha = 1.
bk_converged = False
#Backtrack
for j in range(max_backtrack):
u.assign(u_backtrack)
u.vector().axpy(-alpha, du)
Fnext = assemble(F)
if Fnext < Fn + alpha*c_armijo*du_gn:
Fn = Fnext
bk_converged = True
break
alpha = alpha/2.
if not bk_converged:
self.reason = 2
break
gn = assemble(grad)
gn_norm = gn.norm("l2")
if prt_level > 0:
print "{0:3d} {1:15f} {2:15f} {3:15f} {4:15f} {5:15f} {6:5d}".format(
self.it, Fn, gn_norm, du_gn, alpha, cg_tol, lin_it)
if gn_norm < tol:
self.converged = True
self.reason = 1
break
self.final_grad_norm = gn_norm
if prt_level > -1:
print self.termination_reasons[self.reason]
if self.converged:
print "Inexact Newton CG converged in ", self.it, \
"nonlinear iterations and ", self.total_cg_iter, "linear iterations."
else:
print "Inexact Newton CG did NOT converge after ", self.it, \
"nonlinear iterations and ", self.total_cg_iter, "linear iterations."
print "Final norm of the gradient", self.final_grad_norm
print "Value of the cost functional", Fn
class CGSampler:
"""
This class implements the CG sampler algorithm to generate samples from :math:`\mathcal{N}(0, A^{-1})`.
Reference:
`Albert Parker and Colin Fox Sampling Gaussian Distributions in Krylov Spaces with Conjugate Gradient
SIAM J SCI COMPUT, Vol 34, No. 3 pp. B312-B334`
"""
def __init__(self):
"""
Construct the solver with default parameters
:code:`tolerance = 1e-4`
:code:`print_level = 0`
:code:`verbose = 0`
"""
self.parameters = {}
self.parameters["tolerance"] = 1e-4
self.parameters["print_level"] = 0
self.parameters["verbose"] = 0
self.A = None
self.converged = False
self.iter = 0
self.b = Vector()
self.r = Vector()
self.p = Vector()
self.Ap = Vector()
def set_operator(self, A):
"""
Set the operator :code:`A`, such that :math:`x \sim \mathcal{N}(0, A^{-1})`.
.. note:: :code:`A` is any object that provides the methods :code:`init_vector()` and :code:`mult()`
"""
self.A = A
self.A.init_vector(self.r, 0)
self.A.init_vector(self.p, 0)
self.A.init_vector(self.Ap, 0)
self.A.init_vector(self.b, 0)
parRandom.normal(1., self.b)
def sample(self, noise, s):
"""
Generate a sample :math:`s ~ N(0, A^{-1})`.
:code:`noise` is a :code:`numpy.array` of i.i.d. normal variables used as input.
For a fixed realization of noise the algorithm is fully deterministic.
The size of noise determine the maximum number of CG iterations.
"""
s.zero()
self.iter = 0
self.converged = False
# r0 = b
self.r.zero()
self.r.axpy(1., self.b)
#p0 = r0
self.p.zero()
self.p.axpy(1., self.r)
self.A.mult(self.p, self.Ap)
d = self.p.inner(self.Ap)
tol2 = self.parameters["tolerance"] * self.parameters["tolerance"]
rnorm2_old = self.r.inner(self.r)
if self.parameters["verbose"] > 0:
print("initial residual = {0:g}".format(math.sqrt(rnorm2_old)))
while (not self.converged) and (self.iter < noise.shape[0]):
gamma = rnorm2_old / d
s.axpy(noise[self.iter] / math.sqrt(d), self.p)
self.r.axpy(-gamma, self.Ap)
rnorm2 = self.r.inner(self.r)
beta = rnorm2 / rnorm2_old
# p_new = r + beta p
self.p *= beta
self.p.axpy(1., self.r)
self.A.mult(self.p, self.Ap)
d = self.p.inner(self.Ap)
rnorm2_old = rnorm2
if rnorm2 < tol2:
self.converged = True
else:
rnorm2_old = rnorm2
self.iter = self.iter + 1
if self.parameters["verbose"] > 0:
print("Final residual {0} after {1} iterations".format(
math.sqrt(rnorm2_old), self.iter))
class TraceEstimator:
"""
An unbiased stochastic estimator for the trace of A.
d = \sum_{j=1}^k (vj, A vj)
where
- vj are i.i.d. Rademacher or Gaussian random vectors
- (.,.) represents the inner product
The number of samples k is estimated at run time based on
the variance of the estimator.
REFERENCE:
Haim Avron and Sivan Toledo,
Randomized algorithms for estimating the trace of an implicit symmetric positive semi-definite matrix,
Journal of the ACM (JACM), 58 (2011), p. 17.
"""
def __init__(self,
A,
solve_mode=False,
accurancy=1e-1,
init_vector=None,
random_engine=rademacher_engine):
"""
Constructor:
- A: an operator
- solve_mode: if True we estimate the trace of A^{-1}, otherwise of A.
- accurancy: we stop when the standard deviation of the estimator is less then
accurancy*tr(A).
- init_vector: use a custom function to initialize a vector compatible with the
range/domain of A
- random_engine: which type of i.i.d. random variables to use (Rademacher or Gaussian)
"""
self.A = A
self.accurancy = accurancy
self.random_engine = random_engine
self.iter = 0
self.z = Vector()
self.Az = Vector()
if solve_mode:
self._apply = self._apply_solve
else:
self._apply = self._apply_mult
if init_vector is None:
A.init_vector(self.z, 0)
A.init_vector(self.Az, 0)
else:
init_vector(self.z, 0)
init_vector(self.Az, 0)
def _apply_mult(self, z, Az):
self.A.mult(z, Az)
def _apply_solve(self, z, Az):
self.A.solve(Az, z)
def __call__(self, min_iter=5, max_iter=100):
"""
Estimate the trace of A (or A^-1) using at least
min_iter and at most max_iter samples.
"""
sum_tr = 0
sum_tr2 = 0
self.iter = 0
size = len(self.z.array())
while self.iter < min_iter:
self.iter += 1
self.z.set_local(self.random_engine(size))
self._apply(self.z, self.Az)
tr = self.z.inner(self.Az)
sum_tr += tr
sum_tr2 += tr * tr
exp_tr = sum_tr / float(self.iter)
exp_tr2 = sum_tr2 / float(self.iter)
var_tr = exp_tr2 - exp_tr * exp_tr
# print(exp_tr, math.sqrt( var_tr ), self.accurancy*exp_tr)
self.converged = True
while (math.sqrt(var_tr) > self.accurancy * exp_tr):
self.iter += 1
self.z.set_local(self.random_engine(size))
self._apply(self.z, self.Az)
tr = self.z.inner(self.Az)
sum_tr += tr
sum_tr2 += tr * tr
exp_tr = sum_tr / float(self.iter)
exp_tr2 = sum_tr2 / float(self.iter)
var_tr = exp_tr2 - exp_tr * exp_tr
# print(exp_tr, math.sqrt( var_tr ), self.accurancy*exp_tr)
if (self.iter > max_iter):
self.converged = False
break
return exp_tr, var_tr
class CGSampler:
"""
This class implements the CG sampler algorithm to generate samples from :math:`\mathcal{N}(0, A^{-1})`.
Reference:
`Albert Parker and Colin Fox Sampling Gaussian Distributions in Krylov Spaces with Conjugate Gradient
SIAM J SCI COMPUT, Vol 34, No. 3 pp. B312-B334`
"""
def __init__(self):
"""
Construct the solver with default parameters
:code:`tolerance = 1e-4`
:code:`print_level = 0`
:code:`verbose = 0`
"""
self.parameters = {}
self.parameters["tolerance"] = 1e-4
self.parameters["print_level"] = 0
self.parameters["verbose"] = 0
self.A = None
self.converged = False
self.iter = 0
self.b = Vector()
self.r = Vector()
self.p = Vector()
self.Ap = Vector()
def set_operator(self, A):
"""
Set the operator :code:`A`, such that :math:`x \sim \mathcal{N}(0, A^{-1})`.
.. note:: :code:`A` is any object that provides the methods :code:`init_vector()` and :code:`mult()`
"""
self.A = A
self.A.init_vector(self.r,0)
self.A.init_vector(self.p,0)
self.A.init_vector(self.Ap,0)
self.A.init_vector(self.b,0)
parRandom.normal(1., self.b)
def sample(self, noise, s):
"""
Generate a sample :math:`s ~ N(0, A^{-1})`.
:code:`noise` is a :code:`numpy.array` of i.i.d. normal variables used as input.
For a fixed realization of noise the algorithm is fully deterministic.
The size of noise determine the maximum number of CG iterations.
"""
s.zero()
self.iter = 0
self.converged = False
# r0 = b
self.r.zero()
self.r.axpy(1., self.b)
#p0 = r0
self.p.zero()
self.p.axpy(1., self.r)
self.A.mult(self.p, self.Ap)
d = self.p.inner(self.Ap)
tol2 = self.parameters["tolerance"]*self.parameters["tolerance"]
rnorm2_old = self.r.inner(self.r)
if self.parameters["verbose"] > 0:
print("initial residual = {0:g}".format( math.sqrt(rnorm2_old) ))
while (not self.converged) and (self.iter < noise.shape[0]):
gamma = rnorm2_old/d
s.axpy(noise[self.iter]/math.sqrt(d), self.p)
self.r.axpy(-gamma, self.Ap)
rnorm2 = self.r.inner(self.r)
beta = rnorm2/rnorm2_old
# p_new = r + beta p
self.p *= beta
self.p.axpy(1., self.r)
self.A.mult(self.p, self.Ap)
d = self.p.inner(self.Ap)
rnorm2_old = rnorm2
if rnorm2 < tol2:
self.converged = True
else:
rnorm2_old = rnorm2
self.iter = self.iter+1
if self.parameters["verbose"] > 0:
print("Final residual {0} after {1} iterations".format( math.sqrt(rnorm2_old), self.iter))
def solve(self, F, u, grad=None, H=None):
if grad is None:
print("Using Symbolic Differentiation to compute the gradient")
grad = derivative(F, u)
if H is None:
print("Using Symbolic Differentiation to compute the Hessian")
H = derivative(grad, u)
rtol = self.parameters["rel_tolerance"]
atol = self.parameters["abs_tolerance"]
gdu_tol = self.parameters["gdu_tolerance"]
max_iter = self.parameters["max_iter"]
c_armijo = self.parameters["c_armijo"]
max_backtrack = self.parameters["max_backtracking_iter"]
prt_level = self.parameters["print_level"]
cg_coarsest_tol = self.parameters["cg_coarse_tolerance"]
Fn = assemble(F)
gn = assemble(grad)
g0_norm = gn.norm("l2")
gn_norm = g0_norm
tol = max(g0_norm * rtol, atol)
du = Vector()
self.converged = False
self.reason = 0
if prt_level > 0:
print(
"{0:>3} {1:>15} {2:>15} {3:>15} {4:>15} {5:>15} {6:>7}".format(
"It", "Energy", "||g||", "(g,du)", "alpha", "tol_cg",
"cg_it"))
for self.it in range(max_iter):
Hn = assemble(H)
Hn.init_vector(du, 1)
solver = PETScKrylovSolver("cg", "petsc_amg")
solver.set_operator(Hn)
solver.parameters["nonzero_initial_guess"] = False
cg_tol = min(cg_coarsest_tol, math.sqrt(gn_norm / g0_norm))
solver.parameters["relative_tolerance"] = cg_tol
lin_it = solver.solve(du, gn)
self.total_cg_iter += lin_it
du_gn = -du.inner(gn)
if (-du_gn < gdu_tol):
self.converged = True
self.reason = 3
break
u_backtrack = u.copy(deepcopy=True)
alpha = 1.
bk_converged = False
#Backtrack
for j in range(max_backtrack):
u.assign(u_backtrack)
u.vector().axpy(-alpha, du)
Fnext = assemble(F)
if Fnext < Fn + alpha * c_armijo * du_gn:
Fn = Fnext
bk_converged = True
break
alpha = alpha / 2.
if not bk_converged:
self.reason = 2
break
gn = assemble(grad)
gn_norm = gn.norm("l2")
if prt_level > 0:
print("{0:3d} {1:15e} {2:15e} {3:15e} {4:15e} {5:15e} {6:7d}".
format(self.it, Fn, gn_norm, du_gn, alpha, cg_tol,
lin_it))
if gn_norm < tol:
self.converged = True
self.reason = 1
break
self.final_grad_norm = gn_norm
if prt_level > -1:
print(self.termination_reasons[self.reason])
if self.converged:
print( "Inexact Newton CG converged in ", self.it, \
"nonlinear iterations and ", self.total_cg_iter, "linear iterations." )
else:
print( "Inexact Newton CG did NOT converge after ", self.it, \
"nonlinear iterations and ", self.total_cg_iter, "linear iterations.")
print("Final norm of the gradient", self.final_grad_norm)
print("Value of the cost functional", Fn)
class CGSolverSteihaug:
"""
Solve the linear system A x = b using preconditioned conjugate gradient ( B preconditioner)
and the Steihaug stopping criterion:
- reason of termination 0: we reached the maximum number of iterations (no convergence)
- reason of termination 1: we reduced the residual up to the given tolerance (convergence)
- reason of termination 2: we reached a negative direction (premature termination due to not spd matrix)
The operator A is set using the method set_operator(A).
A must provide the following two methods:
- A.mult(x,y): y = A*x
- A.init_vector(x, dim): initialize the vector x so that it is compatible with the range (dim = 0) or
the domain (dim = 1) of A.
The preconditioner B is set using the method set_preconditioner(B).
B must provide the following method:
- B.solve(z,r): z is the action of the preconditioner B on the vector r
To solve the linear system A*x = b call solve(x,b). Here x and b are assumed
to be FEniCS::Vector objects
Maximum number of iterations, tolerances, verbosity level etc can be
set using the parameters attributes.
"""
reason = ["Maximum Number of Iterations Reached",
"Relative/Absolute residual less than tol",
"Reached a negative direction"
]
def __init__(self):
self.parameters = {}
self.parameters["rel_tolerance"] = 1e-9
self.parameters["abs_tolerance"] = 1e-12
self.parameters["max_iter"] = 1000
self.parameters["zero_initial_guess"] = True
self.parameters["print_level"] = 0
self.A = None
self.B = None
self.converged = False
self.iter = 0
self.reasonid = 0
self.final_norm = 0
self.r = Vector()
self.z = Vector()
self.d = Vector()
if dolfin.__version__[2] == '5': self.vrs150 = True
else: self.vrs150 = False
def set_operator(self, A):
self.A = A
if self.vrs150:
self.A.init_vector(self.r,0)
self.A.init_vector(self.z,0)
self.A.init_vector(self.d,0)
else:
self.r = self.A.init_vector130()
self.z = self.A.init_vector130()
self.d = self.A.init_vector130()
def set_preconditioner(self, B):
self.B = B
def solve(self,x,b):
self.iter = 0
self.converged = False
self.reasonid = 0
betanom = 0.0
alpha = 0.0
beta = 0.0
if self.parameters["zero_initial_guess"]:
self.r.zero()
self.r.axpy(1.0, b)
x.zero()
else:
self.A.mult(x,self.r)
self.r *= -1.0
self.r.axpy(1.0, b)
self.z.zero()
self.B.solve(self.z,self.r) #z = B^-1 r
self.d.zero()
self.d.axpy(1.,self.z); #d = z
nom0 = self.d.inner(self.r)
nom = nom0
if self.parameters["print_level"] == 1:
print " Iterartion : ", 0, " (B r, r) = ", nom
rtol2 = nom * self.parameters["rel_tolerance"] * self.parameters["rel_tolerance"]
atol2 = self.parameters["abs_tolerance"] * self.parameters["abs_tolerance"]
r0 = max(rtol2, atol2)
if nom <= r0:
self.converged = True
self.reasonid = 1
self.final_norm = math.sqrt(nom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
return
self.A.mult(self.d, self.z) #z = A d
den = self.z.inner(self.d)
if den <= 0.0:
self.converged = True
self.reasonid = 2
self.final_norm = math.sqrt(nom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
return
# start iteration
self.iter = 1
while True:
alpha = nom/den
x.axpy(alpha,self.d) # x = x + alpha d
self.r.axpy(-alpha, self.z) # r = r - alpha A d
self.B.solve(self.z, self.r) # z = B^-1 r
betanom = self.r.inner(self.z)
if self.parameters["print_level"] == 1:
print " Iteration : ", self.iter, " (B r, r) = ", betanom
if betanom < r0:
self.converged = True
self.reasonid = 1
self.final_norm = math.sqrt(betanom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
break
self.iter += 1
if self.iter > self.parameters["max_iter"]:
self.converged = False
self.reasonid = 0
self.final_norm = math.sqrt(betanom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Not Converged. Final residual norm ", self.final_norm
break
beta = betanom/nom
self.d *= beta
self.d.axpy(1., self.z) #d = z + beta d
self.A.mult(self.d,self.z) # z = A d
den = self.d.inner(self.z)
if den <= 0.0:
self.converged = True
self.reasonid = 2
self.final_norm = math.sqrt(nom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
break
nom = betanom
def inner(self,x,y):
Hx = Vector(self.help.mpi_comm())
self.init_vector(Hx, 0)
self.mult(x, Hx)
return Hx.inner(y)
class CGSolverSteihaug:
"""
Solve the linear system A x = b using preconditioned conjugate gradient ( B preconditioner)
and the Steihaug stopping criterion:
- reason of termination 0: we reached the maximum number of iterations (no convergence)
- reason of termination 1: we reduced the residual up to the given tolerance (convergence)
- reason of termination 2: we reached a negative direction (premature termination due to not spd matrix)
The stopping criterion is based on either
- the absolute preconditioned residual norm check: || r^* ||_{B^{-1}} < atol
- the relative preconditioned residual norm check: || r^* ||_{B^{-1}}/|| r^0 ||_{B^{-1}} < rtol
where r^* = b - Ax^* is the residual at convergence and r^0 = b - Ax^0 is the initial residual.
The operator A is set using the method set_operator(A).
A must provide the following two methods:
- A.mult(x,y): y = A*x
- A.init_vector(x, dim): initialize the vector x so that it is compatible with the range (dim = 0) or
the domain (dim = 1) of A.
The preconditioner B is set using the method set_preconditioner(B).
B must provide the following method:
- B.solve(z,r): z is the action of the preconditioner B on the vector r
To solve the linear system A*x = b call self.solve(x,b).
Here x and b are assumed to be FEniCS::Vector objects.
The parameter attributes allows to set:
- rel_tolerance : the relative tolerance for the stopping criterion
- abs_tolerance : the absolute tolerance for the stopping criterion
- max_iter : the maximum number of iterations
- zero_initial_guess: if True we start with a 0 initial guess
if False we use the x as initial guess.
- print_level : verbosity level:
-1 --> no output on screen
0 --> only final residual at convergence
or reason for not not convergence
"""
reason = [
"Maximum Number of Iterations Reached",
"Relative/Absolute residual less than tol",
"Reached a negative direction"
]
def __init__(self):
self.parameters = {}
self.parameters["rel_tolerance"] = 1e-9
self.parameters["abs_tolerance"] = 1e-12
self.parameters["max_iter"] = 1000
self.parameters["zero_initial_guess"] = True
self.parameters["print_level"] = 0
self.A = None
self.B = None
self.converged = False
self.iter = 0
self.reasonid = 0
self.final_norm = 0
self.r = Vector()
self.z = Vector()
self.d = Vector()
def set_operator(self, A):
"""
Set the operator A.
"""
self.A = A
self.A.init_vector(self.r, 0)
self.A.init_vector(self.z, 0)
self.A.init_vector(self.d, 0)
def set_preconditioner(self, B):
"""
Set the preconditioner B.
"""
self.B = B
def solve(self, x, b):
"""
Solve the linear system Ax = b
"""
self.iter = 0
self.converged = False
self.reasonid = 0
betanom = 0.0
alpha = 0.0
beta = 0.0
if self.parameters["zero_initial_guess"]:
self.r.zero()
self.r.axpy(1.0, b)
x.zero()
else:
self.A.mult(x, self.r)
self.r *= -1.0
self.r.axpy(1.0, b)
self.z.zero()
self.B.solve(self.z, self.r) #z = B^-1 r
self.d.zero()
self.d.axpy(1., self.z)
#d = z
nom0 = self.d.inner(self.r)
nom = nom0
if self.parameters["print_level"] == 1:
print(" Iterartion : ", 0, " (B r, r) = ", nom)
rtol2 = nom * self.parameters["rel_tolerance"] * self.parameters[
"rel_tolerance"]
atol2 = self.parameters["abs_tolerance"] * self.parameters[
"abs_tolerance"]
r0 = max(rtol2, atol2)
if nom <= r0:
self.converged = True
self.reasonid = 1
self.final_norm = math.sqrt(nom)
if (self.parameters["print_level"] >= 0):
print(self.reason[self.reasonid])
print("Converged in ", self.iter,
" iterations with final norm ", self.final_norm)
return
self.A.mult(self.d, self.z) #z = A d
den = self.z.inner(self.d)
if den <= 0.0:
self.converged = True
self.reasonid = 2
self.final_norm = math.sqrt(nom)
if (self.parameters["print_level"] >= 0):
print(self.reason[self.reasonid])
print("Converged in ", self.iter,
" iterations with final norm ", self.final_norm)
return
# start iteration
self.iter = 1
while True:
alpha = nom / den
x.axpy(alpha, self.d) # x = x + alpha d
self.r.axpy(-alpha, self.z) # r = r - alpha A d
self.B.solve(self.z, self.r) # z = B^-1 r
betanom = self.r.inner(self.z)
if self.parameters["print_level"] == 1:
print(" Iteration : ", self.iter, " (B r, r) = ", betanom)
if betanom < r0:
self.converged = True
self.reasonid = 1
self.final_norm = math.sqrt(betanom)
if (self.parameters["print_level"] >= 0):
print(self.reason[self.reasonid])
print("Converged in ", self.iter,
" iterations with final norm ", self.final_norm)
break
self.iter += 1
if self.iter > self.parameters["max_iter"]:
self.converged = False
self.reasonid = 0
self.final_norm = math.sqrt(betanom)
if (self.parameters["print_level"] >= 0):
print(self.reason[self.reasonid])
print("Not Converged. Final residual norm ",
self.final_norm)
break
beta = betanom / nom
self.d *= beta
self.d.axpy(1., self.z) #d = z + beta d
self.A.mult(self.d, self.z) # z = A d
den = self.d.inner(self.z)
if den <= 0.0:
self.converged = True
self.reasonid = 2
self.final_norm = math.sqrt(nom)
if (self.parameters["print_level"] >= 0):
print(self.reason[self.reasonid])
print("Converged in ", self.iter,
" iterations with final norm ", self.final_norm)
break
nom = betanom
def inner(self, x, y):
Hx = Vector()
self.init_vector(Hx, 0)
self.mult(x, Hx)
return Hx.inner(y)
class CGSolverSteihaug:
"""
Solve the linear system A x = b using preconditioned conjugate gradient ( B preconditioner)
and the Steihaug stopping criterion:
- reason of termination 0: we reached the maximum number of iterations (no convergence)
- reason of termination 1: we reduced the residual up to the given tolerance (convergence)
- reason of termination 2: we reached a negative direction (premature termination due to not spd matrix)
- reason of termination 3: we reached the boundary of the trust region
The stopping criterion is based on either
- the absolute preconditioned residual norm check: || r^* ||_{B^{-1}} < atol
- the relative preconditioned residual norm check: || r^* ||_{B^{-1}}/|| r^0 ||_{B^{-1}} < rtol
where r^* = b - Ax^* is the residual at convergence and r^0 = b - Ax^0 is the initial residual.
The operator A is set using the method set_operator(A).
A must provide the following two methods:
- A.mult(x,y): y = A*x
- A.init_vector(x, dim): initialize the vector x so that it is compatible with the range (dim = 0) or
the domain (dim = 1) of A.
The preconditioner B is set using the method set_preconditioner(B).
B must provide the following method:
- B.solve(z,r): z is the action of the preconditioner B on the vector r
To solve the linear system A*x = b call self.solve(x,b).
Here x and b are assumed to be FEniCS::Vector objects.
Type: CGSolverSteihaug_ParameterList().showMe() for default parameters and their descriptions
"""
reason = ["Maximum Number of Iterations Reached",
"Relative/Absolute residual less than tol",
"Reached a negative direction",
"Reached trust region boundary"
]
def __init__(self, parameters=CGSolverSteihaug_ParameterList()):
self.parameters = parameters
self.A = None
self.B_solver = None
self.B_op = None
self.converged = False
self.iter = 0
self.reasonid = 0
self.final_norm = 0
self.TR_radius_2 = None
self.update_x = self.update_x_without_TR
self.r = Vector()
self.z = Vector()
self.d = Vector()
self.Bx = Vector()
def set_operator(self, A):
"""
Set the operator A.
"""
self.A = A
self.A.init_vector(self.r,0)
self.A.init_vector(self.z,0)
self.A.init_vector(self.d,0)
def set_preconditioner(self, B_solver):
"""
Set the preconditioner B.
"""
self.B_solver = B_solver
def set_TR(self,radius,B_op):
assert self.parameters["zero_initial_guess"]
self.TR_radius_2 = radius*radius
self.update_x = self.update_x_with_TR
self.B_op = B_op
self.B_op.init_vector(self.Bx,0)
def update_x_without_TR(self,x,alpha,d):
x.axpy(alpha,d)
return False
def update_x_with_TR(self,x,alpha,d):
x_bk = x.copy()
x.axpy(alpha,d)
self.Bx.zero()
self.B_op.mult(x, self.Bx)
x_Bnorm2 = self.Bx.inner(x)
if x_Bnorm2 < self.TR_radius_2:
return False
else:
# Move point to boundary of trust region
self.Bx.zero()
self.B_op.mult(x_bk, self.Bx)
x_Bnorm2 = self.Bx.inner(x_bk)
Bd = Vector()
self.B_op.init_vector(Bd,0)
Bd.zero()
self.B_op.mult(self.d,Bd)
d_Bnorm2 = Bd.inner(d)
d_Bx = Bd.inner(x_bk)
a_tau = alpha*alpha*d_Bnorm2
b_tau_half = alpha* d_Bx
c_tau = x_Bnorm2- self.TR_radius_2
# Solve quadratic for tau
tau = (-b_tau_half + math.sqrt(b_tau_half*b_tau_half - a_tau*c_tau))/a_tau
x.zero()
x.axpy(1,x_bk)
x.axpy(tau*alpha, d)
return True
def solve(self,x,b):
"""
Solve the linear system Ax = b
"""
self.iter = 0
self.converged = False
self.reasonid = 0
betanom = 0.0
alpha = 0.0
beta = 0.0
if self.parameters["zero_initial_guess"]:
self.r.zero()
self.r.axpy(1.0, b)
x.zero()
else:
assert self.TR_radius_2==None
self.A.mult(x,self.r)
self.r *= -1.0
self.r.axpy(1.0, b)
self.z.zero()
self.B_solver.solve(self.z,self.r) #z = B^-1 r
self.d.zero()
self.d.axpy(1.,self.z); #d = z
nom0 = self.d.inner(self.r)
nom = nom0
if self.parameters["print_level"] == 1:
print " Iterartion : ", 0, " (B r, r) = ", nom
rtol2 = nom * self.parameters["rel_tolerance"] * self.parameters["rel_tolerance"]
atol2 = self.parameters["abs_tolerance"] * self.parameters["abs_tolerance"]
r0 = max(rtol2, atol2)
if nom <= r0:
self.converged = True
self.reasonid = 1
self.final_norm = math.sqrt(nom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
return
self.A.mult(self.d, self.z) #z = A d
den = self.z.inner(self.d)
if den <= 0.0:
self.converged = True
self.reasonid = 2
x.axpy(1., self.d)
self.r.axpy(-1., self.z)
self.B_solver.solve(self.z, self.r)
nom = self.r.inner(self.z)
self.final_norm = math.sqrt(nom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
return
# start iteration
self.iter = 1
while True:
alpha = nom/den
TrustBool = self.update_x(x,alpha,self.d) # x = x + alpha d
if TrustBool == True:
self.converged = True
self.reasonid = 3
self.final_norm = math.sqrt(betanom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
break
self.r.axpy(-alpha, self.z) # r = r - alpha A d
self.B_solver.solve(self.z, self.r) # z = B^-1 r
betanom = self.r.inner(self.z)
if self.parameters["print_level"] == 1:
print " Iteration : ", self.iter, " (B r, r) = ", betanom
if betanom < r0:
self.converged = True
self.reasonid = 1
self.final_norm = math.sqrt(betanom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
break
self.iter += 1
if self.iter > self.parameters["max_iter"]:
self.converged = False
self.reasonid = 0
self.final_norm = math.sqrt(betanom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Not Converged. Final residual norm ", self.final_norm
break
beta = betanom/nom
self.d *= beta
self.d.axpy(1., self.z) #d = z + beta d
self.A.mult(self.d,self.z) # z = A d
den = self.d.inner(self.z)
if den <= 0.0:
self.converged = True
self.reasonid = 2
self.final_norm = math.sqrt(nom)
if(self.parameters["print_level"] >= 0):
print self.reason[self.reasonid]
print "Converged in ", self.iter, " iterations with final norm ", self.final_norm
break
nom = betanom
class CGSampler:
"""
This class implements the CG sampler algorithm to generate samples from N(0, A^-1).
REFERENCE:
Albert Parker and Colin Fox
Sampling Gaussian Distributions in Krylov Spaces with Conjugate Gradient
SIAM J SCI COMPUT, Vol 34, No. 3 pp. B312-B334
"""
def __init__(self):
"""
Construct the solver with default parameters
tolerance = 1e-4
print_level = 0
verbose = 0
"""
self.parameters = {}
self.parameters["tolerance"] = 1e-4
self.parameters["print_level"] = 0
self.parameters["verbose"] = 0
self.A = None
self.converged = False
self.iter = 0
self.b = Vector()
self.r = Vector()
self.p = Vector()
self.Ap = Vector()
def set_operator(self, A):
"""
Set the operator A, such that x ~ N(0, A^-1).
Note A is any object that provides the methods init_vector and mult.
"""
self.A = A
self.A.init_vector(self.r, 0)
self.A.init_vector(self.p, 0)
self.A.init_vector(self.Ap, 0)
self.A.init_vector(self.b, 0)
self.b.set_local(np.random.randn(self.b.array().shape[0]))
def sample(self, noise, s):
"""
Generate a sample s ~ N(0, A^-1).
noise is a numpy.array of i.i.d. normal variables used as input.
For a fixed realization of noise the algorithm is fully deterministic.
The size of noise determine the maximum number of CG iterations.
"""
s.zero()
self.iter = 0
self.converged = False
# r0 = b
self.r.zero()
self.r.axpy(1., self.b)
#p0 = r0
self.p.zero()
self.p.axpy(1., self.r)
self.A.mult(self.p, self.Ap)
d = self.p.inner(self.Ap)
tol2 = self.parameters["tolerance"] * self.parameters["tolerance"]
rnorm2_old = self.r.inner(self.r)
if self.parameters["verbose"] > 0:
print "initial residual = ", math.sqrt(rnorm2_old)
while (not self.converged) and (self.iter < noise.shape[0]):
gamma = rnorm2_old / d
s.axpy(noise[self.iter] / math.sqrt(d), self.p)
self.r.axpy(-gamma, self.Ap)
rnorm2 = self.r.inner(self.r)
beta = rnorm2 / rnorm2_old
# p_new = r + beta p
self.p *= beta
self.p.axpy(1., self.r)
self.A.mult(self.p, self.Ap)
d = self.p.inner(self.Ap)
rnorm2_old = rnorm2
if rnorm2 < tol2:
self.converged = True
else:
rnorm2_old = rnorm2
self.iter = self.iter + 1
if self.parameters["verbose"] > 0:
print "Final residual {0} after {1} iterations".format(
math.sqrt(rnorm2_old), self.iter)
class TraceEstimator:
"""
An unbiased stochastic estimator for the trace of :math:`A,\\, d = \\sum_{j=1}^k (v_j, A v_j)`, where
- :math:`v_j` are i.i.d. Rademacher or Gaussian random vectors.
- :math:`(\\cdot,\\cdot)` represents the inner product.
The number of samples :math:`k` is estimated at run time based on the variance of the estimator.
Reference: Haim Avron and Sivan Toledo, Randomized algorithms for estimating the trace of an implicit symmetric positive semi-definite matrix,
Journal of the ACM (JACM), 58 (2011), p. 17.
"""
def __init__(self, A, solve_mode=False, accurancy = 1e-1, init_vector=None, random_engine=rademacher_engine, mpi_comm=mpi_comm_world()):
"""
Constructor:
- :code:`A`: an operator
- :code:`solve_mode`: if :code:`True` we estimate the trace of :code:`A`:math:`^{-1}`, otherwise of :code:`A`.
- code:`accurancy`: we stop when the standard deviation of the estimator is less then
:code:`accurancy`*tr(:code:`A`).
- :code:`init_vector`: use a custom function to initialize a vector compatible with the
range/domain of :code:`A`.
- :code:`random_engine`: which type of i.i.d. random variables to use (Rademacher or Gaussian).
"""
if solve_mode:
self.A = Solver2Operator(A)
else:
self.A = A
self.accurancy = accurancy
self.random_engine = random_engine
self.iter = 0
self.z = Vector(mpi_comm)
self.Az = Vector(mpi_comm)
if init_vector is None:
A.init_vector(self.z, 0)
A.init_vector(self.Az, 0)
else:
init_vector(self.z, 0)
init_vector(self.Az, 0)
def __call__(self, min_iter=5, max_iter=100):
"""
Estimate the trace of :code:`A` (or :code:`A`:math:`^-1`) using at least
:code:`min_iter` and at most :code:`max_iter` samples.
"""
sum_tr = 0
sum_tr2 = 0
self.iter = 0
while self.iter < min_iter:
self.iter += 1
self.random_engine(self.z)
self.A.mult(self.z, self.Az)
tr = self.z.inner(self.Az)
sum_tr += tr
sum_tr2 += tr*tr
exp_tr = sum_tr / float(self.iter)
exp_tr2 = sum_tr2 / float(self.iter)
var_tr = exp_tr2 - exp_tr*exp_tr
# print( exp_tr, math.sqrt( var_tr ), self.accurancy*exp_tr)
self.converged = True
while (math.sqrt( var_tr ) > self.accurancy*exp_tr):
self.iter += 1
self.random_engine(self.z)
self.A.mult(self.z, self.Az)
tr = self.z.inner(self.Az)
sum_tr += tr
sum_tr2 += tr*tr
exp_tr = sum_tr / float(self.iter)
exp_tr2 = sum_tr2 / float(self.iter)
var_tr = exp_tr2 - exp_tr*exp_tr
# print( exp_tr, math.sqrt( var_tr ), self.accurancy*exp_tr)
if (self.iter > max_iter):
self.converged = False
break
return exp_tr, var_tr
