def update_x_with_TR(self,x,alpha,d): x_bk = x.copy() x.axpy(alpha,d) self.Bx.zero() self.B_op.mult(x, self.Bx) x_Bnorm2 = self.Bx.inner(x) if x_Bnorm2 < self.TR_radius_2: return False else: # Move point to boundary of trust region self.Bx.zero() self.B_op.mult(x_bk, self.Bx) x_Bnorm2 = self.Bx.inner(x_bk) Bd = Vector() self.B_op.init_vector(Bd,0) Bd.zero() self.B_op.mult(self.d,Bd) d_Bnorm2 = Bd.inner(d) d_Bx = Bd.inner(x_bk) a_tau = alpha*alpha*d_Bnorm2 b_tau_half = alpha* d_Bx c_tau = x_Bnorm2- self.TR_radius_2 # Solve quadratic for tau tau = (-b_tau_half + math.sqrt(b_tau_half*b_tau_half - a_tau*c_tau))/a_tau x.zero() x.axpy(1,x_bk) x.axpy(tau*alpha, d) return True
class Solver2Operator: def __init__(self,S,mpi_comm=mpi_comm_world(), init_vector = None): self.S = S self.tmp = Vector(mpi_comm) self.my_init_vector = init_vector if self.my_init_vector is None: if hasattr(self.S, "init_vector"): self.my_init_vector = self.S.init_vector elif hasattr(self.S, "operator"): self.my_init_vector = self.S.operator().init_vector elif hasattr(self.S, "get_operator"): self.my_init_vector = self.S.get_operator().init_vector def init_vector(self, x, dim): if self.my_init_vector: self.my_init_vector(x,dim) else: raise NotImplementedError("Solver2Operator.init_vector") def mult(self,x,y): self.S.solve(y,x) def inner(self, x, y): self.S.solve(self.tmp,y) return self.tmp.inner(x)
class Solver2Operator: def __init__(self, S, mpi_comm=mpi_comm_world(), init_vector=None): self.S = S self.tmp = Vector(mpi_comm) self.my_init_vector = init_vector if self.my_init_vector is None: if hasattr(self.S, "init_vector"): self.my_init_vector = self.S.init_vector elif hasattr(self.S, "operator"): self.my_init_vector = self.S.operator().init_vector elif hasattr(self.S, "get_operator"): self.my_init_vector = self.S.get_operator().init_vector def init_vector(self, x, dim): if self.my_init_vector: self.my_init_vector(x, dim) else: raise NotImplementedError("Solver2Operator.init_vector") def mult(self, x, y): self.S.solve(y, x) def inner(self, x, y): self.S.solve(self.tmp, y) return self.tmp.inner(x)
class Operator2Solver: def __init__(self,op, mpi_comm=mpi_comm_world()): self.op = op self.tmp = Vector(mpi_comm) def init_vector(self, x, dim): if hasattr(self.op, "init_vector"): self.op.init_vector(x,dim) else: raise def solve(self,y,x): self.op.mult(x,y) def inner(self, x, y): self.op.mult(y,self.tmp) return self.tmp.inner(x)
class Operator2Solver: def __init__(self, op, mpi_comm=mpi_comm_world()): self.op = op self.tmp = Vector(mpi_comm) def init_vector(self, x, dim): if hasattr(self.op, "init_vector"): self.op.init_vector(x, dim) else: raise def solve(self, y, x): self.op.mult(x, y) def inner(self, x, y): self.op.mult(y, self.tmp) return self.tmp.inner(x)
class Solver2Operator: def __init__(self, S): self.S = S self.tmp = Vector() def init_vector(self, x, dim): if hasattr(self.S, "init_vector"): self.S.init_vector(x, dim) elif hasattr(self.S, "operator"): self.S.operator().init_vector(x, dim) else: raise def mult(self, x, y): self.S.solve(y, x) def inner(self, x, y): self.S.solve(self.tmp, y) return self.tmp.inner(x)
def inner(self, x, y): Hx = Vector(self.help.mpi_comm()) self.init_vector(Hx, 0) self.mult(x, Hx) return Hx.inner(y)
class CGSolverSteihaug: """ Solve the linear system A x = b using preconditioned conjugate gradient ( B preconditioner) and the Steihaug stopping criterion: - reason of termination 0: we reached the maximum number of iterations (no convergence) - reason of termination 1: we reduced the residual up to the given tolerance (convergence) - reason of termination 2: we reached a negative direction (premature termination due to not spd matrix) The operator A is set using the method set_operator(A). A must provide the following two methods: - A.mult(x,y): y = A*x - A.init_vector(x, dim): initialize the vector x so that it is compatible with the range (dim = 0) or the domain (dim = 1) of A. The preconditioner B is set using the method set_preconditioner(B). B must provide the following method: - B.solve(z,r): z is the action of the preconditioner B on the vector r To solve the linear system A*x = b call solve(x,b). Here x and b are assumed to be FEniCS::Vector objects Maximum number of iterations, tolerances, verbosity level etc can be set using the parameters attributes. """ reason = [ "Maximum Number of Iterations Reached", "Relative/Absolute residual less than tol", "Reached a negative direction" ] def __init__(self): self.parameters = {} self.parameters["rel_tolerance"] = 1e-9 self.parameters["abs_tolerance"] = 1e-12 self.parameters["max_iter"] = 1000 self.parameters["zero_initial_guess"] = True self.parameters["print_level"] = 0 self.A = None self.B = None self.converged = False self.iter = 0 self.reasonid = 0 self.final_norm = 0 self.r = Vector() self.z = Vector() self.d = Vector() if dolfin.__version__[2] == '3': self.vrs130 = True else: self.vrs130 = False def set_operator(self, A): self.A = A if self.vrs130: self.r = self.A.init_vector130() self.z = self.A.init_vector130() self.d = self.A.init_vector130() else: self.A.init_vector(self.r, 0) self.A.init_vector(self.z, 0) self.A.init_vector(self.d, 0) def set_preconditioner(self, B): self.B = B def solve(self, x, b): self.iter = 0 self.converged = False self.reasonid = 0 betanom = 0.0 alpha = 0.0 beta = 0.0 if self.parameters["zero_initial_guess"]: self.r.zero() self.r.axpy(1.0, b) x.zero() else: self.A.mult(x, self.r) self.r *= -1.0 self.r.axpy(1.0, b) self.z.zero() self.B.solve(self.z, self.r) #z = B^-1 r self.d.zero() self.d.axpy(1., self.z) #d = z nom0 = self.d.inner(self.r) nom = nom0 if self.parameters["print_level"] == 1: print " Iterartion : ", 0, " (B r, r) = ", nom rtol2 = nom * self.parameters["rel_tolerance"] * self.parameters[ "rel_tolerance"] atol2 = self.parameters["abs_tolerance"] * self.parameters[ "abs_tolerance"] r0 = max(rtol2, atol2) if nom <= r0: self.converged = True self.reasonid = 1 self.final_norm = math.sqrt(nom) if (self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm return self.A.mult(self.d, self.z) #z = A d den = self.z.inner(self.d) if den <= 0.0: self.converged = True self.reasonid = 2 self.final_norm = math.sqrt(nom) if (self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm return # start iteration self.iter = 1 while True: alpha = nom / den x.axpy(alpha, self.d) # x = x + alpha d self.r.axpy(-alpha, self.z) # r = r - alpha A d self.B.solve(self.z, self.r) # z = B^-1 r betanom = self.r.inner(self.z) if self.parameters["print_level"] == 1: print " Iteration : ", self.iter, " (B r, r) = ", betanom if betanom < r0: self.converged = True self.reasonid = 1 self.final_norm = math.sqrt(betanom) if (self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm break self.iter += 1 if self.iter > self.parameters["max_iter"]: self.converged = False self.reasonid = 0 self.final_norm = math.sqrt(betanom) if (self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Not Converged. Final residual norm ", self.final_norm break beta = betanom / nom self.d *= beta self.d.axpy(1., self.z) #d = z + beta d self.A.mult(self.d, self.z) # z = A d den = self.d.inner(self.z) if den <= 0.0: self.converged = True self.reasonid = 2 self.final_norm = math.sqrt(nom) if (self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm break nom = betanom
def solve(self, F, u, grad = None, H = None): if grad is None: print "Using Automatic Differentiation to compute the gradient" grad = derivative(F,u) if H is None: print "Using Automatic Differentiation to compute the Hessian" H = derivative(grad, u) rtol = self.parameters["rel_tolerance"] atol = self.parameters["abs_tolerance"] gdu_tol = self.parameters["gdu_tolerance"] max_iter = self.parameters["max_iter"] c_armijo = self.parameters["c_armijo"] max_backtrack = self.parameters["max_backtracking_iter"] prt_level = self.parameters["print_level"] cg_coarsest_tol = self.parameters["cg_coarse_tolerance"] Fn = assemble(F) gn = assemble(grad) g0_norm = gn.norm("l2") gn_norm = g0_norm tol = max(g0_norm*rtol, atol) du = Vector() self.converged = False self.reason = 0 if prt_level > 0: print "{0:3} {1:15} {2:15} {3:15} {4:15} {5:15} {6:5}".format( "It", "Energy", "||g||", "(g,du)", "alpha", "tol_cg", "cg_it") for self.it in range(max_iter): Hn = assemble(H) Hn.init_vector(du,1) solver = PETScKrylovSolver("cg", "petsc_amg") solver.set_operator(Hn) solver.parameters["nonzero_initial_guess"] = False cg_tol = min(cg_coarsest_tol, math.sqrt( gn_norm/g0_norm) ) solver.parameters["relative_tolerance"] = cg_tol lin_it = solver.solve(du,gn) self.total_cg_iter += lin_it du_gn = -du.inner(gn) if(-du_gn < gdu_tol): self.converged=True self.reason = 3 break u_backtrack = u.copy(deepcopy=True) alpha = 1. bk_converged = False #Backtrack for j in range(max_backtrack): u.assign(u_backtrack) u.vector().axpy(-alpha, du) Fnext = assemble(F) if Fnext < Fn + alpha*c_armijo*du_gn: Fn = Fnext bk_converged = True break alpha = alpha/2. if not bk_converged: self.reason = 2 break gn = assemble(grad) gn_norm = gn.norm("l2") if prt_level > 0: print "{0:3d} {1:15f} {2:15f} {3:15f} {4:15f} {5:15f} {6:5d}".format( self.it, Fn, gn_norm, du_gn, alpha, cg_tol, lin_it) if gn_norm < tol: self.converged = True self.reason = 1 break self.final_grad_norm = gn_norm if prt_level > -1: print self.termination_reasons[self.reason] if self.converged: print "Inexact Newton CG converged in ", self.it, \ "nonlinear iterations and ", self.total_cg_iter, "linear iterations." else: print "Inexact Newton CG did NOT converge after ", self.it, \ "nonlinear iterations and ", self.total_cg_iter, "linear iterations." print "Final norm of the gradient", self.final_grad_norm print "Value of the cost functional", Fn
class CGSampler: """ This class implements the CG sampler algorithm to generate samples from :math:`\mathcal{N}(0, A^{-1})`. Reference: `Albert Parker and Colin Fox Sampling Gaussian Distributions in Krylov Spaces with Conjugate Gradient SIAM J SCI COMPUT, Vol 34, No. 3 pp. B312-B334` """ def __init__(self): """ Construct the solver with default parameters :code:`tolerance = 1e-4` :code:`print_level = 0` :code:`verbose = 0` """ self.parameters = {} self.parameters["tolerance"] = 1e-4 self.parameters["print_level"] = 0 self.parameters["verbose"] = 0 self.A = None self.converged = False self.iter = 0 self.b = Vector() self.r = Vector() self.p = Vector() self.Ap = Vector() def set_operator(self, A): """ Set the operator :code:`A`, such that :math:`x \sim \mathcal{N}(0, A^{-1})`. .. note:: :code:`A` is any object that provides the methods :code:`init_vector()` and :code:`mult()` """ self.A = A self.A.init_vector(self.r, 0) self.A.init_vector(self.p, 0) self.A.init_vector(self.Ap, 0) self.A.init_vector(self.b, 0) parRandom.normal(1., self.b) def sample(self, noise, s): """ Generate a sample :math:`s ~ N(0, A^{-1})`. :code:`noise` is a :code:`numpy.array` of i.i.d. normal variables used as input. For a fixed realization of noise the algorithm is fully deterministic. The size of noise determine the maximum number of CG iterations. """ s.zero() self.iter = 0 self.converged = False # r0 = b self.r.zero() self.r.axpy(1., self.b) #p0 = r0 self.p.zero() self.p.axpy(1., self.r) self.A.mult(self.p, self.Ap) d = self.p.inner(self.Ap) tol2 = self.parameters["tolerance"] * self.parameters["tolerance"] rnorm2_old = self.r.inner(self.r) if self.parameters["verbose"] > 0: print("initial residual = {0:g}".format(math.sqrt(rnorm2_old))) while (not self.converged) and (self.iter < noise.shape[0]): gamma = rnorm2_old / d s.axpy(noise[self.iter] / math.sqrt(d), self.p) self.r.axpy(-gamma, self.Ap) rnorm2 = self.r.inner(self.r) beta = rnorm2 / rnorm2_old # p_new = r + beta p self.p *= beta self.p.axpy(1., self.r) self.A.mult(self.p, self.Ap) d = self.p.inner(self.Ap) rnorm2_old = rnorm2 if rnorm2 < tol2: self.converged = True else: rnorm2_old = rnorm2 self.iter = self.iter + 1 if self.parameters["verbose"] > 0: print("Final residual {0} after {1} iterations".format( math.sqrt(rnorm2_old), self.iter))
class TraceEstimator: """ An unbiased stochastic estimator for the trace of A. d = \sum_{j=1}^k (vj, A vj) where - vj are i.i.d. Rademacher or Gaussian random vectors - (.,.) represents the inner product The number of samples k is estimated at run time based on the variance of the estimator. REFERENCE: Haim Avron and Sivan Toledo, Randomized algorithms for estimating the trace of an implicit symmetric positive semi-definite matrix, Journal of the ACM (JACM), 58 (2011), p. 17. """ def __init__(self, A, solve_mode=False, accurancy=1e-1, init_vector=None, random_engine=rademacher_engine): """ Constructor: - A: an operator - solve_mode: if True we estimate the trace of A^{-1}, otherwise of A. - accurancy: we stop when the standard deviation of the estimator is less then accurancy*tr(A). - init_vector: use a custom function to initialize a vector compatible with the range/domain of A - random_engine: which type of i.i.d. random variables to use (Rademacher or Gaussian) """ self.A = A self.accurancy = accurancy self.random_engine = random_engine self.iter = 0 self.z = Vector() self.Az = Vector() if solve_mode: self._apply = self._apply_solve else: self._apply = self._apply_mult if init_vector is None: A.init_vector(self.z, 0) A.init_vector(self.Az, 0) else: init_vector(self.z, 0) init_vector(self.Az, 0) def _apply_mult(self, z, Az): self.A.mult(z, Az) def _apply_solve(self, z, Az): self.A.solve(Az, z) def __call__(self, min_iter=5, max_iter=100): """ Estimate the trace of A (or A^-1) using at least min_iter and at most max_iter samples. """ sum_tr = 0 sum_tr2 = 0 self.iter = 0 size = len(self.z.array()) while self.iter < min_iter: self.iter += 1 self.z.set_local(self.random_engine(size)) self._apply(self.z, self.Az) tr = self.z.inner(self.Az) sum_tr += tr sum_tr2 += tr * tr exp_tr = sum_tr / float(self.iter) exp_tr2 = sum_tr2 / float(self.iter) var_tr = exp_tr2 - exp_tr * exp_tr # print(exp_tr, math.sqrt( var_tr ), self.accurancy*exp_tr) self.converged = True while (math.sqrt(var_tr) > self.accurancy * exp_tr): self.iter += 1 self.z.set_local(self.random_engine(size)) self._apply(self.z, self.Az) tr = self.z.inner(self.Az) sum_tr += tr sum_tr2 += tr * tr exp_tr = sum_tr / float(self.iter) exp_tr2 = sum_tr2 / float(self.iter) var_tr = exp_tr2 - exp_tr * exp_tr # print(exp_tr, math.sqrt( var_tr ), self.accurancy*exp_tr) if (self.iter > max_iter): self.converged = False break return exp_tr, var_tr
class CGSampler: """ This class implements the CG sampler algorithm to generate samples from :math:`\mathcal{N}(0, A^{-1})`. Reference: `Albert Parker and Colin Fox Sampling Gaussian Distributions in Krylov Spaces with Conjugate Gradient SIAM J SCI COMPUT, Vol 34, No. 3 pp. B312-B334` """ def __init__(self): """ Construct the solver with default parameters :code:`tolerance = 1e-4` :code:`print_level = 0` :code:`verbose = 0` """ self.parameters = {} self.parameters["tolerance"] = 1e-4 self.parameters["print_level"] = 0 self.parameters["verbose"] = 0 self.A = None self.converged = False self.iter = 0 self.b = Vector() self.r = Vector() self.p = Vector() self.Ap = Vector() def set_operator(self, A): """ Set the operator :code:`A`, such that :math:`x \sim \mathcal{N}(0, A^{-1})`. .. note:: :code:`A` is any object that provides the methods :code:`init_vector()` and :code:`mult()` """ self.A = A self.A.init_vector(self.r,0) self.A.init_vector(self.p,0) self.A.init_vector(self.Ap,0) self.A.init_vector(self.b,0) parRandom.normal(1., self.b) def sample(self, noise, s): """ Generate a sample :math:`s ~ N(0, A^{-1})`. :code:`noise` is a :code:`numpy.array` of i.i.d. normal variables used as input. For a fixed realization of noise the algorithm is fully deterministic. The size of noise determine the maximum number of CG iterations. """ s.zero() self.iter = 0 self.converged = False # r0 = b self.r.zero() self.r.axpy(1., self.b) #p0 = r0 self.p.zero() self.p.axpy(1., self.r) self.A.mult(self.p, self.Ap) d = self.p.inner(self.Ap) tol2 = self.parameters["tolerance"]*self.parameters["tolerance"] rnorm2_old = self.r.inner(self.r) if self.parameters["verbose"] > 0: print("initial residual = {0:g}".format( math.sqrt(rnorm2_old) )) while (not self.converged) and (self.iter < noise.shape[0]): gamma = rnorm2_old/d s.axpy(noise[self.iter]/math.sqrt(d), self.p) self.r.axpy(-gamma, self.Ap) rnorm2 = self.r.inner(self.r) beta = rnorm2/rnorm2_old # p_new = r + beta p self.p *= beta self.p.axpy(1., self.r) self.A.mult(self.p, self.Ap) d = self.p.inner(self.Ap) rnorm2_old = rnorm2 if rnorm2 < tol2: self.converged = True else: rnorm2_old = rnorm2 self.iter = self.iter+1 if self.parameters["verbose"] > 0: print("Final residual {0} after {1} iterations".format( math.sqrt(rnorm2_old), self.iter))
def solve(self, F, u, grad=None, H=None): if grad is None: print("Using Symbolic Differentiation to compute the gradient") grad = derivative(F, u) if H is None: print("Using Symbolic Differentiation to compute the Hessian") H = derivative(grad, u) rtol = self.parameters["rel_tolerance"] atol = self.parameters["abs_tolerance"] gdu_tol = self.parameters["gdu_tolerance"] max_iter = self.parameters["max_iter"] c_armijo = self.parameters["c_armijo"] max_backtrack = self.parameters["max_backtracking_iter"] prt_level = self.parameters["print_level"] cg_coarsest_tol = self.parameters["cg_coarse_tolerance"] Fn = assemble(F) gn = assemble(grad) g0_norm = gn.norm("l2") gn_norm = g0_norm tol = max(g0_norm * rtol, atol) du = Vector() self.converged = False self.reason = 0 if prt_level > 0: print( "{0:>3} {1:>15} {2:>15} {3:>15} {4:>15} {5:>15} {6:>7}".format( "It", "Energy", "||g||", "(g,du)", "alpha", "tol_cg", "cg_it")) for self.it in range(max_iter): Hn = assemble(H) Hn.init_vector(du, 1) solver = PETScKrylovSolver("cg", "petsc_amg") solver.set_operator(Hn) solver.parameters["nonzero_initial_guess"] = False cg_tol = min(cg_coarsest_tol, math.sqrt(gn_norm / g0_norm)) solver.parameters["relative_tolerance"] = cg_tol lin_it = solver.solve(du, gn) self.total_cg_iter += lin_it du_gn = -du.inner(gn) if (-du_gn < gdu_tol): self.converged = True self.reason = 3 break u_backtrack = u.copy(deepcopy=True) alpha = 1. bk_converged = False #Backtrack for j in range(max_backtrack): u.assign(u_backtrack) u.vector().axpy(-alpha, du) Fnext = assemble(F) if Fnext < Fn + alpha * c_armijo * du_gn: Fn = Fnext bk_converged = True break alpha = alpha / 2. if not bk_converged: self.reason = 2 break gn = assemble(grad) gn_norm = gn.norm("l2") if prt_level > 0: print("{0:3d} {1:15e} {2:15e} {3:15e} {4:15e} {5:15e} {6:7d}". format(self.it, Fn, gn_norm, du_gn, alpha, cg_tol, lin_it)) if gn_norm < tol: self.converged = True self.reason = 1 break self.final_grad_norm = gn_norm if prt_level > -1: print(self.termination_reasons[self.reason]) if self.converged: print( "Inexact Newton CG converged in ", self.it, \ "nonlinear iterations and ", self.total_cg_iter, "linear iterations." ) else: print( "Inexact Newton CG did NOT converge after ", self.it, \ "nonlinear iterations and ", self.total_cg_iter, "linear iterations.") print("Final norm of the gradient", self.final_grad_norm) print("Value of the cost functional", Fn)
class CGSolverSteihaug: """ Solve the linear system A x = b using preconditioned conjugate gradient ( B preconditioner) and the Steihaug stopping criterion: - reason of termination 0: we reached the maximum number of iterations (no convergence) - reason of termination 1: we reduced the residual up to the given tolerance (convergence) - reason of termination 2: we reached a negative direction (premature termination due to not spd matrix) The operator A is set using the method set_operator(A). A must provide the following two methods: - A.mult(x,y): y = A*x - A.init_vector(x, dim): initialize the vector x so that it is compatible with the range (dim = 0) or the domain (dim = 1) of A. The preconditioner B is set using the method set_preconditioner(B). B must provide the following method: - B.solve(z,r): z is the action of the preconditioner B on the vector r To solve the linear system A*x = b call solve(x,b). Here x and b are assumed to be FEniCS::Vector objects Maximum number of iterations, tolerances, verbosity level etc can be set using the parameters attributes. """ reason = ["Maximum Number of Iterations Reached", "Relative/Absolute residual less than tol", "Reached a negative direction" ] def __init__(self): self.parameters = {} self.parameters["rel_tolerance"] = 1e-9 self.parameters["abs_tolerance"] = 1e-12 self.parameters["max_iter"] = 1000 self.parameters["zero_initial_guess"] = True self.parameters["print_level"] = 0 self.A = None self.B = None self.converged = False self.iter = 0 self.reasonid = 0 self.final_norm = 0 self.r = Vector() self.z = Vector() self.d = Vector() if dolfin.__version__[2] == '5': self.vrs150 = True else: self.vrs150 = False def set_operator(self, A): self.A = A if self.vrs150: self.A.init_vector(self.r,0) self.A.init_vector(self.z,0) self.A.init_vector(self.d,0) else: self.r = self.A.init_vector130() self.z = self.A.init_vector130() self.d = self.A.init_vector130() def set_preconditioner(self, B): self.B = B def solve(self,x,b): self.iter = 0 self.converged = False self.reasonid = 0 betanom = 0.0 alpha = 0.0 beta = 0.0 if self.parameters["zero_initial_guess"]: self.r.zero() self.r.axpy(1.0, b) x.zero() else: self.A.mult(x,self.r) self.r *= -1.0 self.r.axpy(1.0, b) self.z.zero() self.B.solve(self.z,self.r) #z = B^-1 r self.d.zero() self.d.axpy(1.,self.z); #d = z nom0 = self.d.inner(self.r) nom = nom0 if self.parameters["print_level"] == 1: print " Iterartion : ", 0, " (B r, r) = ", nom rtol2 = nom * self.parameters["rel_tolerance"] * self.parameters["rel_tolerance"] atol2 = self.parameters["abs_tolerance"] * self.parameters["abs_tolerance"] r0 = max(rtol2, atol2) if nom <= r0: self.converged = True self.reasonid = 1 self.final_norm = math.sqrt(nom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm return self.A.mult(self.d, self.z) #z = A d den = self.z.inner(self.d) if den <= 0.0: self.converged = True self.reasonid = 2 self.final_norm = math.sqrt(nom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm return # start iteration self.iter = 1 while True: alpha = nom/den x.axpy(alpha,self.d) # x = x + alpha d self.r.axpy(-alpha, self.z) # r = r - alpha A d self.B.solve(self.z, self.r) # z = B^-1 r betanom = self.r.inner(self.z) if self.parameters["print_level"] == 1: print " Iteration : ", self.iter, " (B r, r) = ", betanom if betanom < r0: self.converged = True self.reasonid = 1 self.final_norm = math.sqrt(betanom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm break self.iter += 1 if self.iter > self.parameters["max_iter"]: self.converged = False self.reasonid = 0 self.final_norm = math.sqrt(betanom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Not Converged. Final residual norm ", self.final_norm break beta = betanom/nom self.d *= beta self.d.axpy(1., self.z) #d = z + beta d self.A.mult(self.d,self.z) # z = A d den = self.d.inner(self.z) if den <= 0.0: self.converged = True self.reasonid = 2 self.final_norm = math.sqrt(nom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm break nom = betanom
def inner(self,x,y): Hx = Vector(self.help.mpi_comm()) self.init_vector(Hx, 0) self.mult(x, Hx) return Hx.inner(y)
class CGSolverSteihaug: """ Solve the linear system A x = b using preconditioned conjugate gradient ( B preconditioner) and the Steihaug stopping criterion: - reason of termination 0: we reached the maximum number of iterations (no convergence) - reason of termination 1: we reduced the residual up to the given tolerance (convergence) - reason of termination 2: we reached a negative direction (premature termination due to not spd matrix) The stopping criterion is based on either - the absolute preconditioned residual norm check: || r^* ||_{B^{-1}} < atol - the relative preconditioned residual norm check: || r^* ||_{B^{-1}}/|| r^0 ||_{B^{-1}} < rtol where r^* = b - Ax^* is the residual at convergence and r^0 = b - Ax^0 is the initial residual. The operator A is set using the method set_operator(A). A must provide the following two methods: - A.mult(x,y): y = A*x - A.init_vector(x, dim): initialize the vector x so that it is compatible with the range (dim = 0) or the domain (dim = 1) of A. The preconditioner B is set using the method set_preconditioner(B). B must provide the following method: - B.solve(z,r): z is the action of the preconditioner B on the vector r To solve the linear system A*x = b call self.solve(x,b). Here x and b are assumed to be FEniCS::Vector objects. The parameter attributes allows to set: - rel_tolerance : the relative tolerance for the stopping criterion - abs_tolerance : the absolute tolerance for the stopping criterion - max_iter : the maximum number of iterations - zero_initial_guess: if True we start with a 0 initial guess if False we use the x as initial guess. - print_level : verbosity level: -1 --> no output on screen 0 --> only final residual at convergence or reason for not not convergence """ reason = [ "Maximum Number of Iterations Reached", "Relative/Absolute residual less than tol", "Reached a negative direction" ] def __init__(self): self.parameters = {} self.parameters["rel_tolerance"] = 1e-9 self.parameters["abs_tolerance"] = 1e-12 self.parameters["max_iter"] = 1000 self.parameters["zero_initial_guess"] = True self.parameters["print_level"] = 0 self.A = None self.B = None self.converged = False self.iter = 0 self.reasonid = 0 self.final_norm = 0 self.r = Vector() self.z = Vector() self.d = Vector() def set_operator(self, A): """ Set the operator A. """ self.A = A self.A.init_vector(self.r, 0) self.A.init_vector(self.z, 0) self.A.init_vector(self.d, 0) def set_preconditioner(self, B): """ Set the preconditioner B. """ self.B = B def solve(self, x, b): """ Solve the linear system Ax = b """ self.iter = 0 self.converged = False self.reasonid = 0 betanom = 0.0 alpha = 0.0 beta = 0.0 if self.parameters["zero_initial_guess"]: self.r.zero() self.r.axpy(1.0, b) x.zero() else: self.A.mult(x, self.r) self.r *= -1.0 self.r.axpy(1.0, b) self.z.zero() self.B.solve(self.z, self.r) #z = B^-1 r self.d.zero() self.d.axpy(1., self.z) #d = z nom0 = self.d.inner(self.r) nom = nom0 if self.parameters["print_level"] == 1: print(" Iterartion : ", 0, " (B r, r) = ", nom) rtol2 = nom * self.parameters["rel_tolerance"] * self.parameters[ "rel_tolerance"] atol2 = self.parameters["abs_tolerance"] * self.parameters[ "abs_tolerance"] r0 = max(rtol2, atol2) if nom <= r0: self.converged = True self.reasonid = 1 self.final_norm = math.sqrt(nom) if (self.parameters["print_level"] >= 0): print(self.reason[self.reasonid]) print("Converged in ", self.iter, " iterations with final norm ", self.final_norm) return self.A.mult(self.d, self.z) #z = A d den = self.z.inner(self.d) if den <= 0.0: self.converged = True self.reasonid = 2 self.final_norm = math.sqrt(nom) if (self.parameters["print_level"] >= 0): print(self.reason[self.reasonid]) print("Converged in ", self.iter, " iterations with final norm ", self.final_norm) return # start iteration self.iter = 1 while True: alpha = nom / den x.axpy(alpha, self.d) # x = x + alpha d self.r.axpy(-alpha, self.z) # r = r - alpha A d self.B.solve(self.z, self.r) # z = B^-1 r betanom = self.r.inner(self.z) if self.parameters["print_level"] == 1: print(" Iteration : ", self.iter, " (B r, r) = ", betanom) if betanom < r0: self.converged = True self.reasonid = 1 self.final_norm = math.sqrt(betanom) if (self.parameters["print_level"] >= 0): print(self.reason[self.reasonid]) print("Converged in ", self.iter, " iterations with final norm ", self.final_norm) break self.iter += 1 if self.iter > self.parameters["max_iter"]: self.converged = False self.reasonid = 0 self.final_norm = math.sqrt(betanom) if (self.parameters["print_level"] >= 0): print(self.reason[self.reasonid]) print("Not Converged. Final residual norm ", self.final_norm) break beta = betanom / nom self.d *= beta self.d.axpy(1., self.z) #d = z + beta d self.A.mult(self.d, self.z) # z = A d den = self.d.inner(self.z) if den <= 0.0: self.converged = True self.reasonid = 2 self.final_norm = math.sqrt(nom) if (self.parameters["print_level"] >= 0): print(self.reason[self.reasonid]) print("Converged in ", self.iter, " iterations with final norm ", self.final_norm) break nom = betanom
def inner(self, x, y): Hx = Vector() self.init_vector(Hx, 0) self.mult(x, Hx) return Hx.inner(y)
class CGSolverSteihaug: """ Solve the linear system A x = b using preconditioned conjugate gradient ( B preconditioner) and the Steihaug stopping criterion: - reason of termination 0: we reached the maximum number of iterations (no convergence) - reason of termination 1: we reduced the residual up to the given tolerance (convergence) - reason of termination 2: we reached a negative direction (premature termination due to not spd matrix) - reason of termination 3: we reached the boundary of the trust region The stopping criterion is based on either - the absolute preconditioned residual norm check: || r^* ||_{B^{-1}} < atol - the relative preconditioned residual norm check: || r^* ||_{B^{-1}}/|| r^0 ||_{B^{-1}} < rtol where r^* = b - Ax^* is the residual at convergence and r^0 = b - Ax^0 is the initial residual. The operator A is set using the method set_operator(A). A must provide the following two methods: - A.mult(x,y): y = A*x - A.init_vector(x, dim): initialize the vector x so that it is compatible with the range (dim = 0) or the domain (dim = 1) of A. The preconditioner B is set using the method set_preconditioner(B). B must provide the following method: - B.solve(z,r): z is the action of the preconditioner B on the vector r To solve the linear system A*x = b call self.solve(x,b). Here x and b are assumed to be FEniCS::Vector objects. Type: CGSolverSteihaug_ParameterList().showMe() for default parameters and their descriptions """ reason = ["Maximum Number of Iterations Reached", "Relative/Absolute residual less than tol", "Reached a negative direction", "Reached trust region boundary" ] def __init__(self, parameters=CGSolverSteihaug_ParameterList()): self.parameters = parameters self.A = None self.B_solver = None self.B_op = None self.converged = False self.iter = 0 self.reasonid = 0 self.final_norm = 0 self.TR_radius_2 = None self.update_x = self.update_x_without_TR self.r = Vector() self.z = Vector() self.d = Vector() self.Bx = Vector() def set_operator(self, A): """ Set the operator A. """ self.A = A self.A.init_vector(self.r,0) self.A.init_vector(self.z,0) self.A.init_vector(self.d,0) def set_preconditioner(self, B_solver): """ Set the preconditioner B. """ self.B_solver = B_solver def set_TR(self,radius,B_op): assert self.parameters["zero_initial_guess"] self.TR_radius_2 = radius*radius self.update_x = self.update_x_with_TR self.B_op = B_op self.B_op.init_vector(self.Bx,0) def update_x_without_TR(self,x,alpha,d): x.axpy(alpha,d) return False def update_x_with_TR(self,x,alpha,d): x_bk = x.copy() x.axpy(alpha,d) self.Bx.zero() self.B_op.mult(x, self.Bx) x_Bnorm2 = self.Bx.inner(x) if x_Bnorm2 < self.TR_radius_2: return False else: # Move point to boundary of trust region self.Bx.zero() self.B_op.mult(x_bk, self.Bx) x_Bnorm2 = self.Bx.inner(x_bk) Bd = Vector() self.B_op.init_vector(Bd,0) Bd.zero() self.B_op.mult(self.d,Bd) d_Bnorm2 = Bd.inner(d) d_Bx = Bd.inner(x_bk) a_tau = alpha*alpha*d_Bnorm2 b_tau_half = alpha* d_Bx c_tau = x_Bnorm2- self.TR_radius_2 # Solve quadratic for tau tau = (-b_tau_half + math.sqrt(b_tau_half*b_tau_half - a_tau*c_tau))/a_tau x.zero() x.axpy(1,x_bk) x.axpy(tau*alpha, d) return True def solve(self,x,b): """ Solve the linear system Ax = b """ self.iter = 0 self.converged = False self.reasonid = 0 betanom = 0.0 alpha = 0.0 beta = 0.0 if self.parameters["zero_initial_guess"]: self.r.zero() self.r.axpy(1.0, b) x.zero() else: assert self.TR_radius_2==None self.A.mult(x,self.r) self.r *= -1.0 self.r.axpy(1.0, b) self.z.zero() self.B_solver.solve(self.z,self.r) #z = B^-1 r self.d.zero() self.d.axpy(1.,self.z); #d = z nom0 = self.d.inner(self.r) nom = nom0 if self.parameters["print_level"] == 1: print " Iterartion : ", 0, " (B r, r) = ", nom rtol2 = nom * self.parameters["rel_tolerance"] * self.parameters["rel_tolerance"] atol2 = self.parameters["abs_tolerance"] * self.parameters["abs_tolerance"] r0 = max(rtol2, atol2) if nom <= r0: self.converged = True self.reasonid = 1 self.final_norm = math.sqrt(nom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm return self.A.mult(self.d, self.z) #z = A d den = self.z.inner(self.d) if den <= 0.0: self.converged = True self.reasonid = 2 x.axpy(1., self.d) self.r.axpy(-1., self.z) self.B_solver.solve(self.z, self.r) nom = self.r.inner(self.z) self.final_norm = math.sqrt(nom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm return # start iteration self.iter = 1 while True: alpha = nom/den TrustBool = self.update_x(x,alpha,self.d) # x = x + alpha d if TrustBool == True: self.converged = True self.reasonid = 3 self.final_norm = math.sqrt(betanom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm break self.r.axpy(-alpha, self.z) # r = r - alpha A d self.B_solver.solve(self.z, self.r) # z = B^-1 r betanom = self.r.inner(self.z) if self.parameters["print_level"] == 1: print " Iteration : ", self.iter, " (B r, r) = ", betanom if betanom < r0: self.converged = True self.reasonid = 1 self.final_norm = math.sqrt(betanom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm break self.iter += 1 if self.iter > self.parameters["max_iter"]: self.converged = False self.reasonid = 0 self.final_norm = math.sqrt(betanom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Not Converged. Final residual norm ", self.final_norm break beta = betanom/nom self.d *= beta self.d.axpy(1., self.z) #d = z + beta d self.A.mult(self.d,self.z) # z = A d den = self.d.inner(self.z) if den <= 0.0: self.converged = True self.reasonid = 2 self.final_norm = math.sqrt(nom) if(self.parameters["print_level"] >= 0): print self.reason[self.reasonid] print "Converged in ", self.iter, " iterations with final norm ", self.final_norm break nom = betanom
class CGSampler: """ This class implements the CG sampler algorithm to generate samples from N(0, A^-1). REFERENCE: Albert Parker and Colin Fox Sampling Gaussian Distributions in Krylov Spaces with Conjugate Gradient SIAM J SCI COMPUT, Vol 34, No. 3 pp. B312-B334 """ def __init__(self): """ Construct the solver with default parameters tolerance = 1e-4 print_level = 0 verbose = 0 """ self.parameters = {} self.parameters["tolerance"] = 1e-4 self.parameters["print_level"] = 0 self.parameters["verbose"] = 0 self.A = None self.converged = False self.iter = 0 self.b = Vector() self.r = Vector() self.p = Vector() self.Ap = Vector() def set_operator(self, A): """ Set the operator A, such that x ~ N(0, A^-1). Note A is any object that provides the methods init_vector and mult. """ self.A = A self.A.init_vector(self.r, 0) self.A.init_vector(self.p, 0) self.A.init_vector(self.Ap, 0) self.A.init_vector(self.b, 0) self.b.set_local(np.random.randn(self.b.array().shape[0])) def sample(self, noise, s): """ Generate a sample s ~ N(0, A^-1). noise is a numpy.array of i.i.d. normal variables used as input. For a fixed realization of noise the algorithm is fully deterministic. The size of noise determine the maximum number of CG iterations. """ s.zero() self.iter = 0 self.converged = False # r0 = b self.r.zero() self.r.axpy(1., self.b) #p0 = r0 self.p.zero() self.p.axpy(1., self.r) self.A.mult(self.p, self.Ap) d = self.p.inner(self.Ap) tol2 = self.parameters["tolerance"] * self.parameters["tolerance"] rnorm2_old = self.r.inner(self.r) if self.parameters["verbose"] > 0: print "initial residual = ", math.sqrt(rnorm2_old) while (not self.converged) and (self.iter < noise.shape[0]): gamma = rnorm2_old / d s.axpy(noise[self.iter] / math.sqrt(d), self.p) self.r.axpy(-gamma, self.Ap) rnorm2 = self.r.inner(self.r) beta = rnorm2 / rnorm2_old # p_new = r + beta p self.p *= beta self.p.axpy(1., self.r) self.A.mult(self.p, self.Ap) d = self.p.inner(self.Ap) rnorm2_old = rnorm2 if rnorm2 < tol2: self.converged = True else: rnorm2_old = rnorm2 self.iter = self.iter + 1 if self.parameters["verbose"] > 0: print "Final residual {0} after {1} iterations".format( math.sqrt(rnorm2_old), self.iter)
class TraceEstimator: """ An unbiased stochastic estimator for the trace of :math:`A,\\, d = \\sum_{j=1}^k (v_j, A v_j)`, where - :math:`v_j` are i.i.d. Rademacher or Gaussian random vectors. - :math:`(\\cdot,\\cdot)` represents the inner product. The number of samples :math:`k` is estimated at run time based on the variance of the estimator. Reference: Haim Avron and Sivan Toledo, Randomized algorithms for estimating the trace of an implicit symmetric positive semi-definite matrix, Journal of the ACM (JACM), 58 (2011), p. 17. """ def __init__(self, A, solve_mode=False, accurancy = 1e-1, init_vector=None, random_engine=rademacher_engine, mpi_comm=mpi_comm_world()): """ Constructor: - :code:`A`: an operator - :code:`solve_mode`: if :code:`True` we estimate the trace of :code:`A`:math:`^{-1}`, otherwise of :code:`A`. - code:`accurancy`: we stop when the standard deviation of the estimator is less then :code:`accurancy`*tr(:code:`A`). - :code:`init_vector`: use a custom function to initialize a vector compatible with the range/domain of :code:`A`. - :code:`random_engine`: which type of i.i.d. random variables to use (Rademacher or Gaussian). """ if solve_mode: self.A = Solver2Operator(A) else: self.A = A self.accurancy = accurancy self.random_engine = random_engine self.iter = 0 self.z = Vector(mpi_comm) self.Az = Vector(mpi_comm) if init_vector is None: A.init_vector(self.z, 0) A.init_vector(self.Az, 0) else: init_vector(self.z, 0) init_vector(self.Az, 0) def __call__(self, min_iter=5, max_iter=100): """ Estimate the trace of :code:`A` (or :code:`A`:math:`^-1`) using at least :code:`min_iter` and at most :code:`max_iter` samples. """ sum_tr = 0 sum_tr2 = 0 self.iter = 0 while self.iter < min_iter: self.iter += 1 self.random_engine(self.z) self.A.mult(self.z, self.Az) tr = self.z.inner(self.Az) sum_tr += tr sum_tr2 += tr*tr exp_tr = sum_tr / float(self.iter) exp_tr2 = sum_tr2 / float(self.iter) var_tr = exp_tr2 - exp_tr*exp_tr # print( exp_tr, math.sqrt( var_tr ), self.accurancy*exp_tr) self.converged = True while (math.sqrt( var_tr ) > self.accurancy*exp_tr): self.iter += 1 self.random_engine(self.z) self.A.mult(self.z, self.Az) tr = self.z.inner(self.Az) sum_tr += tr sum_tr2 += tr*tr exp_tr = sum_tr / float(self.iter) exp_tr2 = sum_tr2 / float(self.iter) var_tr = exp_tr2 - exp_tr*exp_tr # print( exp_tr, math.sqrt( var_tr ), self.accurancy*exp_tr) if (self.iter > max_iter): self.converged = False break return exp_tr, var_tr