def schur(*names, **kwargs): # shucr("A1", "B1", scale=(1.0, 1e3)) prc = kwargs.pop('prc') blockname = kwargs.pop('blockname') r0 = prc.get_row_by_name(blockname) c0 = prc.get_col_by_name(blockname) A0 = prc.get_operator_block(r0, c0) scales = kwargs.pop('scale', [1] * len(names)) print_level = kwargs.pop('print_level', -1) for name, scale in zip(names, scales): r1 = prc.get_row_by_name(name) c1 = prc.get_col_by_name(name) B = prc.get_operator_block(r0, c1) Bt = prc.get_operator_block(r1, c0) Bt = Bt.Copy() B0 = get_block(A, r1, c1) if use_parallel: Md = mfem.HyprePaarVector(MPI.COMM_WORLD, B0.GetGlobalNumRows(), B0.GetColStarts()) else: Md = mfem.Vector() A0.GetDiag(Md) Md *= scale if use_parallel: Bt.InvScaleRows(Md) S = mfem.ParMult(B, Bt) invA0 = mfem.HypreBoomerAMG(S) else: S = mfem.Mult(B, Bt) invA0 = mfem.DSmoother(S) invA0.iterative_mode = False invA0.SetPrintLevel(print_level) return invA0
def __init__(self, spaces, mass, offsets): self.pressure_mass = mass self.block_offsets = offsets self.spaces = spaces super(JacobianPreconditioner, self).__init__(offsets[2]) self.gamma = 0.00001 # The mass matrix and preconditioner do not change every Newton cycle, so we # only need to define them once self.mass_prec = mfem.HypreBoomerAMG() self.mass_prec.SetPrintLevel(0) mass_pcg = mfem.CGSolver(MPI.COMM_WORLD) mass_pcg.SetRelTol(1e-12) mass_pcg.SetAbsTol(1e-12) mass_pcg.SetMaxIter(200) mass_pcg.SetPrintLevel(0) mass_pcg.SetPreconditioner(self.mass_prec) mass_pcg.SetOperator(self.pressure_mass) mass_pcg.iterative_mode = False self.mass_pcg = mass_pcg # The stiffness matrix does change every Newton cycle, so we will define it # during SetOperator self.stiff_pcg = None self.stiff_prec = None
def schur(*names, **kwargs): # schur("A1", "B1", scale=(1.0, 1e3)) prc = kwargs.pop('prc') blockname = kwargs.pop('blockname') r0 = prc.get_row_by_name(blockname) c0 = prc.get_col_by_name(blockname) scales = kwargs.pop('scale', [1] * len(names)) print_level = kwargs.pop('print_level', -1) S = [] for name, scale in zip(names, scales): r1 = prc.get_row_by_name(name) c1 = prc.get_col_by_name(name) B = prc.get_operator_block(r0, c1) Bt = prc.get_operator_block(r1, c0) B0 = prc.get_operator_block(r1, c1) if use_parallel: Bt = Bt.Transpose() Bt = Bt.Transpose() Md = mfem.HypreParVector(MPI.COMM_WORLD, B0.GetGlobalNumRows(), B0.GetColStarts()) else: Bt = Bt.Copy() Md = mfem.Vector() B0.GetDiag(Md) Md *= scale if use_parallel: Bt.InvScaleRows(Md) S.append(mfem.ParMult(B, Bt)) else: S.append(mfem.Mult(B, Bt)) if use_parallel: from mfem.common.parcsr_extra import ToHypreParCSR, ToScipyCoo S2 = [ToScipyCoo(s) for s in S] for s in S2[1:]: S2[0] = S2[0] + s S = ToHypreParCSR(S2[0].tocsr()) invA0 = mfem.HypreBoomerAMG(S) else: from mfem.common.sparse_utils import sparsemat_to_scipycsr S2 = [sparsemat_to_scipycsr(s).tocoo() for s in S] for s in S2[1:]: S2[0] = S2[0] + s S = mfem.SparseMatrix(S2.tocsr()) invA0 = mfem.DSmoother(S) invA0.iterative_mode = False invA0.SetPrintLevel(print_level) invA0._S = S return invA0
def boomerAMG(**kwargs): prc = kwargs.pop('prc') blockname = kwargs.pop('blockname') print_level = kwargs.pop('print_level', -1) row = prc.get_row_by_name(blockname) col = prc.get_col_by_name(blockname) mat = prc.get_operator_block(row, col) inv_boomeramg = mfem.HypreBoomerAMG(mat) inv_boomeramg.SetPrintLevel(print_level) inv_boomeramg.iterative_mode = False return inv_boomeramg
def SetOperator(self, op): self.jacobian = mfem.Opr2BlockOpr(op) if (self.stiff_prec == None): # Initialize the stiffness preconditioner and solver stiff_prec_amg = mfem.HypreBoomerAMG() stiff_prec_amg.SetPrintLevel(0) stiff_prec_amg.SetElasticityOptions(self.spaces[0]) self.stiff_prec = stiff_prec_amg stiff_pcg_iter = mfem.GMRESSolver(MPI.COMM_WORLD) stiff_pcg_iter.SetRelTol(1e-8) stiff_pcg_iter.SetAbsTol(1e-8) stiff_pcg_iter.SetMaxIter(200) stiff_pcg_iter.SetPrintLevel(0) stiff_pcg_iter.SetPreconditioner(self.stiff_prec) stiff_pcg_iter.iterative_mode = False self.stiff_pcg = stiff_pcg_iter # At each Newton cycle, compute the new stiffness preconditioner by updating # the iterative solver which, in turn, updates its preconditioner self.stiff_pcg.SetOperator(self.jacobian.GetBlock(0, 0))
a.Assemble() b.Assemble() # 16. Project the exact solution to the essential DOFs. x.ProjectBdrCoefficient(bdr, ess_bdr) # 17. Create and solve the parallel linear system. ess_tdof_list = mfem.intArray() fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list) A = mfem.HypreParMatrix() B = mfem.Vector() X = mfem.Vector() a.FormLinearSystem(ess_tdof_list, x, b, A, X, B) amg = mfem.HypreBoomerAMG(A) amg.SetPrintLevel(0) pcg = mfem.HyprePCG(A) pcg.SetTol(1e-12) pcg.SetMaxIter(200) pcg.SetPrintLevel(0) pcg.SetPreconditioner(amg) pcg.Mult(B, X) # 18. Extract the local solution on each processor. a.RecoverFEMSolution(X, b, x) # 19. Send the solution by socket to a GLVis server if visualization: sout.send_text("parallel " + str(num_procs) + " " + str(myid)) sout.precision(8)
SinvF = mfem.HypreParVector(test_space) matSinv.Mult(trueF, SinvF) B.MultTranspose(SinvF, b) # 11. Set up a block-diagonal preconditioner for the 2x2 normal equation # # [ S0^{-1} 0 ] # [ 0 Shat^{-1} ] Shat = (Bhat^T Sinv Bhat) # # corresponding to the primal (x0) and interfacial (xhat) unknowns. # Since the Shat operator is equivalent to an H(div) matrix reduced to # the interfacial skeleton, we approximate its inverse with one V-cycle # of the ADS preconditioner from the hypre library (in 2D we use AMS for # the rotated H(curl) problem). S0inv = mfem.HypreBoomerAMG(matS0) S0inv.SetPrintLevel(0) Shat = mfem.RAP(matSinv, matBhat) if (dim == 2): Shatinv = mfem.HypreAMS(Shat, xhat_space) else: Shatinv = mfem.HypreADS(Shat, xhat_space) P = mfem.BlockDiagonalPreconditioner(true_offsets) P.SetDiagonalBlock(0, S0inv) P.SetDiagonalBlock(1, Shatinv) # 12. Solve the normal equation system using the PCG iterative solver. # Check the weighted norm of residual for the DPG least square problem. # Wrap the primal variable in a GridFunction for visualization purposes. pcg = mfem.CGSolver(MPI.COMM_WORLD) pcg.SetOperator(A)
darcyOp = mfem.BlockOperator(block_trueOffsets) darcyOp.SetBlock(0, 0, M) darcyOp.SetBlock(0, 1, BT) darcyOp.SetBlock(1, 0, B) #M2 = M.Transpose() #M3 = M2.Transpose() MinvBt = B.Transpose() Md = mfem.HypreParVector(MPI.COMM_WORLD, M.GetGlobalNumRows(), M.GetRowStarts()) M.GetDiag(Md) MinvBt.InvScaleRows(Md) S = mfem.hypre.ParMult(B, MinvBt) invM = mfem.HypreDiagScale(M) invS = mfem.HypreBoomerAMG(S) invM.iterative_mode = False invS.iterative_mode = False darcyPr = mfem.BlockDiagonalPreconditioner(block_trueOffsets) darcyPr.SetDiagonalBlock(0, invM) darcyPr.SetDiagonalBlock(1, invS) maxIter = 500 rtol = 1e-6 atol = 1e-10 import time stime = time.clock() solver = mfem.MINRESSolver(MPI.COMM_WORLD)
def run(order = 1, static_cond = False, meshfile = def_meshfile, visualization = False, use_strumpack = False): mesh = mfem.Mesh(meshfile, 1,1) dim = mesh.Dimension() ref_levels = int(np.floor(np.log(10000./mesh.GetNE())/np.log(2.)/dim)) for x in range(ref_levels): mesh.UniformRefinement(); mesh.ReorientTetMesh(); pmesh = mfem.ParMesh(MPI.COMM_WORLD, mesh) del mesh par_ref_levels = 2 for l in range(par_ref_levels): pmesh.UniformRefinement(); if order > 0: fec = mfem.H1_FECollection(order, dim) elif mesh.GetNodes(): fec = mesh.GetNodes().OwnFEC() print( "Using isoparametric FEs: " + str(fec.Name())); else: order = 1 fec = mfem.H1_FECollection(order, dim) fespace =mfem.ParFiniteElementSpace(pmesh, fec) fe_size = fespace.GlobalTrueVSize() if (myid == 0): print('Number of finite element unknowns: '+ str(fe_size)) ess_tdof_list = mfem.intArray() if pmesh.bdr_attributes.Size()>0: ess_bdr = mfem.intArray(pmesh.bdr_attributes.Max()) ess_bdr.Assign(1) fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list) # the basis functions in the finite element fespace. b = mfem.ParLinearForm(fespace) one = mfem.ConstantCoefficient(1.0) b.AddDomainIntegrator(mfem.DomainLFIntegrator(one)) b.Assemble(); x = mfem.ParGridFunction(fespace); x.Assign(0.0) a = mfem.ParBilinearForm(fespace); a.AddDomainIntegrator(mfem.DiffusionIntegrator(one)) if static_cond: a.EnableStaticCondensation() a.Assemble(); A = mfem.HypreParMatrix() B = mfem.Vector() X = mfem.Vector() a.FormLinearSystem(ess_tdof_list, x, b, A, X, B) if (myid == 0): print("Size of linear system: " + str(x.Size())) print("Size of linear system: " + str(A.GetGlobalNumRows())) if use_strumpack: import mfem.par.strumpack as strmpk Arow = strmpk.STRUMPACKRowLocMatrix(A) args = ["--sp_hss_min_sep_size", "128", "--sp_enable_hss"] strumpack = strmpk.STRUMPACKSolver(args, MPI.COMM_WORLD) strumpack.SetPrintFactorStatistics(True) strumpack.SetPrintSolveStatistics(False) strumpack.SetKrylovSolver(strmpk.KrylovSolver_DIRECT); strumpack.SetReorderingStrategy(strmpk.ReorderingStrategy_METIS) strumpack.SetMC64Job(strmpk.MC64Job_NONE) # strumpack.SetSymmetricPattern(True) strumpack.SetOperator(Arow) strumpack.SetFromCommandLine() strumpack.Mult(B, X); else: amg = mfem.HypreBoomerAMG(A) cg = mfem.CGSolver(MPI.COMM_WORLD) cg.SetRelTol(1e-12) cg.SetMaxIter(200) cg.SetPrintLevel(1) cg.SetPreconditioner(amg) cg.SetOperator(A) cg.Mult(B, X); a.RecoverFEMSolution(X, b, x) smyid = '{:0>6d}'.format(myid) mesh_name = "mesh."+smyid sol_name = "sol."+smyid pmesh.Print(mesh_name, 8) x.Save(sol_name, 8)
def run(order = 1, static_cond = False, meshfile = def_meshfile, visualization = False): mesh = mfem.Mesh(meshfile, 1,1) dim = mesh.Dimension() ref_levels = int(np.floor(np.log(10000./mesh.GetNE())/np.log(2.)/dim)) for x in range(ref_levels): mesh.UniformRefinement(); mesh.ReorientTetMesh(); pmesh = mfem.ParMesh(MPI.COMM_WORLD, mesh) del mesh par_ref_levels = 2 for l in range(par_ref_levels): pmesh.UniformRefinement(); if order > 0: fec = mfem.H1_FECollection(order, dim) elif mesh.GetNodes(): fec = mesh.GetNodes().OwnFEC() prinr( "Using isoparametric FEs: " + str(fec.Name())); else: order = 1 fec = mfem.H1_FECollection(order, dim) fespace =mfem.ParFiniteElementSpace(pmesh, fec) fe_size = fespace.GlobalTrueVSize() if (myid == 0): print('Number of finite element unknowns: '+ str(fe_size)) ess_tdof_list = mfem.intArray() if pmesh.bdr_attributes.Size()>0: ess_bdr = mfem.intArray(pmesh.bdr_attributes.Max()) ess_bdr.Assign(1) fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list) # the basis functions in the finite element fespace. b = mfem.ParLinearForm(fespace) one = mfem.ConstantCoefficient(1.0) b.AddDomainIntegrator(mfem.DomainLFIntegrator(one)) b.Assemble(); x = mfem.ParGridFunction(fespace); x.Assign(0.0) a = mfem.ParBilinearForm(fespace); a.AddDomainIntegrator(mfem.DiffusionIntegrator(one)) if static_cond: a.EnableStaticCondensation() a.Assemble(); A = mfem.HypreParMatrix() B = mfem.Vector() X = mfem.Vector() a.FormLinearSystem(ess_tdof_list, x, b, A, X, B) if (myid == 0): print("Size of linear system: " + str(x.Size())) print("Size of linear system: " + str(A.GetGlobalNumRows())) amg = mfem.HypreBoomerAMG(A) pcg = mfem.HyprePCG(A) pcg.SetTol(1e-12) pcg.SetMaxIter(200) pcg.SetPrintLevel(2) pcg.SetPreconditioner(amg) pcg.Mult(B, X); a.RecoverFEMSolution(X, b, x) smyid = '{:0>6d}'.format(myid) mesh_name = "mesh."+smyid sol_name = "sol."+smyid pmesh.PrintToFile(mesh_name, 8) x.SaveToFile(sol_name, 8)
def solve_parallel(self, A, b, x=None): from mpi4py import MPI myid = MPI.COMM_WORLD.rank nproc = MPI.COMM_WORLD.size from petram.helper.mpi_recipes import gather_vector def get_block(Op, i, j): try: return Op._linked_op[(i, j)] except KeyError: return None offset = A.RowOffsets() rows = A.NumRowBlocks() cols = A.NumColBlocks() if self.gui.write_mat: for i in range(cols): for j in range(rows): m = get_block(A, i, j) if m is None: continue m.Print('matrix_' + str(i) + '_' + str(j)) for i, bb in enumerate(b): for j in range(rows): v = bb.GetBlock(j) v.Print('rhs_' + str(i) + '_' + str(j) + '.' + smyid) if x is not None: for j in range(rows): xx = x.GetBlock(j) xx.Print('x_' + str(i) + '_' + str(j) + '.' + smyid) M = mfem.BlockDiagonalPreconditioner(offset) prcs = dict(self.gui.preconditioners) name = self.Aname assert not self.gui.parent.is_complex(), "can not solve complex" if self.gui.parent.is_converted_from_complex(): name = sum([[n, n] for n in name], []) for k, n in enumerate(name): prc = prcs[n][1] if prc == "None": continue name = "".join([tmp for tmp in prc if not tmp.isdigit()]) A0 = get_block(A, k, k) if A0 is None and not name.startswith('schur'): continue if hasattr(mfem.HypreSmoother, prc): invA0 = mfem.HypreSmoother(A0) invA0.SetType(getattr(mfem.HypreSmoother, prc)) elif prc == 'ams': depvar = self.engine.r_dep_vars[k] dprint1("setting up AMS for ", depvar) prec_fespace = self.engine.fespaces[depvar] invA0 = mfem.HypreAMS(A0, prec_fespace) invA0.SetSingularProblem() elif name == 'MUMPS': cls = SparseSmootherCls[name][0] invA0 = cls(A0, gui=self.gui[prc], engine=self.engine) elif name.startswith('schur'): args = name.split("(")[-1].split(")")[0].split(",") dprint1("setting up schur for ", args) if len(args) > 1: assert False, "not yet supported" for arg in args: r1 = self.engine.dep_var_offset(arg.strip()) c1 = self.engine.r_dep_var_offset(arg.strip()) B = get_block(A, k, c1) Bt = get_block(A, r1, k).Transpose() Bt = Bt.Transpose() B0 = get_block(A, r1, c1) Md = mfem.HypreParVector(MPI.COMM_WORLD, B0.GetGlobalNumRows(), B0.GetColStarts()) B0.GetDiag(Md) Bt.InvScaleRows(Md) S = mfem.ParMult(B, Bt) invA0 = mfem.HypreBoomerAMG(S) invA0.iterative_mode = False else: cls = SparseSmootherCls[name][0] invA0 = cls(A0, gui=self.gui[prc]) invA0.iterative_mode = False M.SetDiagonalBlock(k, invA0) ''' We should support Shur complement type preconditioner if offset.Size() > 2: B = get_block(A, 1, 0) MinvBt = get_block(A, 0, 1) #Md = mfem.HypreParVector(MPI.COMM_WORLD, # A0.GetGlobalNumRows(), # A0.GetRowStarts()) Md = mfem.Vector() A0.GetDiag(Md) MinvBt.InvScaleRows(Md) S = mfem.ParMult(B, MinvBt) invS = mfem.HypreBoomerAMG(S) invS.iterative_mode = False M.SetDiagonalBlock(1, invS) ''' maxiter = int(self.maxiter) atol = self.abstol rtol = self.reltol kdim = int(self.kdim) printit = 1 sol = [] solver = mfem.GMRESSolver(MPI.COMM_WORLD) solver.SetKDim(kdim) #solver = mfem.MINRESSolver(MPI.COMM_WORLD) #solver.SetOperator(A) #solver = mfem.CGSolver(MPI.COMM_WORLD) solver.SetOperator(A) solver.SetAbsTol(atol) solver.SetRelTol(rtol) solver.SetMaxIter(maxiter) solver.SetPreconditioner(M) solver.SetPrintLevel(1) # solve the problem and gather solution to head node... # may not be the best approach for bb in b: rows = MPI.COMM_WORLD.allgather(np.int32(bb.Size())) rowstarts = np.hstack((0, np.cumsum(rows))) dprint1("rowstarts/offser", rowstarts, offset.ToList()) if x is None: xx = mfem.BlockVector(offset) xx.Assign(0.0) else: xx = x #for j in range(cols): # dprint1(x.GetBlock(j).Size()) # dprint1(x.GetBlock(j).GetDataArray()) #assert False, "must implement this" solver.Mult(bb, xx) s = [] for i in range(offset.Size() - 1): v = xx.GetBlock(i).GetDataArray() vv = gather_vector(v) if myid == 0: s.append(vv) else: pass if myid == 0: sol.append(np.hstack(s)) if myid == 0: sol = np.transpose(np.vstack(sol)) return sol else: return None
a.FormLinearSystem(ess_tdof_list, x, b, A, X, B) glob_size = A.GetGlobalNumRows() if myid == 0: print("Size of linear system: " + str(glob_size)) # 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or # the 3D ADS preconditioners from hypre. If using hybridization, the # system is preconditioned with hypre's BoomerAMG. pcg = mfem.CGSolver(A.GetComm()) pcg.SetOperator(A) pcg.SetRelTol(1e-12) pcg.SetMaxIter(500) pcg.SetPrintLevel(1) if hybridization: prec = mfem.HypreBoomerAMG(A) else: if a.StaticCondensationIsEnabled(): prec_fespace = a.SCParFESpace() else: prec_fespace = fespace if dim == 2: prec = mfem.HypreAMS(A, prec_fespace) else: prec = mfem.HypreADS(A, prec_fespace) pcg.SetPreconditioner(prec) pcg.Mult(B, X) # 13. Recover the parallel grid function corresponding to X. This is the # local finite element solution on each processor. a.RecoverFEMSolution(X, b, x)