def ToHypreParVec(vec): import mfem.par as mfem from mpi4py import MPI if mfem.sizeof_HYPRE_Int() == 4: dtype = 'int32' else: dtype = 'int64' comm = MPI.COMM_WORLD num_proc = MPI.COMM_WORLD.size myid = MPI.COMM_WORLD.rank vec = vec.flatten() ml = vec.shape[0] # collect col array to determin partitioning m_array = comm.allgather(ml) cols = [0] + list(np.cumsum(m_array)) glob_size = cols[-1] col_starts = np.array([cols[myid], cols[myid + 1], glob_size], dtype=dtype) vec = vec.astype('float', copy=False) v = mfem.HypreParVector(MPI.COMM_WORLD, glob_size, [vec, col_starts]) return v
def schur(*names, **kwargs): # schur("A1", "B1", scale=(1.0, 1e3)) prc = kwargs.pop('prc') blockname = kwargs.pop('blockname') r0 = prc.get_row_by_name(blockname) c0 = prc.get_col_by_name(blockname) scales = kwargs.pop('scale', [1] * len(names)) print_level = kwargs.pop('print_level', -1) S = [] for name, scale in zip(names, scales): r1 = prc.get_row_by_name(name) c1 = prc.get_col_by_name(name) B = prc.get_operator_block(r0, c1) Bt = prc.get_operator_block(r1, c0) B0 = prc.get_operator_block(r1, c1) if use_parallel: Bt = Bt.Transpose() Bt = Bt.Transpose() Md = mfem.HypreParVector(MPI.COMM_WORLD, B0.GetGlobalNumRows(), B0.GetColStarts()) else: Bt = Bt.Copy() Md = mfem.Vector() B0.GetDiag(Md) Md *= scale if use_parallel: Bt.InvScaleRows(Md) S.append(mfem.ParMult(B, Bt)) else: S.append(mfem.Mult(B, Bt)) if use_parallel: from mfem.common.parcsr_extra import ToHypreParCSR, ToScipyCoo S2 = [ToScipyCoo(s) for s in S] for s in S2[1:]: S2[0] = S2[0] + s S = ToHypreParCSR(S2[0].tocsr()) invA0 = mfem.HypreBoomerAMG(S) else: from mfem.common.sparse_utils import sparsemat_to_scipycsr S2 = [sparsemat_to_scipycsr(s).tocoo() for s in S] for s in S2[1:]: S2[0] = S2[0] + s S = mfem.SparseMatrix(S2.tocsr()) invA0 = mfem.DSmoother(S) invA0.iterative_mode = False invA0.SetPrintLevel(print_level) invA0._S = S return invA0
def ParMultVecComplex(A, v): ''' A*v ''' from mpi4py import MPI comm = MPI.COMM_WORLD num_proc = MPI.COMM_WORLD.size myid = MPI.COMM_WORLD.rank R_A, I_A = A R_v, I_v = v if I_A is None and I_v is None: ans_r = mfem.HypreParVector(R_v) R_A.Mult(R_v, ans_r) return (ans_r, None) ans_r = mfem.HypreParVector(R_v) ans_i = mfem.HypreParVector(R_v) if I_A is None: R_A.Mult(R_v, ans_r) R_A.Mult(I_v, ans_i) elif I_v is None: R_A.Mult(R_v, ans_r) I_A.Mult(R_v, ans_i) else: ans_r2 = mfem.HypreParVector(R_v) ans_i2 = mfem.HypreParVector(I_v) R_A.Mult(R_v, ans_r) I_A.Mult(I_v, ans_r2) ans_r -= ans_r2 R_A.Mult(I_v, ans_i) I_A.Mult(R_v, ans_i2) ans_i += ans_i2 return (ans_r, ans_i)
x.Assign(0.0) b.Assign(0.0) # 10. Set up the 1x2 block Least Squares DPG operator, B = [B0 Bhat], # the normal equation operator, A = B^t Sinv B, and # the normal equation right-hand-size, b = B^t Sinv F. B = mfem.BlockOperator(true_offsets_test, true_offsets) B.SetBlock(0, 0, matB0) B.SetBlock(0, 1, matBhat) A = mfem.RAPOperator(B, matSinv, B) trueF = F.ParallelAssemble() SinvF = mfem.HypreParVector(test_space) matSinv.Mult(trueF, SinvF) B.MultTranspose(SinvF, b) # 11. Set up a block-diagonal preconditioner for the 2x2 normal equation # # [ S0^{-1} 0 ] # [ 0 Shat^{-1} ] Shat = (Bhat^T Sinv Bhat) # # corresponding to the primal (x0) and interfacial (xhat) unknowns. # Since the Shat operator is equivalent to an H(div) matrix reduced to # the interfacial skeleton, we approximate its inverse with one V-cycle # of the ADS preconditioner from the hypre library (in 2D we use AMS for # the rotated H(curl) problem). S0inv = mfem.HypreBoomerAMG(matS0) S0inv.SetPrintLevel(0)
bVarf.AddDomainIntegrator(mfem.VectorFEDivergenceIntegrator()) bVarf.Assemble() bVarf.Finalize() B = bVarf.ParallelAssemble() B *= -1 BT = B.Transpose() darcyOp = mfem.BlockOperator(block_trueOffsets) darcyOp.SetBlock(0, 0, M) darcyOp.SetBlock(0, 1, BT) darcyOp.SetBlock(1, 0, B) #M2 = M.Transpose() #M3 = M2.Transpose() MinvBt = B.Transpose() Md = mfem.HypreParVector(MPI.COMM_WORLD, M.GetGlobalNumRows(), M.GetRowStarts()) M.GetDiag(Md) MinvBt.InvScaleRows(Md) S = mfem.hypre.ParMult(B, MinvBt) invM = mfem.HypreDiagScale(M) invS = mfem.HypreBoomerAMG(S) invM.iterative_mode = False invS.iterative_mode = False darcyPr = mfem.BlockDiagonalPreconditioner(block_trueOffsets) darcyPr.SetDiagonalBlock(0, invM) darcyPr.SetDiagonalBlock(1, invS) maxIter = 500 rtol = 1e-6
def solve_parallel(self, A, b, x=None): from mpi4py import MPI myid = MPI.COMM_WORLD.rank nproc = MPI.COMM_WORLD.size from petram.helper.mpi_recipes import gather_vector def get_block(Op, i, j): try: return Op._linked_op[(i, j)] except KeyError: return None offset = A.RowOffsets() rows = A.NumRowBlocks() cols = A.NumColBlocks() if self.gui.write_mat: for i in range(cols): for j in range(rows): m = get_block(A, i, j) if m is None: continue m.Print('matrix_' + str(i) + '_' + str(j)) for i, bb in enumerate(b): for j in range(rows): v = bb.GetBlock(j) v.Print('rhs_' + str(i) + '_' + str(j) + '.' + smyid) if x is not None: for j in range(rows): xx = x.GetBlock(j) xx.Print('x_' + str(i) + '_' + str(j) + '.' + smyid) M = mfem.BlockDiagonalPreconditioner(offset) prcs = dict(self.gui.preconditioners) name = self.Aname assert not self.gui.parent.is_complex(), "can not solve complex" if self.gui.parent.is_converted_from_complex(): name = sum([[n, n] for n in name], []) for k, n in enumerate(name): prc = prcs[n][1] if prc == "None": continue name = "".join([tmp for tmp in prc if not tmp.isdigit()]) A0 = get_block(A, k, k) if A0 is None and not name.startswith('schur'): continue if hasattr(mfem.HypreSmoother, prc): invA0 = mfem.HypreSmoother(A0) invA0.SetType(getattr(mfem.HypreSmoother, prc)) elif prc == 'ams': depvar = self.engine.r_dep_vars[k] dprint1("setting up AMS for ", depvar) prec_fespace = self.engine.fespaces[depvar] invA0 = mfem.HypreAMS(A0, prec_fespace) invA0.SetSingularProblem() elif name == 'MUMPS': cls = SparseSmootherCls[name][0] invA0 = cls(A0, gui=self.gui[prc], engine=self.engine) elif name.startswith('schur'): args = name.split("(")[-1].split(")")[0].split(",") dprint1("setting up schur for ", args) if len(args) > 1: assert False, "not yet supported" for arg in args: r1 = self.engine.dep_var_offset(arg.strip()) c1 = self.engine.r_dep_var_offset(arg.strip()) B = get_block(A, k, c1) Bt = get_block(A, r1, k).Transpose() Bt = Bt.Transpose() B0 = get_block(A, r1, c1) Md = mfem.HypreParVector(MPI.COMM_WORLD, B0.GetGlobalNumRows(), B0.GetColStarts()) B0.GetDiag(Md) Bt.InvScaleRows(Md) S = mfem.ParMult(B, Bt) invA0 = mfem.HypreBoomerAMG(S) invA0.iterative_mode = False else: cls = SparseSmootherCls[name][0] invA0 = cls(A0, gui=self.gui[prc]) invA0.iterative_mode = False M.SetDiagonalBlock(k, invA0) ''' We should support Shur complement type preconditioner if offset.Size() > 2: B = get_block(A, 1, 0) MinvBt = get_block(A, 0, 1) #Md = mfem.HypreParVector(MPI.COMM_WORLD, # A0.GetGlobalNumRows(), # A0.GetRowStarts()) Md = mfem.Vector() A0.GetDiag(Md) MinvBt.InvScaleRows(Md) S = mfem.ParMult(B, MinvBt) invS = mfem.HypreBoomerAMG(S) invS.iterative_mode = False M.SetDiagonalBlock(1, invS) ''' maxiter = int(self.maxiter) atol = self.abstol rtol = self.reltol kdim = int(self.kdim) printit = 1 sol = [] solver = mfem.GMRESSolver(MPI.COMM_WORLD) solver.SetKDim(kdim) #solver = mfem.MINRESSolver(MPI.COMM_WORLD) #solver.SetOperator(A) #solver = mfem.CGSolver(MPI.COMM_WORLD) solver.SetOperator(A) solver.SetAbsTol(atol) solver.SetRelTol(rtol) solver.SetMaxIter(maxiter) solver.SetPreconditioner(M) solver.SetPrintLevel(1) # solve the problem and gather solution to head node... # may not be the best approach for bb in b: rows = MPI.COMM_WORLD.allgather(np.int32(bb.Size())) rowstarts = np.hstack((0, np.cumsum(rows))) dprint1("rowstarts/offser", rowstarts, offset.ToList()) if x is None: xx = mfem.BlockVector(offset) xx.Assign(0.0) else: xx = x #for j in range(cols): # dprint1(x.GetBlock(j).Size()) # dprint1(x.GetBlock(j).GetDataArray()) #assert False, "must implement this" solver.Mult(bb, xx) s = [] for i in range(offset.Size() - 1): v = xx.GetBlock(i).GetDataArray() vv = gather_vector(v) if myid == 0: s.append(vv) else: pass if myid == 0: sol.append(np.hstack(s)) if myid == 0: sol = np.transpose(np.vstack(sol)) return sol else: return None