def schur(*names, **kwargs):
# shucr("A1", "B1", scale=(1.0, 1e3))
prc = kwargs.pop('prc')
blockname = kwargs.pop('blockname')
r0 = prc.get_row_by_name(blockname)
c0 = prc.get_col_by_name(blockname)
A0 = prc.get_operator_block(r0, c0)
scales = kwargs.pop('scale', [1] * len(names))
print_level = kwargs.pop('print_level', -1)
for name, scale in zip(names, scales):
r1 = prc.get_row_by_name(name)
c1 = prc.get_col_by_name(name)
B = prc.get_operator_block(r0, c1)
Bt = prc.get_operator_block(r1, c0)
Bt = Bt.Copy()
B0 = get_block(A, r1, c1)
if use_parallel:
Md = mfem.HyprePaarVector(MPI.COMM_WORLD, B0.GetGlobalNumRows(),
B0.GetColStarts())
else:
Md = mfem.Vector()
A0.GetDiag(Md)
Md *= scale
if use_parallel:
Bt.InvScaleRows(Md)
S = mfem.ParMult(B, Bt)
invA0 = mfem.HypreBoomerAMG(S)
else:
S = mfem.Mult(B, Bt)
invA0 = mfem.DSmoother(S)
invA0.iterative_mode = False
invA0.SetPrintLevel(print_level)
return invA0
def __init__(self, spaces, mass, offsets):
self.pressure_mass = mass
self.block_offsets = offsets
self.spaces = spaces
super(JacobianPreconditioner, self).__init__(offsets[2])
self.gamma = 0.00001
# The mass matrix and preconditioner do not change every Newton cycle, so we
# only need to define them once
self.mass_prec = mfem.HypreBoomerAMG()
self.mass_prec.SetPrintLevel(0)
mass_pcg = mfem.CGSolver(MPI.COMM_WORLD)
mass_pcg.SetRelTol(1e-12)
mass_pcg.SetAbsTol(1e-12)
mass_pcg.SetMaxIter(200)
mass_pcg.SetPrintLevel(0)
mass_pcg.SetPreconditioner(self.mass_prec)
mass_pcg.SetOperator(self.pressure_mass)
mass_pcg.iterative_mode = False
self.mass_pcg = mass_pcg
# The stiffness matrix does change every Newton cycle, so we will define it
# during SetOperator
self.stiff_pcg = None
self.stiff_prec = None
def schur(*names, **kwargs):
# schur("A1", "B1", scale=(1.0, 1e3))
prc = kwargs.pop('prc')
blockname = kwargs.pop('blockname')
r0 = prc.get_row_by_name(blockname)
c0 = prc.get_col_by_name(blockname)
scales = kwargs.pop('scale', [1] * len(names))
print_level = kwargs.pop('print_level', -1)
S = []
for name, scale in zip(names, scales):
r1 = prc.get_row_by_name(name)
c1 = prc.get_col_by_name(name)
B = prc.get_operator_block(r0, c1)
Bt = prc.get_operator_block(r1, c0)
B0 = prc.get_operator_block(r1, c1)
if use_parallel:
Bt = Bt.Transpose()
Bt = Bt.Transpose()
Md = mfem.HypreParVector(MPI.COMM_WORLD, B0.GetGlobalNumRows(),
B0.GetColStarts())
else:
Bt = Bt.Copy()
Md = mfem.Vector()
B0.GetDiag(Md)
Md *= scale
if use_parallel:
Bt.InvScaleRows(Md)
S.append(mfem.ParMult(B, Bt))
else:
S.append(mfem.Mult(B, Bt))
if use_parallel:
from mfem.common.parcsr_extra import ToHypreParCSR, ToScipyCoo
S2 = [ToScipyCoo(s) for s in S]
for s in S2[1:]:
S2[0] = S2[0] + s
S = ToHypreParCSR(S2[0].tocsr())
invA0 = mfem.HypreBoomerAMG(S)
else:
from mfem.common.sparse_utils import sparsemat_to_scipycsr
S2 = [sparsemat_to_scipycsr(s).tocoo() for s in S]
for s in S2[1:]:
S2[0] = S2[0] + s
S = mfem.SparseMatrix(S2.tocsr())
invA0 = mfem.DSmoother(S)
invA0.iterative_mode = False
invA0.SetPrintLevel(print_level)
invA0._S = S
return invA0
def boomerAMG(**kwargs):
prc = kwargs.pop('prc')
blockname = kwargs.pop('blockname')
print_level = kwargs.pop('print_level', -1)
row = prc.get_row_by_name(blockname)
col = prc.get_col_by_name(blockname)
mat = prc.get_operator_block(row, col)
inv_boomeramg = mfem.HypreBoomerAMG(mat)
inv_boomeramg.SetPrintLevel(print_level)
inv_boomeramg.iterative_mode = False
return inv_boomeramg
def SetOperator(self, op):
self.jacobian = mfem.Opr2BlockOpr(op)
if (self.stiff_prec == None):
# Initialize the stiffness preconditioner and solver
stiff_prec_amg = mfem.HypreBoomerAMG()
stiff_prec_amg.SetPrintLevel(0)
stiff_prec_amg.SetElasticityOptions(self.spaces[0])
self.stiff_prec = stiff_prec_amg
stiff_pcg_iter = mfem.GMRESSolver(MPI.COMM_WORLD)
stiff_pcg_iter.SetRelTol(1e-8)
stiff_pcg_iter.SetAbsTol(1e-8)
stiff_pcg_iter.SetMaxIter(200)
stiff_pcg_iter.SetPrintLevel(0)
stiff_pcg_iter.SetPreconditioner(self.stiff_prec)
stiff_pcg_iter.iterative_mode = False
self.stiff_pcg = stiff_pcg_iter
# At each Newton cycle, compute the new stiffness preconditioner by updating
# the iterative solver which, in turn, updates its preconditioner
self.stiff_pcg.SetOperator(self.jacobian.GetBlock(0, 0))
a.Assemble()
b.Assemble()
# 16. Project the exact solution to the essential DOFs.
x.ProjectBdrCoefficient(bdr, ess_bdr)
# 17. Create and solve the parallel linear system.
ess_tdof_list = mfem.intArray()
fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list)
A = mfem.HypreParMatrix()
B = mfem.Vector()
X = mfem.Vector()
a.FormLinearSystem(ess_tdof_list, x, b, A, X, B)
amg = mfem.HypreBoomerAMG(A)
amg.SetPrintLevel(0)
pcg = mfem.HyprePCG(A)
pcg.SetTol(1e-12)
pcg.SetMaxIter(200)
pcg.SetPrintLevel(0)
pcg.SetPreconditioner(amg)
pcg.Mult(B, X)
# 18. Extract the local solution on each processor.
a.RecoverFEMSolution(X, b, x)
# 19. Send the solution by socket to a GLVis server
if visualization:
sout.send_text("parallel " + str(num_procs) + " " + str(myid))
sout.precision(8)
SinvF = mfem.HypreParVector(test_space)
matSinv.Mult(trueF, SinvF)
B.MultTranspose(SinvF, b)
# 11. Set up a block-diagonal preconditioner for the 2x2 normal equation
#
# [ S0^{-1} 0 ]
# [ 0 Shat^{-1} ] Shat = (Bhat^T Sinv Bhat)
#
# corresponding to the primal (x0) and interfacial (xhat) unknowns.
# Since the Shat operator is equivalent to an H(div) matrix reduced to
# the interfacial skeleton, we approximate its inverse with one V-cycle
# of the ADS preconditioner from the hypre library (in 2D we use AMS for
# the rotated H(curl) problem).
S0inv = mfem.HypreBoomerAMG(matS0)
S0inv.SetPrintLevel(0)
Shat = mfem.RAP(matSinv, matBhat)
if (dim == 2): Shatinv = mfem.HypreAMS(Shat, xhat_space)
else: Shatinv = mfem.HypreADS(Shat, xhat_space)
P = mfem.BlockDiagonalPreconditioner(true_offsets)
P.SetDiagonalBlock(0, S0inv)
P.SetDiagonalBlock(1, Shatinv)
# 12. Solve the normal equation system using the PCG iterative solver.
# Check the weighted norm of residual for the DPG least square problem.
# Wrap the primal variable in a GridFunction for visualization purposes.
pcg = mfem.CGSolver(MPI.COMM_WORLD)
pcg.SetOperator(A)
darcyOp = mfem.BlockOperator(block_trueOffsets)
darcyOp.SetBlock(0, 0, M)
darcyOp.SetBlock(0, 1, BT)
darcyOp.SetBlock(1, 0, B)
#M2 = M.Transpose()
#M3 = M2.Transpose()
MinvBt = B.Transpose()
Md = mfem.HypreParVector(MPI.COMM_WORLD, M.GetGlobalNumRows(),
M.GetRowStarts())
M.GetDiag(Md)
MinvBt.InvScaleRows(Md)
S = mfem.hypre.ParMult(B, MinvBt)
invM = mfem.HypreDiagScale(M)
invS = mfem.HypreBoomerAMG(S)
invM.iterative_mode = False
invS.iterative_mode = False
darcyPr = mfem.BlockDiagonalPreconditioner(block_trueOffsets)
darcyPr.SetDiagonalBlock(0, invM)
darcyPr.SetDiagonalBlock(1, invS)
maxIter = 500
rtol = 1e-6
atol = 1e-10
import time
stime = time.clock()
solver = mfem.MINRESSolver(MPI.COMM_WORLD)
def run(order = 1, static_cond = False,
meshfile = def_meshfile, visualization = False,
use_strumpack = False):
mesh = mfem.Mesh(meshfile, 1,1)
dim = mesh.Dimension()
ref_levels = int(np.floor(np.log(10000./mesh.GetNE())/np.log(2.)/dim))
for x in range(ref_levels):
mesh.UniformRefinement();
mesh.ReorientTetMesh();
pmesh = mfem.ParMesh(MPI.COMM_WORLD, mesh)
del mesh
par_ref_levels = 2
for l in range(par_ref_levels):
pmesh.UniformRefinement();
if order > 0:
fec = mfem.H1_FECollection(order, dim)
elif mesh.GetNodes():
fec = mesh.GetNodes().OwnFEC()
print( "Using isoparametric FEs: " + str(fec.Name()));
else:
order = 1
fec = mfem.H1_FECollection(order, dim)
fespace =mfem.ParFiniteElementSpace(pmesh, fec)
fe_size = fespace.GlobalTrueVSize()
if (myid == 0):
print('Number of finite element unknowns: '+ str(fe_size))
ess_tdof_list = mfem.intArray()
if pmesh.bdr_attributes.Size()>0:
ess_bdr = mfem.intArray(pmesh.bdr_attributes.Max())
ess_bdr.Assign(1)
fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list)
# the basis functions in the finite element fespace.
b = mfem.ParLinearForm(fespace)
one = mfem.ConstantCoefficient(1.0)
b.AddDomainIntegrator(mfem.DomainLFIntegrator(one))
b.Assemble();
x = mfem.ParGridFunction(fespace);
x.Assign(0.0)
a = mfem.ParBilinearForm(fespace);
a.AddDomainIntegrator(mfem.DiffusionIntegrator(one))
if static_cond: a.EnableStaticCondensation()
a.Assemble();
A = mfem.HypreParMatrix()
B = mfem.Vector()
X = mfem.Vector()
a.FormLinearSystem(ess_tdof_list, x, b, A, X, B)
if (myid == 0):
print("Size of linear system: " + str(x.Size()))
print("Size of linear system: " + str(A.GetGlobalNumRows()))
if use_strumpack:
import mfem.par.strumpack as strmpk
Arow = strmpk.STRUMPACKRowLocMatrix(A)
args = ["--sp_hss_min_sep_size", "128", "--sp_enable_hss"]
strumpack = strmpk.STRUMPACKSolver(args, MPI.COMM_WORLD)
strumpack.SetPrintFactorStatistics(True)
strumpack.SetPrintSolveStatistics(False)
strumpack.SetKrylovSolver(strmpk.KrylovSolver_DIRECT);
strumpack.SetReorderingStrategy(strmpk.ReorderingStrategy_METIS)
strumpack.SetMC64Job(strmpk.MC64Job_NONE)
# strumpack.SetSymmetricPattern(True)
strumpack.SetOperator(Arow)
strumpack.SetFromCommandLine()
strumpack.Mult(B, X);
else:
amg = mfem.HypreBoomerAMG(A)
cg = mfem.CGSolver(MPI.COMM_WORLD)
cg.SetRelTol(1e-12)
cg.SetMaxIter(200)
cg.SetPrintLevel(1)
cg.SetPreconditioner(amg)
cg.SetOperator(A)
cg.Mult(B, X);
a.RecoverFEMSolution(X, b, x)
smyid = '{:0>6d}'.format(myid)
mesh_name = "mesh."+smyid
sol_name = "sol."+smyid
pmesh.Print(mesh_name, 8)
x.Save(sol_name, 8)
def run(order = 1, static_cond = False,
meshfile = def_meshfile, visualization = False):
mesh = mfem.Mesh(meshfile, 1,1)
dim = mesh.Dimension()
ref_levels = int(np.floor(np.log(10000./mesh.GetNE())/np.log(2.)/dim))
for x in range(ref_levels):
mesh.UniformRefinement();
mesh.ReorientTetMesh();
pmesh = mfem.ParMesh(MPI.COMM_WORLD, mesh)
del mesh
par_ref_levels = 2
for l in range(par_ref_levels):
pmesh.UniformRefinement();
if order > 0:
fec = mfem.H1_FECollection(order, dim)
elif mesh.GetNodes():
fec = mesh.GetNodes().OwnFEC()
prinr( "Using isoparametric FEs: " + str(fec.Name()));
else:
order = 1
fec = mfem.H1_FECollection(order, dim)
fespace =mfem.ParFiniteElementSpace(pmesh, fec)
fe_size = fespace.GlobalTrueVSize()
if (myid == 0):
print('Number of finite element unknowns: '+ str(fe_size))
ess_tdof_list = mfem.intArray()
if pmesh.bdr_attributes.Size()>0:
ess_bdr = mfem.intArray(pmesh.bdr_attributes.Max())
ess_bdr.Assign(1)
fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list)
# the basis functions in the finite element fespace.
b = mfem.ParLinearForm(fespace)
one = mfem.ConstantCoefficient(1.0)
b.AddDomainIntegrator(mfem.DomainLFIntegrator(one))
b.Assemble();
x = mfem.ParGridFunction(fespace);
x.Assign(0.0)
a = mfem.ParBilinearForm(fespace);
a.AddDomainIntegrator(mfem.DiffusionIntegrator(one))
if static_cond: a.EnableStaticCondensation()
a.Assemble();
A = mfem.HypreParMatrix()
B = mfem.Vector()
X = mfem.Vector()
a.FormLinearSystem(ess_tdof_list, x, b, A, X, B)
if (myid == 0):
print("Size of linear system: " + str(x.Size()))
print("Size of linear system: " + str(A.GetGlobalNumRows()))
amg = mfem.HypreBoomerAMG(A)
pcg = mfem.HyprePCG(A)
pcg.SetTol(1e-12)
pcg.SetMaxIter(200)
pcg.SetPrintLevel(2)
pcg.SetPreconditioner(amg)
pcg.Mult(B, X);
a.RecoverFEMSolution(X, b, x)
smyid = '{:0>6d}'.format(myid)
mesh_name = "mesh."+smyid
sol_name = "sol."+smyid
pmesh.PrintToFile(mesh_name, 8)
x.SaveToFile(sol_name, 8)
def solve_parallel(self, A, b, x=None):
from mpi4py import MPI
myid = MPI.COMM_WORLD.rank
nproc = MPI.COMM_WORLD.size
from petram.helper.mpi_recipes import gather_vector
def get_block(Op, i, j):
try:
return Op._linked_op[(i, j)]
except KeyError:
return None
offset = A.RowOffsets()
rows = A.NumRowBlocks()
cols = A.NumColBlocks()
if self.gui.write_mat:
for i in range(cols):
for j in range(rows):
m = get_block(A, i, j)
if m is None: continue
m.Print('matrix_' + str(i) + '_' + str(j))
for i, bb in enumerate(b):
for j in range(rows):
v = bb.GetBlock(j)
v.Print('rhs_' + str(i) + '_' + str(j) + '.' + smyid)
if x is not None:
for j in range(rows):
xx = x.GetBlock(j)
xx.Print('x_' + str(i) + '_' + str(j) + '.' + smyid)
M = mfem.BlockDiagonalPreconditioner(offset)
prcs = dict(self.gui.preconditioners)
name = self.Aname
assert not self.gui.parent.is_complex(), "can not solve complex"
if self.gui.parent.is_converted_from_complex():
name = sum([[n, n] for n in name], [])
for k, n in enumerate(name):
prc = prcs[n][1]
if prc == "None": continue
name = "".join([tmp for tmp in prc if not tmp.isdigit()])
A0 = get_block(A, k, k)
if A0 is None and not name.startswith('schur'): continue
if hasattr(mfem.HypreSmoother, prc):
invA0 = mfem.HypreSmoother(A0)
invA0.SetType(getattr(mfem.HypreSmoother, prc))
elif prc == 'ams':
depvar = self.engine.r_dep_vars[k]
dprint1("setting up AMS for ", depvar)
prec_fespace = self.engine.fespaces[depvar]
invA0 = mfem.HypreAMS(A0, prec_fespace)
invA0.SetSingularProblem()
elif name == 'MUMPS':
cls = SparseSmootherCls[name][0]
invA0 = cls(A0, gui=self.gui[prc], engine=self.engine)
elif name.startswith('schur'):
args = name.split("(")[-1].split(")")[0].split(",")
dprint1("setting up schur for ", args)
if len(args) > 1:
assert False, "not yet supported"
for arg in args:
r1 = self.engine.dep_var_offset(arg.strip())
c1 = self.engine.r_dep_var_offset(arg.strip())
B = get_block(A, k, c1)
Bt = get_block(A, r1, k).Transpose()
Bt = Bt.Transpose()
B0 = get_block(A, r1, c1)
Md = mfem.HypreParVector(MPI.COMM_WORLD,
B0.GetGlobalNumRows(),
B0.GetColStarts())
B0.GetDiag(Md)
Bt.InvScaleRows(Md)
S = mfem.ParMult(B, Bt)
invA0 = mfem.HypreBoomerAMG(S)
invA0.iterative_mode = False
else:
cls = SparseSmootherCls[name][0]
invA0 = cls(A0, gui=self.gui[prc])
invA0.iterative_mode = False
M.SetDiagonalBlock(k, invA0)
'''
We should support Shur complement type preconditioner
if offset.Size() > 2:
B = get_block(A, 1, 0)
MinvBt = get_block(A, 0, 1)
#Md = mfem.HypreParVector(MPI.COMM_WORLD,
# A0.GetGlobalNumRows(),
# A0.GetRowStarts())
Md = mfem.Vector()
A0.GetDiag(Md)
MinvBt.InvScaleRows(Md)
S = mfem.ParMult(B, MinvBt)
invS = mfem.HypreBoomerAMG(S)
invS.iterative_mode = False
M.SetDiagonalBlock(1, invS)
'''
maxiter = int(self.maxiter)
atol = self.abstol
rtol = self.reltol
kdim = int(self.kdim)
printit = 1
sol = []
solver = mfem.GMRESSolver(MPI.COMM_WORLD)
solver.SetKDim(kdim)
#solver = mfem.MINRESSolver(MPI.COMM_WORLD)
#solver.SetOperator(A)
#solver = mfem.CGSolver(MPI.COMM_WORLD)
solver.SetOperator(A)
solver.SetAbsTol(atol)
solver.SetRelTol(rtol)
solver.SetMaxIter(maxiter)
solver.SetPreconditioner(M)
solver.SetPrintLevel(1)
# solve the problem and gather solution to head node...
# may not be the best approach
for bb in b:
rows = MPI.COMM_WORLD.allgather(np.int32(bb.Size()))
rowstarts = np.hstack((0, np.cumsum(rows)))
dprint1("rowstarts/offser", rowstarts, offset.ToList())
if x is None:
xx = mfem.BlockVector(offset)
xx.Assign(0.0)
else:
xx = x
#for j in range(cols):
# dprint1(x.GetBlock(j).Size())
# dprint1(x.GetBlock(j).GetDataArray())
#assert False, "must implement this"
solver.Mult(bb, xx)
s = []
for i in range(offset.Size() - 1):
v = xx.GetBlock(i).GetDataArray()
vv = gather_vector(v)
if myid == 0:
s.append(vv)
else:
pass
if myid == 0:
sol.append(np.hstack(s))
if myid == 0:
sol = np.transpose(np.vstack(sol))
return sol
else:
return None
a.FormLinearSystem(ess_tdof_list, x, b, A, X, B)
glob_size = A.GetGlobalNumRows()
if myid == 0:
print("Size of linear system: " + str(glob_size))
# 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
# the 3D ADS preconditioners from hypre. If using hybridization, the
# system is preconditioned with hypre's BoomerAMG.
pcg = mfem.CGSolver(A.GetComm())
pcg.SetOperator(A)
pcg.SetRelTol(1e-12)
pcg.SetMaxIter(500)
pcg.SetPrintLevel(1)
if hybridization: prec = mfem.HypreBoomerAMG(A)
else:
if a.StaticCondensationIsEnabled():
prec_fespace = a.SCParFESpace()
else:
prec_fespace = fespace
if dim == 2: prec = mfem.HypreAMS(A, prec_fespace)
else: prec = mfem.HypreADS(A, prec_fespace)
pcg.SetPreconditioner(prec)
pcg.Mult(B, X)
# 13. Recover the parallel grid function corresponding to X. This is the
# local finite element solution on each processor.
a.RecoverFEMSolution(X, b, x)
