def Solve(a, f, petsc_opts, kvecs):
ngs.ngsglobals.msg_level = 5
gfu = ngs.GridFunction(a.space)
with ngs.TaskManager():
a.Assemble()
f.Assemble()
ngs.ngsglobals.msg_level = 5
awrap = petsc.PETScMatrix(a.mat,
V.FreeDofs(),
format=petsc.PETScMatrix.AIJ)
awrap.SetNearNullSpace(kvecs)
c = petsc.KSP(awrap,
finalize=True,
name="msaKSP",
petsc_options=petsc_opts)
gfu.vec.data = c * f.vec
return gfu
def systems_and_preconditioners(mesh, X, z, order=1):
# Create a real analogue of our vector space for the wrapper preconditioner.
dirichlet = 'top|right|bottom|left'
X_real = ng.H1(mesh, order=order, dirichlet=dirichlet, complex=False)
# Test and trial functions for the original space.
u, v = X.TnT()
# Real trial and test functions.
ur, vr = X_real.TnT()
# Create a real analogue of the bilinear form a.
a_real = ng.BilinearForm(X_real)
a_real += ng.SymbolicBFI(ng.grad(ur) * ng.grad(vr))
a_real.Assemble()
# Initialize petsc prior to conswtructing preconditioners.
petsc.Initialize()
# Create a bilinear form and a preconditioner for each z.
zbas = []
precs = []
for k in range(len(z)):
# Create a bilinear form for the given z-value.
zba = ng.BilinearForm(X)
zba += ng.SymbolicBFI(z[k] * u * v - ng.grad(u) * ng.grad(v))
# Create a preconditioner for the given z-value.
mat_convert = petsc.PETScMatrix(a_real.mat, freedofs=X_real.FreeDofs())
real_pc = petsc.PETSc2NGsPrecond(mat=mat_convert,
name="real_pc",
petsc_options={"pc_type": "gamg"})
prec = WrapperPrec(real_pc)
# Assemble the given bilinear form.
zba.Assemble()
# Tack the bilinear forms and preconditioners onto their respective lists.
zbas += [zba]
precs += [prec]
return zbas, precs
def Solve(a, f, petsc_opts, kvecs):
ngs.ngsglobals.msg_level = 5
gfu = ngs.GridFunction(a.space)
with ngs.TaskManager():
a.Assemble()
f.Assemble()
ngs.ngsglobals.msg_level = 5
awrap = petsc.PETScMatrix(a.mat, V.FreeDofs(), format=petsc.PETScMatrix.AIJ)
awrap.SetNearNullSpace(kvecs)
c = petsc.KSP(awrap, finalize=True, name="msaKSP", petsc_options = petsc_opts)
ngs.mpi_world.Barrier()
ts = ngs.mpi_world.WTime()
gfu.vec.data = c * f.vec
ngs.mpi_world.Barrier()
ts = ngs.mpi_world.WTime() - ts
print("ndof ", a.space.ndofglobal)
print("dofs / (sec * np) = ", (a.space.ndofglobal * a.space.dim) / (ts * max(ngs.mpi_world.size-1,1)))
return gfu
def SolveProblem(order=order, refines=refines):
# load mesh from file
for i in range(refines):
t0 = timeit.time()
mesh = MakeStructured2DMesh(quads=False, nx=8 * 2**i, ny=8 * 2**i)
t1 = timeit.time()
# C0-HDG FESpaces
VT = H1(mesh, order=order + 1, dirichlet=".*")
if bc == "clamped": # clamped bc
VF = TangentialFacetFESpace(mesh, order=order - 1,
dirichlet=".*") # with bdry condition
else:
VF = TangentialFacetFESpace(mesh,
order=order - 1) # no bdry condition
V = FESpace([VT, VF])
gfu = GridFunction(V)
(xi, uhat), (zeta, vhat) = V.TnT()
n = specialcf.normal(2)
h = specialcf.mesh_size
def tang(v):
return v - (v * n) * n
u = CoefficientFunction((grad(xi)[1], -grad(xi)[0]))
v = CoefficientFunction((grad(zeta)[1], -grad(zeta)[0]))
hess_zeta = zeta.Operator("hesse")
# gradient by row
gradv = CoefficientFunction(
((hess_zeta[1], hess_zeta[3]), (-hess_zeta[0], -hess_zeta[1])),
dims=(2, 2))
hess_xi = xi.Operator("hesse")
gradu = CoefficientFunction(
((hess_xi[1], hess_xi[3]), (-hess_xi[0], -hess_xi[1])),
dims=(2, 2))
f = LinearForm(V)
f += zeta * dx
a = BilinearForm(V, condense=True, symmetric=True)
lap = tau * u * v
bilap = InnerProduct(gradu, gradv)
bilap_BND = -(gradu * n * tang(v - vhat) + gradv * n * tang(u - uhat) -
4 * (order + 1)**2 / h * tang(u - uhat) * tang(v - vhat))
ir = IntegrationRule(SEGM, 2 * order - 1)
a += (lap + bilap) * dx
a += bilap_BND * dx(element_boundary=True, intrules={SEGM: ir})
########### ASP 1: C0--> HDiv
########### ASP 1: C0--> HDiv
########### ASP 1: C0--> HDiv
# two grid (Hdiv-HDG)
VT1 = HDiv(mesh, order=order, hodivfree=True, dirichlet='.*')
V1 = FESpace([VT1, VF])
(u1, uhat1), (v1, vhat1) = V1.TnT()
# Mapping HDiv--> H1
mixmass = BilinearForm(trialspace=V1, testspace=V)
# tangential part
mixmass += u1 * v * dx
# construct Pi-hat
xiT, zetaT = VT.TnT()
uT = CoefficientFunction((grad(xiT)[1], -grad(xiT)[0]))
vT = CoefficientFunction((grad(zetaT)[1], -grad(zetaT)[0]))
uhatT, vhatT = VF.TnT()
# tangential part
massfX = BilinearForm(VT, symmetric=True)
massfX += uT * vT * dx
massfY = BilinearForm(trialspace=VT, testspace=VF)
massfY += -tang(uT) * tang(vhatT) * dx(element_boundary=True)
massfZ = BilinearForm(VF)
massfZ += tang(uhatT) * tang(vhatT) * dx(element_boundary=True)
############### DOFS splits for PROJECTOR
embU = Embedding(V.ndof, V.Range(0))
embV = Embedding(V.ndof, V.Range(1))
a1 = BilinearForm(V1, condense=True)
lap1 = tau * u1 * v1
gradu1 = Grad(u1)
gradv1 = Grad(v1)
bilap1 = InnerProduct(gradu1, gradv1)
jmp_u1 = tang(u1 - uhat1)
jmp_v1 = tang(v1 - vhat1)
bilap_BND1 = -(gradu1 * n * jmp_v1 + gradv1 * n * jmp_u1 - 4 *
(order + 1)**2 / h * jmp_u1 * jmp_v1)
a1 += (lap1 + bilap1) * dx
a1 += bilap_BND1 * dx(element_boundary=True, intrules={SEGM: ir})
########### ASP 2: HDiv--> H1
########### FIXME: weakly impose bdry condition
if bc == "clamped":
V0 = VectorH1(mesh, dirichlet=".*")
else:
V0 = VectorH1(mesh,
dirichletx="left|right",
dirichlety="top|bottom")
u0, v0 = V0.TnT()
a0 = BilinearForm(V0)
a0 += (InnerProduct(Grad(u0), Grad(v0)) + tau * u0 * v0) * dx
# Projection operator H1--> M
mixmass0 = BilinearForm(trialspace=V0, testspace=V1)
# tangential part
mixmass0 += tang(u0) * tang(vhat1) * dx(element_boundary=True)
mixmass0 += (u0 * n) * (v1 * n) * dx(element_boundary=True)
massf0 = BilinearForm(V1)
# tangential part
massf0 += tang(uhat1) * tang(vhat1) * dx(element_boundary=True)
massf0 += (u1 * n) * (v1 * n) * dx(element_boundary=True)
def edgePatchBlocks(mesh, fes):
blocks = []
freedofs = fes.FreeDofs(True)
for e in mesh.edges:
edofs = set()
# get ALL dofs connected to the edge
for el in mesh[e].elements:
edofs |= set(d for d in fes.GetDofNrs(el) if freedofs[d])
blocks.append(edofs)
return blocks
eBlocks = edgePatchBlocks(mesh, V1)
class SymmetricGS(BaseMatrix):
def __init__(self, smoother):
super(SymmetricGS, self).__init__()
self.smoother = smoother
def Mult(self, x, y):
y[:] = 0.0
self.smoother.Smooth(y, x)
self.smoother.SmoothBack(y, x)
def Height(self):
return self.smoother.height
def Width(self):
return self.smoother.height
with TaskManager():
f.Assemble()
a.Assemble()
######## ASP 1
mixmass.Assemble()
massfX.Assemble()
massfY.Assemble()
massfZ.Assemble()
####### RK: PROJECTION (sparse cholesky factorization)
imX = massfX.mat.Inverse(VT.FreeDofs(True),
inverse="sparsecholesky")
imZ = massfZ.mat.CreateSmoother(VF.FreeDofs(True))
m_inv = embU @ imX @ embU.T + embV @ imZ @embV.T \
- embV @ imZ @ massfY.mat @ imX @ embU.T
m_invT = embU @ imX @ embU.T + embV @ imZ @embV.T \
- embU @ imX @ massfY.mat.T @ imZ @ embV.T
E = m_inv @ mixmass.mat
ET = mixmass.mat.T @ m_invT
a1.Assemble()
## ASP 2
bjac = a1.mat.CreateBlockSmoother(eBlocks)
bgs = SymmetricGS(bjac)
mixmass0.Assemble()
massf0.Assemble()
m_inv0 = massf0.mat.CreateSmoother(V1.FreeDofs(True))
a0.Assemble()
pm0 = ngs_petsc.PETScMatrix(a0.mat, V0.FreeDofs())
inva0 = ngs_petsc.PETSc2NGsPrecond(
pm0, petsc_options={"pc_type": "hypre"})
E0 = m_inv0 @ mixmass0.mat
# ASP for rd system
pc_a1 = bgs + E0 @ inva0 @ E0.T
# Direct solver for rd system
pc_a2 = a1.mat.Inverse(V1.FreeDofs(True), inverse="sparsecholesky")
pre1 = E @ pc_a1 @ ET
pre2 = E @ pc_a2 @ ET
inv1 = CGSolver(a.mat,
pre1,
maxsteps=5000,
precision=1e-10,
printrates=True)
inv2 = CGSolver(a.mat,
pre2,
maxsteps=5000,
precision=1e-10,
printrates=True)
f.vec.data += a.harmonic_extension_trans * f.vec
# solver
gfu.vec.data = inv1 * f.vec
gfu.vec.data = inv2 * f.vec
gfu.vec.data += a.harmonic_extension * gfu.vec
gfu.vec.data += a.inner_solve * f.vec
print("bc:%s tau0:%.0e tau1 %.0e ASP:%i EXA: %i" %
(bc, tau0, tau1, inv1.GetSteps(), inv2.GetSteps()))
for RBM in RBMS:
if dim == 3:
ucf = CoefficientFunction((RBM[0], RBM[1], RBM[2]))
else:
ucf = CoefficientFunction((RBM[0], RBM[1]))
upart.Set(ucf)
v = gfu.vec.CreateVector()
v.data = gfu.vec
rbm_vecs.append(v)
return rbm_vecs
petsc.Initialize()
mat_wrap = petsc.PETScMatrix(a.mat,
freedofs=V.FreeDofs(),
format=petsc.PETScMatrix.AIJ)
opts = {"ksp_type": "cg", "ksp_atol": 1e-30, "ksp_rtol": 1e-8, "pc_type": "ml"}
ksp = petsc.KSP(mat=mat_wrap,
name="someksp",
petsc_options=opts,
finalize=False)
ksp.GetMatrix().SetNearNullSpace(rb_modes(V))
# import ngs_amg
# mat_wrap = petsc.FlatPETScMatrix(a.mat, freedofs=V.FreeDofs())
# ngs_amg_opts = {"energy" : "alg", "comp_sm" : True, "force_comp_sm" : True, "max_cv" : 500, "ass_frac" : 0.15, "skip_ass" : 3}
# ngs_pc = ngs_amg.AMG_EL2(blf=a, freedofs=V.FreeDofs(), **ngs_amg_opts)
# ngs_pc = petsc.NGs2PETSc_PC(mat=mat_wrap, pc=ngs_pc)
# ksp.SetPC(ngs_pc)
def SolveProblem(order=order, refines=refines, condense=True, symmetric=False):
for i in range(refines):
t0 = timeit.time()
mesh = MakeStructured3DMesh(hexes=False,
nx=8 * 2**i,
ny=8 * 2**i,
nz=8 * 2**i)
t1 = timeit.time()
print("\nElasped:%.2e MESHING " % (t1 - t0))
# Div-HDG spaces
V = HDiv(mesh, order=order, dirichlet=".*")
if freeBC == True:
M = TangentialFacetFESpace(mesh,
order=order,
highest_order_dc=True)
# aux H1 space
V0 = VectorH1(mesh,
order=1,
dirichlety="left|right",
dirichletx="front|back",
dirichletz="top|bottom")
else: # dir bc
M = TangentialFacetFESpace(mesh,
order=order,
highest_order_dc=True,
dirichlet=".*")
# aux H1 space
V0 = VectorH1(mesh, order=1, dirichlet=".*")
fes = FESpace([V, M])
gfu = GridFunction(fes)
(u, uhat), (v, vhat) = fes.TnT()
(u0, v0) = V0.TnT()
# gradients
gradv, gradu = Grad(v), Grad(u)
# RHS (constant)
f = LinearForm(fes)
f += (v[0] + v[1] + v[2]) * dx
# normal direction and mesh size
n = specialcf.normal(mesh.dim)
h = specialcf.mesh_size
# stability parameter
#alpha = 8*order**2/h
alpha = 8 * order**2 / h
# tangential component
def tang(v):
return v - (v * n) * n
########### HDG operator ah
a = BilinearForm(fes, symmetric=True, condense=True)
# volume term
a += (D * InnerProduct(gradu, gradv) + tau * u * v) * dx
# bdry terms
a += D * (-gradu * n * tang(v - vhat) - gradv * n * tang(u - uhat) +
alpha * tang(u - uhat) * tang(v - vhat)) * dx(
element_boundary=True)
######## face path blocks for smoother
def facePatchBlocks(mesh, fes):
blocks = []
freedofs = fes.FreeDofs(True)
for e in mesh.faces:
edofs = set()
# get ALL dofs connected to the face
for el in mesh[e].elements:
edofs |= set(d for d in fes.GetDofNrs(el) if freedofs[d])
blocks.append(edofs)
return blocks
def faceBlocks(mesh, fes):
blocks = []
freedofs = fes.FreeDofs(True)
for e in mesh.faces:
edofs = set(d for d in fes.GetDofNrs(e) if freedofs[d])
blocks.append(edofs)
return blocks
######## THIS IS MOST TIME CONSUMING PART
fBlocks = facePatchBlocks(mesh, fes)
fBlocks0 = faceBlocks(mesh, fes)
t0 = timeit.time()
# number of DOFS
ntotal = fes.ndof
nglobal = sum(fes.FreeDofs(True))
print("Elasped:%.2e BLOCKING Total DOFs: %.2e Global DOFs: %.2e " %
(t0 - t1, ntotal, nglobal))
class SymmetricGS(BaseMatrix):
def __init__(self, smoother):
super(SymmetricGS, self).__init__()
self.smoother = smoother
def Mult(self, x, y):
y[:] = 0.0
self.smoother.Smooth(y, x)
self.smoother.SmoothBack(y, x)
def Height(self):
return self.smoother.height
def Width(self):
return self.smoother.height
########### ASP operator ah0
a0 = BilinearForm(V0, symmetric=True)
a0 += (D * InnerProduct(Grad(u0), Grad(v0)) + tau * u0 * v0) * dx
# Projection operator V0--> fes
# We set up a mixed mass matrix for V0 -> fes and then solving with the mass matrix in M
mixmass = BilinearForm(trialspace=V0, testspace=fes)
# tangential part
mixmass += tang(u0) * tang(vhat) * dx(element_boundary=True)
# normal part
mixmass += (u0 * n) * (v * n) * dx(element_boundary=True)
massf = BilinearForm(fes)
massf += tang(uhat) * tang(vhat) * dx(element_boundary=True)
massf += (u * n) * (v * n) * dx(element_boundary=True)
with TaskManager():
f.Assemble()
a.Assemble()
t1 = timeit.time()
print("Elasped:%.2e ASSEMBLE " % (t1 - t0))
######### smoother (use edge blocks)
# smoother
jac = a.mat.CreateSmoother(fes.FreeDofs(True))
bjac = a.mat.CreateBlockSmoother(fBlocks)
######## ASP
a0.Assemble()
pm = ngs_petsc.PETScMatrix(a0.mat, V0.FreeDofs())
inva0 = ngs_petsc.PETSc2NGsPrecond(
pm, petsc_options={"pc_type": "hypre"})
massf.Assemble()
mixmass.Assemble()
# massf is block-diagonal in 3D!!!!!!!!!
m_inv = massf.mat.CreateBlockSmoother(fBlocks0)
E = m_inv @ mixmass.mat
ET = mixmass.mat.T @ m_inv
pre_twogrid = E @ inva0 @ ET
# jacobi smoother
pre1 = jac + pre_twogrid
# block gs smoother
pre2 = SymmetricGS(bjac) + pre_twogrid
t2 = timeit.time()
print("Elasped:%.2e PREC " % (t2 - t1))
inv1 = CGSolver(a.mat,
pre1,
maxsteps=10000,
precision=1e-10,
printrates=True)
inv2 = CGSolver(a.mat,
pre2,
maxsteps=10000,
precision=1e-10,
printrates=True)
f.vec.data += a.harmonic_extension_trans * f.vec
# solver
gfu.vec.data = inv1 * f.vec
gfu.vec.data = inv2 * f.vec
gfu.vec.data += a.harmonic_extension * gfu.vec
gfu.vec.data += a.inner_solve * f.vec
t3 = timeit.time()
print("Elasped:%.2e SOLVE " % (t3 - t2))
print("JAC:%i BGS: %i" % (inv1.GetSteps(), inv2.GetSteps()))
print("*****************************************************")
print("*****************************************************")
print("*****************************************************")
op=ParallelMatrix.C2C)
## The scalar preconditioner
us, vs = H1s.TnT()
h1s_blf = BilinearForm(H1s)
h1s_blf += beta * grad(us) * grad(vs) * dx
# Using NGsAMG
if use_ngs_amg:
pc_h1s = Preconditioner(h1s_blf, "ngs_amg.h1_scal")
h1s_blf.Assemble()
# Using GAMG algebraic multigrid from PETSc
if not use_ngs_amg:
pm_scal = ngs_petsc.PETScMatrix(h1s_blf.mat, H1s.FreeDofs())
pc_h1s = ngs_petsc.PETSc2NGsPrecond(pm_scal,
petsc_options={"pc_type": "gamg"})
pc_grad = G @ pc_h1s @ G.T
H1v = VectorH1(mesh, order=1, dirichlet="outer")
## On monday, we saw fast embedding via Dual shapes
## (see fast_embed.py)
from fast_embed import FastEmbed
E = FastEmbed(HC, H1v)
# Again, this is a C2C operation, because it takes a vector-valued
# H1 function (!) nad returns an HCurl function (!)
# solve Stokes problem for initial conditions:
import ngs_petsc as petsc
petsc.Initialize()
opts = {
"ksp_type": "gmres",
"ksp_atol": 1e-30,
"ksp_rtol": 1e-14,
"ksp_max_its": 1e6,
"ksp_monitor": "",
"ksp_converged_reason": "",
#"ksp_view" : "",
"pc_type": "none"
}
# pmat = petsc.FlatPETScMatrix(a.mat, X.FreeDofs())
pmat = petsc.PETScMatrix(a.mat, X.FreeDofs())
inv_stokes = petsc.KSP(mat=pmat,
name="ngs",
petsc_options=opts,
finalize=False)
fs_opts = {"pc_fieldsplit_detect_saddle_point": ""}
# fs_opts = {"pc_fieldsplit_detect_saddle_point" : "", "fieldsplit_1_pc_type" : "jacobi"}
# fs_opts = {"pc_fieldsplit_schur_fact_type" : "full", "fieldsplit_pfield_pc_type" : "bjacobi"}
# fs_opts = dict()
fs_opts = {
"pc_fieldsplit_type": "schur",
"pc_fieldsplit_schur_fact_type": "diag",
# "pc_fieldsplit_detect_saddle_point" : "",
"pc_fieldsplit_schur_precondition": "selfp",
"fieldsplit_0_pc_type": "jacobi",
V = H1(mesh, order=1, dirichlet='.*')
u, v = V.TnT()
a = BilinearForm(V)
a += SymbolicBFI(InnerProduct(grad(u), grad(v)))
f = LinearForm(V)
f += SymbolicLFI(v)
f.Assemble()
gfu = GridFunction(V)
# c = Preconditioner(a, 'bddc')
a.Assemble()
ngp.Initialize()
wrapped_mat = ngp.PETScMatrix(a.mat,
freedofs=V.FreeDofs(),
format=ngp.PETScMatrix.IS_AIJ)
# gives access to the petsc4py mat
p4p_mat = wrapped_mat.GetPETScMat()
if comm.size > 1:
p4p_mat.convert("mpiaij")
# this is a wrapper around a KSP
wrapped_ksp = ngp.KSP(mat=wrapped_mat,
name="someksp",
petsc_options={
"ksp_type": "cg",
"pc_type": "gamg"
},
finalize=False)
hcmass.Assemble()
mixmass = BilinearForm(trialspace=H1v, testspace=HC)
mixmass += uv * tau * dx
mixmass.Assemble()
hcm_inv = hcmass.mat.Inverse(HC.FreeDofs(), inverse="sparsecholesky")
E = hcm_inv @ mixmass.mat
## H1 scalar problem
h1scal_blf = BilinearForm(H1s)
h1scal_blf += beta * grad(us) * grad(vs) * dx
h1scal_blf.Assemble()
h1smat = petsc.PETScMatrix(h1scal_blf.mat, H1s.FreeDofs())
# pc_h1s = h1scal_blf.mat.Inverse(H1s.FreeDofs())
# pc_h1s = petsc.KSP(h1smat, "h1spc", petsc_options={ "ksp_type" : "cg",
# #"ksp_monitor" : "",
# #"ksp_converged_reason" : "",
# "pc_type" : "gamg"})
pc_h1s = petsc.PETSc2NGsPrecond(h1smat,
"h1spc",
petsc_options={"pc_type": "gamg"})
## H1 vector problem
h1v_blf = BilinearForm(H1v)
h1v_blf += alpha * InnerProduct(grad(uv), grad(vv)) * dx
h1v_blf.Assemble()
h1vmat = petsc.PETScMatrix(h1v_blf.mat, H1v.FreeDofs())