def __init__(self, space):
"""
Initialise limiter
:param space : FunctionSpace instance
"""
self.P1DG = space
self.P1CG = FunctionSpace(self.P1DG.mesh(), 'CG',
1) # for min/max limits
self.P0 = FunctionSpace(self.P1DG.mesh(), 'DG', 0) # for centroids
# Storage containers for cell means, max and mins
self.centroids = Function(self.P0)
self.centroids_rhs = Function(self.P0)
self.max_field = Function(self.P1CG)
self.min_field = Function(self.P1CG)
self.centroid_solver = self._construct_centroid_solver()
# Update min and max loop
self._min_max_loop = """
for(int i = 0; i < maxq.dofs; i++) {
maxq[i][0] = fmax(maxq[i][0],q[0][0]);
minq[i][0] = fmin(minq[i][0],q[0][0]);
}
"""
# Perform limiting loop
self._limit_kernel = """
def __init__(self, space):
"""
Initialise limiter
:param space : FunctionSpace instance
"""
if utils.complex_mode:
raise ValueError(
"We haven't decided what limiting complex valued fields means. Please get in touch if you have need."
)
self.P1DG = space
self.P1CG = FunctionSpace(self.P1DG.mesh(), 'CG',
1) # for min/max limits
self.P0 = FunctionSpace(self.P1DG.mesh(), 'DG', 0) # for centroids
# Storage containers for cell means, max and mins
self.centroids = Function(self.P0)
self.centroids_rhs = Function(self.P0)
self.max_field = Function(self.P1CG)
self.min_field = Function(self.P1CG)
self.centroid_solver = self._construct_centroid_solver()
# Update min and max loop
domain = "{[i]: 0 <= i < maxq.dofs}"
instructions = """
for i
maxq[i] = fmax(maxq[i], q[0])
minq[i] = fmin(minq[i], q[0])
end
"""
self._min_max_loop = (domain, instructions)
# Perform limiting loop
domain = "{[i, ii]: 0 <= i < q.dofs and 0 <= ii < q.dofs}"
instructions = """
<float64> alpha = 1
<float64> qavg = qbar[0, 0]
for i
<float64> _alpha1 = fmin(alpha, fmin(1, (qmax[i] - qavg)/(q[i] - qavg)))
<float64> _alpha2 = fmin(alpha, fmin(1, (qavg - qmin[i])/(qavg - q[i])))
alpha = _alpha1 if q[i] > qavg else (_alpha2 if q[i] < qavg else alpha)
end
for ii
q[ii] = qavg + alpha * (q[ii] - qavg)
end
"""
self._limit_kernel = (domain, instructions)
def __init__(self, equation):
"""
Initialise limiter
:param space : equation, as we need the broken space attached to it
"""
self.Vt = equation.space
# check this is the right space, only currently working for 2D extruded mesh
if self.Vt.extruded and self.Vt.mesh().topological_dimension() == 2:
# check that horizontal degree is 1 and vertical degree is 2
if self.Vt.ufl_element().degree()[0] is not 1 or \
self.Vt.ufl_element().degree()[1] is not 2:
raise ValueError('This is not the right limiter for this space.')
# check that continuity of the spaces is correct
# this will fail if the space does not use broken elements
if self.Vt.ufl_element()._element.sobolev_space()[0].name is not 'L2' or \
self.Vt.ufl_element()._element.sobolev_space()[1].name is not 'H1':
raise ValueError('This is not the right limiter for this space.')
else:
logger.warning('This limiter may not work for the space you are using.')
self.Q1DG = FunctionSpace(self.Vt.mesh(), 'DG', 1) # space with only vertex DOFs
self.vertex_limiter = VertexBasedLimiter(self.Q1DG)
self.theta_hat = Function(self.Q1DG) # theta function with only vertex DOFs
self.w = Function(self.Vt)
self.result = Function(self.Vt)
par_loop(_weight_kernel, dx, {"weight": (self.w, INC)})
def initialize(self, obj):
if complex_mode:
raise NotImplementedError("HypreAMS preconditioner not yet implemented in complex mode")
Citations().register("Kolev2009")
A, P = obj.getOperators()
prefix = obj.getOptionsPrefix()
V = get_function_space(obj.getDM())
mesh = V.mesh()
family = str(V.ufl_element().family())
degree = V.ufl_element().degree()
if family != 'Nedelec 1st kind H(curl)' or degree != 1:
raise ValueError("Hypre AMS requires lowest order Nedelec elements! (not %s of degree %d)" % (family, degree))
P1 = FunctionSpace(mesh, "Lagrange", 1)
G = Interpolator(grad(TestFunction(P1)), V).callable().handle
pc = PETSc.PC().create(comm=obj.comm)
pc.incrementTabLevel(1, parent=obj)
pc.setOptionsPrefix(prefix + "hypre_ams_")
pc.setOperators(A, P)
pc.setType('hypre')
pc.setHYPREType('ams')
pc.setHYPREDiscreteGradient(G)
zero_beta = PETSc.Options(prefix).getBool("pc_hypre_ams_zero_beta_poisson", default=False)
if zero_beta:
pc.setHYPRESetBetaPoissonMatrix(None)
VectorP1 = VectorFunctionSpace(mesh, "Lagrange", 1)
pc.setCoordinates(interpolate(SpatialCoordinate(mesh), VectorP1).dat.data_ro.copy())
pc.setUp()
self.pc = pc
def __init__(self, space):
"""
Initialise limiter
:param space : FunctionSpace instance
"""
self.P1DG = space
self.P1CG = FunctionSpace(self.P1DG.mesh(), 'CG',
1) # for min/max limits
self.P0 = FunctionSpace(self.P1DG.mesh(), 'DG', 0) # for centroids
# Storage containers for cell means, max and mins
self.centroids = Function(self.P0)
self.centroids_rhs = Function(self.P0)
self.max_field = Function(self.P1CG)
self.min_field = Function(self.P1CG)
self.centroid_solver = self._construct_centroid_solver()
# Update min and max loop
domain = "{[i]: 0 <= i < maxq.dofs}"
instructions = """
for i
maxq[i] = fmax(maxq[i], q[0])
minq[i] = fmin(minq[i], q[0])
end
"""
self._min_max_loop = (domain, instructions)
# Perform limiting loop
domain = "{[i, ii]: 0 <= i < q.dofs and 0 <= ii < q.dofs}"
instructions = """
<float64> alpha = 1
<float64> qavg = qbar[0, 0]
for i
<float64> _alpha1 = fmin(alpha, fmin(1, (qmax[i] - qavg)/(q[i] - qavg)))
<float64> _alpha2 = fmin(alpha, fmin(1, (qavg - qmin[i])/(qavg - q[i])))
alpha = if(q[i] > qavg, _alpha1, if(q[i] < qavg, _alpha2, alpha))
end
for ii
q[ii] = qavg + alpha * (q[ii] - qavg)
end
"""
self._limit_kernel = (domain, instructions)
def _split_arguments(self):
"""Splits the function space and stores the component
spaces determined by the indices.
"""
from firedrake.functionspace import FunctionSpace, MixedFunctionSpace
from firedrake.ufl_expr import Argument
tensor, = self.operands
nargs = []
for i, arg in enumerate(tensor.arguments()):
V = arg.function_space()
V_is = V.split()
idx = as_tuple(self._blocks[i])
if len(idx) == 1:
fidx, = idx
W = V_is[fidx]
W = FunctionSpace(W.mesh(), W.ufl_element())
else:
W = MixedFunctionSpace([V_is[fidx] for fidx in idx])
nargs.append(Argument(W, arg.number(), part=arg.part()))
return tuple(nargs)
def _split_arguments(self):
"""Splits the function space and stores the component
spaces determined by the indices.
"""
from firedrake.functionspace import FunctionSpace, MixedFunctionSpace
from firedrake.ufl_expr import Argument
tensor, = self.operands
nargs = []
for i, arg in enumerate(tensor.arguments()):
V = arg.function_space()
V_is = V.split()
idx = as_tuple(self._blocks[i])
if len(idx) == 1:
fidx, = idx
W = V_is[fidx]
W = FunctionSpace(W.mesh(), W.ufl_element())
else:
W = MixedFunctionSpace([V_is[fidx] for fidx in idx])
nargs.append(Argument(W, arg.number(), part=arg.part()))
return tuple(nargs)
def initialize(self, obj):
A, P = obj.getOperators()
prefix = obj.getOptionsPrefix()
V = get_function_space(obj.getDM())
mesh = V.mesh()
family = str(V.ufl_element().family())
degree = V.ufl_element().degree()
if family != 'Raviart-Thomas' or degree != 1:
raise ValueError(
"Hypre ADS requires lowest order RT elements! (not %s of degree %d)"
% (family, degree))
P1 = FunctionSpace(mesh, "Lagrange", 1)
NC1 = FunctionSpace(mesh, "N1curl", 1)
# DiscreteGradient
G = Interpolator(grad(TestFunction(P1)), NC1).callable().handle
# DiscreteCurl
C = Interpolator(curl(TestFunction(NC1)), V).callable().handle
pc = PETSc.PC().create(comm=obj.comm)
pc.incrementTabLevel(1, parent=obj)
pc.setOptionsPrefix(prefix + "hypre_ads_")
pc.setOperators(A, P)
pc.setType('hypre')
pc.setHYPREType('ads')
pc.setHYPREDiscreteGradient(G)
pc.setHYPREDiscreteCurl(C)
V = VectorFunctionSpace(mesh, "Lagrange", 1)
linear_coordinates = interpolate(SpatialCoordinate(mesh),
V).dat.data_ro.copy()
pc.setCoordinates(linear_coordinates)
pc.setUp()
self.pc = pc
def sort_entities(self, dm, axis, dir, ndiv):
# compute
# [(pStart, (x, y, z)), (pEnd, (x, y, z))]
mesh = dm.getAttr("__firedrake_mesh__")
ele = mesh.coordinates.function_space().ufl_element()
V = mesh.coordinates.function_space()
if V.finat_element.entity_dofs(
) == V.finat_element.entity_closure_dofs():
# We're using DG or DQ for our coordinates, so we got
# a periodic mesh. We need to interpolate to CGk
# with access descriptor MAX to define a consistent opinion
# about where the vertices are.
CGkele = ele.reconstruct(family="Lagrange")
# Need to supply the actual mesh to the FunctionSpace constructor,
# not its weakref proxy (the variable `mesh`)
# as interpolation fails because they are not hashable
CGk = FunctionSpace(mesh.coordinates.function_space().mesh(),
CGkele)
coordinates = interpolate(mesh.coordinates, CGk, access=op2.MAX)
else:
coordinates = mesh.coordinates
select = partial(select_entity, dm=dm, exclude="pyop2_ghost")
entities = [(p, self.coords(dm, p, coordinates))
for p in filter(select, range(*dm.getChart()))]
minx = min(entities, key=lambda z: z[1][axis])[1][axis]
maxx = max(entities, key=lambda z: z[1][axis])[1][axis]
def keyfunc(z):
coords = tuple(z[1])
return (coords[axis], ) + tuple(coords[:axis] + coords[axis + 1:])
s = sorted(entities, key=keyfunc, reverse=(dir == -1))
divisions = numpy.linspace(minx, maxx, ndiv + 1)
(entities, coords) = zip(*s)
coords = [c[axis] for c in coords]
indices = numpy.searchsorted(coords[::dir], divisions)
out = []
for k in range(ndiv):
out.append(entities[indices[k]:indices[k + 1]])
out.append(entities[indices[-1]:])
return out
def initialize(self, obj):
if complex_mode:
raise NotImplementedError(
"HypreAMS preconditioner not yet implemented in complex mode")
Citations().register("Kolev2009")
A, P = obj.getOperators()
prefix = obj.getOptionsPrefix()
V = get_function_space(obj.getDM())
mesh = V.mesh()
family = str(V.ufl_element().family())
degree = V.ufl_element().degree()
if family != 'Nedelec 1st kind H(curl)' or degree != 1:
raise ValueError(
"Hypre AMS requires lowest order Nedelec elements! (not %s of degree %d)"
% (family, degree))
P1 = FunctionSpace(mesh, "Lagrange", 1)
G = Interpolator(grad(TestFunction(P1)), V).callable().handle
pc = PETSc.PC().create(comm=obj.comm)
pc.incrementTabLevel(1, parent=obj)
pc.setOptionsPrefix(prefix + "hypre_ams_")
pc.setOperators(A, P)
pc.setType('hypre')
pc.setHYPREType('ams')
pc.setHYPREDiscreteGradient(G)
zero_beta = PETSc.Options(prefix).getBool(
"pc_hypre_ams_zero_beta_poisson", default=False)
if zero_beta:
pc.setHYPRESetBetaPoissonMatrix(None)
# Build constants basis for the Nedelec space
cvecs = []
for i in range(mesh.cell_dimension()):
direction = [
1.0 if i == j else 0.0 for j in range(mesh.cell_dimension())
]
c = project(Constant(direction), V)
with c.vector().dat.vec_ro as cvec:
cvecs.append(cvec)
pc.setHYPRESetEdgeConstantVectors(*cvecs)
pc.setUp()
self.pc = pc
def test_InterpolationMatrix_str():
"test the str method of InterpolationMatrix"
nx = 10
coords = np.array([[0.75], [0.5], [0.25], [0.1]])
nd = len(coords)
mesh = UnitIntervalMesh(nx)
V = FunctionSpace(mesh, "CG", 1)
im = InterpolationMatrix(V, coords)
assert str(
im
) == "Interpolation matrix from {} mesh points to {} data points".format(
nx + 1, nd)
def test_InterpolationMatrix_interp_mesh_to_data(fs, coords, meshcoords):
"test method to interpolate from distributed mesh to data gathered at root"
# simple 1D test
nd = len(coords)
im = InterpolationMatrix(fs, coords)
im.assemble()
input_ordered = np.array([3., 2., 7., 4., 0., 0., 2., 1., 1., 1., 5.])
f = Function(fs).vector()
meshcoords_ordered = np.linspace(0., 1., 11)
with f.dat.vec as vec:
imin, imax = vec.getOwnershipRange()
for i in range(imin, imax):
vec.setValue(
i,
input_ordered[np.where(meshcoords_ordered == meshcoords[i])])
if COMM_WORLD.rank == 0:
expected = np.array([1., 0., 5.5, 3.25])
else:
expected = np.zeros(0)
out = im.interp_mesh_to_data(f)
assert_allclose(out, expected, atol=1.e-10)
# failure due to bad input sizes
mesh2 = UnitIntervalMesh(12)
V2 = FunctionSpace(mesh2, "CG", 1)
f2 = Function(V2).vector()
f2.set_local(np.ones(f2.local_size()))
with pytest.raises(AssertionError):
im.interp_mesh_to_data(f2)
im.destroy()
def initialize(self, pc):
from firedrake.assemble import allocate_matrix, create_assembly_callable
_, P = pc.getOperators()
if pc.getType() != "python":
raise ValueError("Expecting PC type python")
opc = pc
appctx = self.get_appctx(pc)
fcp = appctx.get("form_compiler_parameters")
V = get_function_space(pc.getDM())
if len(V) == 1:
V = FunctionSpace(V.mesh(), V.ufl_element())
else:
V = MixedFunctionSpace([V_ for V_ in V])
test = TestFunction(V)
trial = TrialFunction(V)
if P.type == "python":
context = P.getPythonContext()
# It only makes sense to preconditioner/invert a diagonal
# block in general. That's all we're going to allow.
if not context.on_diag:
raise ValueError("Only makes sense to invert diagonal block")
prefix = pc.getOptionsPrefix()
options_prefix = prefix + self._prefix
mat_type = PETSc.Options().getString(options_prefix + "mat_type", "aij")
(a, bcs) = self.form(pc, test, trial)
self.P = allocate_matrix(a, bcs=bcs,
form_compiler_parameters=fcp,
mat_type=mat_type,
options_prefix=options_prefix)
self._assemble_P = create_assembly_callable(a, tensor=self.P,
bcs=bcs,
form_compiler_parameters=fcp,
mat_type=mat_type)
self._assemble_P()
self.P.force_evaluation()
# Transfer nullspace over
Pmat = self.P.petscmat
Pmat.setNullSpace(P.getNullSpace())
tnullsp = P.getTransposeNullSpace()
if tnullsp.handle != 0:
Pmat.setTransposeNullSpace(tnullsp)
# Internally, we just set up a PC object that the user can configure
# however from the PETSc command line. Since PC allows the user to specify
# a KSP, we can do iterative by -assembled_pc_type ksp.
pc = PETSc.PC().create(comm=opc.comm)
pc.incrementTabLevel(1, parent=opc)
# We set a DM and an appropriate SNESContext on the constructed PC so one
# can do e.g. multigrid or patch solves.
from firedrake.variational_solver import NonlinearVariationalProblem
from firedrake.solving_utils import _SNESContext
dm = opc.getDM()
octx = get_appctx(dm)
oproblem = octx._problem
nproblem = NonlinearVariationalProblem(oproblem.F, oproblem.u, bcs, J=a, form_compiler_parameters=fcp)
nctx = _SNESContext(nproblem, mat_type, mat_type, octx.appctx)
push_appctx(dm, nctx)
self._ctx_ref = nctx
pc.setDM(dm)
pc.setOptionsPrefix(options_prefix)
pc.setOperators(Pmat, Pmat)
pc.setFromOptions()
pc.setUp()
self.pc = pc
pop_appctx(dm)
def assemble_mixed_mass_matrix(V_A, V_B):
"""
Construct the mixed mass matrix of two function spaces,
using the TrialFunction from V_A and the TestFunction
from V_B.
"""
if len(V_A) > 1 or len(V_B) > 1:
raise NotImplementedError(
"Sorry, only implemented for non-mixed spaces")
if V_A.ufl_element().mapping() != "identity" or V_B.ufl_element().mapping(
) != "identity":
msg = """
Sorry, only implemented for affine maps for now. To do non-affine, we'd need to
import much more of the assembly engine of UFL/TSFC/etc to do the assembly on
each supermesh cell.
"""
raise NotImplementedError(msg)
mesh_A = V_A.mesh()
mesh_B = V_B.mesh()
dim = mesh_A.geometric_dimension()
assert dim == mesh_B.geometric_dimension()
assert dim == mesh_A.topological_dimension()
assert dim == mesh_B.topological_dimension()
(mh_A, level_A) = get_level(mesh_A)
(mh_B, level_B) = get_level(mesh_B)
if mesh_A is mesh_B:
def likely(cell_A):
return [cell_A]
else:
if (mh_A is None or mh_B is None) or (mh_A is not mh_B):
# No mesh hierarchy structure, call libsupermesh for
# intersection finding
intersections = intersection_finder(mesh_A, mesh_B)
likely = intersections.__getitem__
else:
# We do have a mesh hierarchy, use it
if abs(level_A - level_B) > 1:
raise NotImplementedError(
"Only works for transferring between adjacent levels for now."
)
# What are the cells of B that (probably) intersect with a given cell in A?
if level_A > level_B:
cell_map = mh_A.fine_to_coarse_cells[level_A]
def likely(cell_A):
return cell_map[cell_A]
elif level_A < level_B:
cell_map = mh_A.coarse_to_fine_cells[level_A]
def likely(cell_A):
return cell_map[cell_A]
assert V_A.value_size == V_B.value_size
orig_value_size = V_A.value_size
if V_A.value_size > 1:
V_A = firedrake.FunctionSpace(mesh_A,
V_A.ufl_element().sub_elements()[0])
if V_B.value_size > 1:
V_B = firedrake.FunctionSpace(mesh_B,
V_B.ufl_element().sub_elements()[0])
assert V_A.value_size == 1
assert V_B.value_size == 1
preallocator = PETSc.Mat().create(comm=mesh_A.comm)
preallocator.setType(PETSc.Mat.Type.PREALLOCATOR)
rset = V_B.dof_dset
cset = V_A.dof_dset
nrows = rset.layout_vec.getSizes()
ncols = cset.layout_vec.getSizes()
preallocator.setLGMap(rmap=rset.scalar_lgmap, cmap=cset.scalar_lgmap)
preallocator.setSizes(size=(nrows, ncols), bsize=1)
preallocator.setUp()
zeros = numpy.zeros((V_B.cell_node_map().arity, V_A.cell_node_map().arity),
dtype=ScalarType)
for cell_A, dofs_A in enumerate(V_A.cell_node_map().values):
for cell_B in likely(cell_A):
dofs_B = V_B.cell_node_map().values_with_halo[cell_B, :]
preallocator.setValuesLocal(dofs_B, dofs_A, zeros)
preallocator.assemble()
dnnz, onnz = get_preallocation(preallocator, nrows[0])
# Unroll from block to AIJ
dnnz = dnnz * cset.cdim
dnnz = numpy.repeat(dnnz, rset.cdim)
onnz = onnz * cset.cdim
onnz = numpy.repeat(onnz, cset.cdim)
preallocator.destroy()
assert V_A.value_size == V_B.value_size
rdim = V_B.dof_dset.cdim
cdim = V_A.dof_dset.cdim
#
# Preallocate M_AB.
#
mat = PETSc.Mat().create(comm=mesh_A.comm)
mat.setType(PETSc.Mat.Type.AIJ)
rsizes = tuple(n * rdim for n in nrows)
csizes = tuple(c * cdim for c in ncols)
mat.setSizes(size=(rsizes, csizes), bsize=(rdim, cdim))
mat.setPreallocationNNZ((dnnz, onnz))
mat.setLGMap(rmap=rset.lgmap, cmap=cset.lgmap)
# TODO: Boundary conditions not handled.
mat.setOption(mat.Option.IGNORE_OFF_PROC_ENTRIES, False)
mat.setOption(mat.Option.NEW_NONZERO_ALLOCATION_ERR, True)
mat.setOption(mat.Option.KEEP_NONZERO_PATTERN, True)
mat.setOption(mat.Option.UNUSED_NONZERO_LOCATION_ERR, False)
mat.setOption(mat.Option.IGNORE_ZERO_ENTRIES, True)
mat.setUp()
evaluate_kernel_A = compile_element(ufl.Coefficient(V_A),
name="evaluate_kernel_A")
evaluate_kernel_B = compile_element(ufl.Coefficient(V_B),
name="evaluate_kernel_B")
# We only need one of these since we assume that the two meshes both have CG1 coordinates
to_reference_kernel = to_reference_coordinates(
mesh_A.coordinates.ufl_element())
if dim == 2:
reference_mesh = UnitTriangleMesh(comm=COMM_SELF)
else:
reference_mesh = UnitTetrahedronMesh(comm=COMM_SELF)
evaluate_kernel_S = compile_element(ufl.Coefficient(
reference_mesh.coordinates.function_space()),
name="evaluate_kernel_S")
V_S_A = FunctionSpace(reference_mesh, V_A.ufl_element())
V_S_B = FunctionSpace(reference_mesh, V_B.ufl_element())
M_SS = assemble(inner(TrialFunction(V_S_A), TestFunction(V_S_B)) * dx)
M_SS = M_SS.M.handle[:, :]
node_locations_A = utils.physical_node_locations(
V_S_A).dat.data_ro_with_halos
node_locations_B = utils.physical_node_locations(
V_S_B).dat.data_ro_with_halos
num_nodes_A = node_locations_A.shape[0]
num_nodes_B = node_locations_B.shape[0]
to_reference_kernel = to_reference_coordinates(
mesh_A.coordinates.ufl_element())
supermesh_kernel_str = """
#include "libsupermesh-c.h"
#include <petsc.h>
%(to_reference)s
%(evaluate_S)s
%(evaluate_A)s
%(evaluate_B)s
#define complex_mode %(complex_mode)s
#define PrintInfo(...) do { if (PetscLogPrintInfo) printf(__VA_ARGS__); } while (0)
static void print_array(PetscScalar *arr, int d)
{
for(int j=0; j<d; j++)
PrintInfo(stderr, "%%+.2f ", arr[j]);
}
static void print_coordinates(PetscScalar *simplex, int d)
{
for(int i=0; i<d+1; i++)
{
PrintInfo("\t");
print_array(&simplex[d*i], d);
PrintInfo("\\n");
}
}
#if complex_mode
static void seperate_real_and_imag(PetscScalar *simplex, double *real_simplex, double *imag_simplex, int d)
{
for(int i=0; i<d+1; i++)
{
for(int j=0; j<d; j++)
{
real_simplex[d*i+j] = creal(simplex[d*i+j]);
imag_simplex[d*i+j] = cimag(simplex[d*i+j]);
}
}
}
static void merge_back_to_simplex(PetscScalar* simplex, double* real_simplex, double* imag_simplex, int d)
{
print_coordinates(simplex,d);
for(int i=0; i<d+1; i++)
{
for(int j=0; j<d; j++)
{
simplex[d*i+j] = real_simplex[d*i+j]+imag_simplex[d*i+j]*_Complex_I;
}
}
}
#endif
int supermesh_kernel(PetscScalar* simplex_A, PetscScalar* simplex_B, PetscScalar* simplices_C, PetscScalar* nodes_A, PetscScalar* nodes_B, PetscScalar* M_SS, PetscScalar* outptr, int num_ele)
{
#define d %(dim)s
#define num_nodes_A %(num_nodes_A)s
#define num_nodes_B %(num_nodes_B)s
double simplex_ref_measure;
PrintInfo("simplex_A coordinates\\n");
print_coordinates(simplex_A, d);
PrintInfo("simplex_B coordinates\\n");
print_coordinates(simplex_B, d);
int num_elements = num_ele;
if (d == 2) simplex_ref_measure = 0.5;
else if (d == 3) simplex_ref_measure = 1.0/6;
PetscScalar R_AS[num_nodes_A][num_nodes_A];
PetscScalar R_BS[num_nodes_B][num_nodes_B];
PetscScalar coeffs_A[%(num_nodes_A)s] = {0.};
PetscScalar coeffs_B[%(num_nodes_B)s] = {0.};
PetscScalar reference_nodes_A[num_nodes_A][d];
PetscScalar reference_nodes_B[num_nodes_B][d];
#if complex_mode
double real_simplex_A[d*(d+1)];
double imag_simplex_A[d*(d+1)];
seperate_real_and_imag(simplex_A, real_simplex_A, imag_simplex_A, d);
double real_simplex_B[d*(d+1)];
double imag_simplex_B[d*(d+1)];
seperate_real_and_imag(simplex_B, real_simplex_B, imag_simplex_B, d);
double real_simplices_C[num_elements*d*(d+1)];
double imag_simplices_C[num_elements*d*(d+1)];
for (int ii=0; ii<num_elements*d*(d+1); ++ii) imag_simplices_C[ii] = 0.;
%(libsupermesh_intersect_simplices)s(real_simplex_A, real_simplex_B, real_simplices_C, &num_elements);
merge_back_to_simplex(simplex_A, real_simplex_A, imag_simplex_A, d);
merge_back_to_simplex(simplex_B, real_simplex_B, imag_simplex_B, d);
for(int s=0; s<num_elements; s++)
{
PetscScalar* simplex_C = &simplices_C[s * d * (d+1)];
double* real_simplex_C = &real_simplices_C[s * d * (d+1)];
double* imag_simplex_C = &imag_simplices_C[s * d * (d+1)];
merge_back_to_simplex(simplex_C, real_simplex_C, imag_simplex_C, d);
}
#else
%(libsupermesh_intersect_simplices)s(simplex_A, simplex_B, simplices_C, &num_elements);
#endif
PrintInfo("Supermesh consists of %%i elements\\n", num_elements);
// would like to do this
//PetscScalar MAB[%(num_nodes_A)s][%(num_nodes_B)s] = (PetscScalar (*)[%(num_nodes_B)s])outptr;
// but have to do this instead because we don't grok C
PetscScalar (*MAB)[num_nodes_A] = (PetscScalar (*)[num_nodes_A])outptr;
PetscScalar (*MSS)[num_nodes_A] = (PetscScalar (*)[num_nodes_A])M_SS; // note the underscore
for ( int i = 0; i < num_nodes_B; i++ ) {
for (int j = 0; j < num_nodes_A; j++) {
MAB[i][j] = 0.0;
}
}
for(int s=0; s<num_elements; s++)
{
PetscScalar* simplex_S = &simplices_C[s * d * (d+1)];
double simplex_S_measure;
#if complex_mode
double real_simplex_S[d*(d+1)];
double imag_simplex_S[d*(d+1)];
seperate_real_and_imag(simplex_S, real_simplex_S, imag_simplex_S, d);
%(libsupermesh_simplex_measure)s(real_simplex_S, &simplex_S_measure);
merge_back_to_simplex(simplex_S, real_simplex_S, imag_simplex_S, d);
#else
%(libsupermesh_simplex_measure)s(simplex_S, &simplex_S_measure);
#endif
PrintInfo("simplex_S coordinates with measure %%f\\n", simplex_S_measure);
print_coordinates(simplex_S, d);
PrintInfo("Start mapping nodes for V_A\\n");
PetscScalar physical_nodes_A[num_nodes_A][d];
for(int n=0; n < num_nodes_A; n++) {
PetscScalar* reference_node_location = &nodes_A[n*d];
PetscScalar* physical_node_location = physical_nodes_A[n];
for (int j=0; j < d; j++) physical_node_location[j] = 0.0;
pyop2_kernel_evaluate_kernel_S(physical_node_location, simplex_S, reference_node_location);
PrintInfo("\\tNode ");
print_array(reference_node_location, d);
PrintInfo(" mapped to ");
print_array(physical_node_location, d);
PrintInfo("\\n");
}
PrintInfo("Start mapping nodes for V_B\\n");
PetscScalar physical_nodes_B[num_nodes_B][d];
for(int n=0; n < num_nodes_B; n++) {
PetscScalar* reference_node_location = &nodes_B[n*d];
PetscScalar* physical_node_location = physical_nodes_B[n];
for (int j=0; j < d; j++) physical_node_location[j] = 0.0;
pyop2_kernel_evaluate_kernel_S(physical_node_location, simplex_S, reference_node_location);
PrintInfo("\\tNode ");
print_array(reference_node_location, d);
PrintInfo(" mapped to ");
print_array(physical_node_location, d);
PrintInfo("\\n");
}
PrintInfo("==========================================================\\n");
PrintInfo("Start pulling back dof from S into reference space for A.\\n");
for(int n=0; n < num_nodes_A; n++) {
for(int i=0; i<d; i++) reference_nodes_A[n][i] = 0.;
to_reference_coords_kernel(reference_nodes_A[n], physical_nodes_A[n], simplex_A);
PrintInfo("Pulling back ");
print_array(physical_nodes_A[n], d);
PrintInfo(" to ");
print_array(reference_nodes_A[n], d);
PrintInfo("\\n");
}
PrintInfo("Start pulling back dof from S into reference space for B.\\n");
for(int n=0; n < num_nodes_B; n++) {
for(int i=0; i<d; i++) reference_nodes_B[n][i] = 0.;
to_reference_coords_kernel(reference_nodes_B[n], physical_nodes_B[n], simplex_B);
PrintInfo("Pulling back ");
print_array(physical_nodes_B[n], d);
PrintInfo(" to ");
print_array(reference_nodes_B[n], d);
PrintInfo("\\n");
}
PrintInfo("Start evaluating basis functions of V_A at dofs for V_A on S\\n");
for(int i=0; i<num_nodes_A; i++) {
coeffs_A[i] = 1.;
for(int j=0; j<num_nodes_A; j++) {
R_AS[i][j] = 0.;
pyop2_kernel_evaluate_kernel_A(&R_AS[i][j], coeffs_A, reference_nodes_A[j]);
}
print_array(R_AS[i], num_nodes_A);
PrintInfo("\\n");
coeffs_A[i] = 0.;
}
PrintInfo("Start evaluating basis functions of V_B at dofs for V_B on S\\n");
for(int i=0; i<num_nodes_B; i++) {
coeffs_B[i] = 1.;
for(int j=0; j<num_nodes_B; j++) {
R_BS[i][j] = 0.;
pyop2_kernel_evaluate_kernel_B(&R_BS[i][j], coeffs_B, reference_nodes_B[j]);
}
print_array(R_BS[i], num_nodes_B);
PrintInfo("\\n");
coeffs_B[i] = 0.;
}
PrintInfo("Start doing the matmatmat mult\\n");
for ( int i = 0; i < num_nodes_B; i++ ) {
for (int j = 0; j < num_nodes_A; j++) {
for ( int k = 0; k < num_nodes_B; k++) {
for ( int l = 0; l < num_nodes_A; l++) {
MAB[i][j] += (simplex_S_measure/simplex_ref_measure) * R_BS[i][k] * MSS[k][l] * R_AS[j][l];
}
}
}
}
}
return num_elements;
}
""" % {
"evaluate_S":
str(evaluate_kernel_S),
"evaluate_A":
str(evaluate_kernel_A),
"evaluate_B":
str(evaluate_kernel_B),
"to_reference":
str(to_reference_kernel),
"num_nodes_A":
num_nodes_A,
"num_nodes_B":
num_nodes_B,
"libsupermesh_simplex_measure":
"libsupermesh_triangle_area"
if dim == 2 else "libsupermesh_tetrahedron_volume",
"libsupermesh_intersect_simplices":
"libsupermesh_intersect_tris_real"
if dim == 2 else "libsupermesh_intersect_tets_real",
"dim":
dim,
"complex_mode":
1 if complex_mode else 0
}
dirs = get_petsc_dir() + (sys.prefix, )
includes = ["-I%s/include" % d for d in dirs]
libs = ["-L%s/lib" % d for d in dirs]
libs = libs + ["-Wl,-rpath,%s/lib" % d
for d in dirs] + ["-lpetsc", "-lsupermesh"]
lib = load(supermesh_kernel_str,
"c",
"supermesh_kernel",
cppargs=includes,
ldargs=libs,
argtypes=[
ctypes.c_voidp, ctypes.c_voidp, ctypes.c_voidp,
ctypes.c_voidp, ctypes.c_voidp, ctypes.c_voidp,
ctypes.c_voidp
],
restype=ctypes.c_int)
ammm(V_A, V_B, likely, node_locations_A, node_locations_B, M_SS,
ctypes.addressof(lib), mat)
if orig_value_size == 1:
return mat
else:
(lrows, grows), (lcols, gcols) = mat.getSizes()
lrows *= orig_value_size
grows *= orig_value_size
lcols *= orig_value_size
gcols *= orig_value_size
size = ((lrows, grows), (lcols, gcols))
context = BlockMatrix(mat, orig_value_size)
blockmat = PETSc.Mat().createPython(size,
context=context,
comm=mat.comm)
blockmat.setUp()
return blockmat
def initialize(self, pc):
from firedrake.assemble import allocate_matrix, create_assembly_callable
_, P = pc.getOperators()
if pc.getType() != "python":
raise ValueError("Expecting PC type python")
opc = pc
appctx = self.get_appctx(pc)
fcp = appctx.get("form_compiler_parameters")
V = get_function_space(pc.getDM())
if len(V) == 1:
V = FunctionSpace(V.mesh(), V.ufl_element())
else:
V = MixedFunctionSpace([V_ for V_ in V])
test = TestFunction(V)
trial = TrialFunction(V)
if P.type == "python":
context = P.getPythonContext()
# It only makes sense to preconditioner/invert a diagonal
# block in general. That's all we're going to allow.
if not context.on_diag:
raise ValueError("Only makes sense to invert diagonal block")
prefix = pc.getOptionsPrefix()
options_prefix = prefix + self._prefix
mat_type = PETSc.Options().getString(options_prefix + "mat_type", "aij")
(a, bcs) = self.form(pc, test, trial)
self.P = allocate_matrix(a, bcs=bcs,
form_compiler_parameters=fcp,
mat_type=mat_type,
options_prefix=options_prefix)
self._assemble_P = create_assembly_callable(a, tensor=self.P,
bcs=bcs,
form_compiler_parameters=fcp,
mat_type=mat_type)
self._assemble_P()
self.P.force_evaluation()
# Transfer nullspace over
Pmat = self.P.petscmat
Pmat.setNullSpace(P.getNullSpace())
tnullsp = P.getTransposeNullSpace()
if tnullsp.handle != 0:
Pmat.setTransposeNullSpace(tnullsp)
# Internally, we just set up a PC object that the user can configure
# however from the PETSc command line. Since PC allows the user to specify
# a KSP, we can do iterative by -assembled_pc_type ksp.
pc = PETSc.PC().create(comm=opc.comm)
pc.incrementTabLevel(1, parent=opc)
# We set a DM and an appropriate SNESContext on the constructed PC so one
# can do e.g. multigrid or patch solves.
from firedrake.variational_solver import NonlinearVariationalProblem
from firedrake.solving_utils import _SNESContext
dm = opc.getDM()
octx = get_appctx(dm)
oproblem = octx._problem
nproblem = NonlinearVariationalProblem(oproblem.F, oproblem.u, bcs, J=a, form_compiler_parameters=fcp)
nctx = _SNESContext(nproblem, mat_type, mat_type, octx.appctx)
push_appctx(dm, nctx)
pc.setDM(dm)
pc.setOptionsPrefix(options_prefix)
pc.setOperators(Pmat, Pmat)
pc.setFromOptions()
pc.setUp()
self.pc = pc
pop_appctx(dm)
def _ad_function_space(self, mesh):
element = self.ufl_element()
fs_element = element.reconstruct(cell=mesh.ufl_cell())
return FunctionSpace(mesh, fs_element)
def initialize(self, pc):
"""Set up the problem context. This takes the incoming
three-field hybridized system and constructs the static
condensation operators using Slate expressions.
A KSP is created for the reduced system for the Lagrange
multipliers. The scalar and flux fields are reconstructed
locally.
"""
from firedrake.assemble import (allocate_matrix,
create_assembly_callable)
from firedrake.bcs import DirichletBC
from firedrake.function import Function
from firedrake.functionspace import FunctionSpace
from firedrake.interpolation import interpolate
prefix = pc.getOptionsPrefix() + "hybrid_sc_"
_, P = pc.getOperators()
self.cxt = P.getPythonContext()
if not isinstance(self.cxt, ImplicitMatrixContext):
raise ValueError("Context must be an ImplicitMatrixContext")
# Retrieve the mixed function space, which is expected to
# be of the form: W = (DG_k)^n \times DG_k \times DG_trace
W = self.cxt.a.arguments()[0].function_space()
if len(W) != 3:
raise RuntimeError("Expecting three function spaces.")
# Assert a specific ordering of the spaces
# TODO: Clean this up
assert W[2].ufl_element().family() == "HDiv Trace"
# Extract trace space
T = W[2]
# Need to duplicate a trace space which is NOT
# associated with a subspace of a mixed space.
Tr = FunctionSpace(T.mesh(), T.ufl_element())
bcs = []
cxt_bcs = self.cxt.row_bcs
for bc in cxt_bcs:
assert bc.function_space() == T, (
"BCs should be imposing vanishing conditions on traces")
if isinstance(bc.function_arg, Function):
bc_arg = interpolate(bc.function_arg, Tr)
else:
# Constants don't need to be interpolated
bc_arg = bc.function_arg
bcs.append(DirichletBC(Tr, bc_arg, bc.sub_domain))
mat_type = PETSc.Options().getString(prefix + "mat_type", "aij")
self.r_lambda = Function(T)
self.residual = Function(W)
self.solution = Function(W)
# Perform symbolics only once
S_expr, r_lambda_expr, u_h_expr, q_h_expr = self._slate_expressions
self.S = allocate_matrix(S_expr,
bcs=bcs,
form_compiler_parameters=self.cxt.fc_params,
mat_type=mat_type)
self._assemble_S = create_assembly_callable(
S_expr,
tensor=self.S,
bcs=bcs,
form_compiler_parameters=self.cxt.fc_params,
mat_type=mat_type)
self._assemble_S()
Smat = self.S.petscmat
# Set up ksp for the trace problem
trace_ksp = PETSc.KSP().create(comm=pc.comm)
trace_ksp.incrementTabLevel(1, parent=pc)
trace_ksp.setOptionsPrefix(prefix)
trace_ksp.setOperators(Smat)
trace_ksp.setUp()
trace_ksp.setFromOptions()
self.trace_ksp = trace_ksp
self._assemble_Srhs = create_assembly_callable(
r_lambda_expr,
tensor=self.r_lambda,
form_compiler_parameters=self.cxt.fc_params)
q_h, u_h, lambda_h = self.solution.split()
# Assemble u_h using lambda_h
self._assemble_u = create_assembly_callable(
u_h_expr, tensor=u_h, form_compiler_parameters=self.cxt.fc_params)
# Recover q_h using both u_h and lambda_h
self._assemble_q = create_assembly_callable(
q_h_expr, tensor=q_h, form_compiler_parameters=self.cxt.fc_params)
def initialize(self, pc):
"""Set up the problem context. This takes the incoming
three-field system and constructs the static
condensation operators using Slate expressions.
A KSP is created for the reduced system. The eliminated
variables are recovered via back-substitution.
"""
from firedrake.assemble import (allocate_matrix,
create_assembly_callable)
from firedrake.bcs import DirichletBC
from firedrake.function import Function
from firedrake.functionspace import FunctionSpace
from firedrake.interpolation import interpolate
prefix = pc.getOptionsPrefix() + "condensed_field_"
A, P = pc.getOperators()
self.cxt = A.getPythonContext()
if not isinstance(self.cxt, ImplicitMatrixContext):
raise ValueError("Context must be an ImplicitMatrixContext")
self.bilinear_form = self.cxt.a
# Retrieve the mixed function space
W = self.bilinear_form.arguments()[0].function_space()
if len(W) > 3:
raise NotImplementedError("Only supports up to three function spaces.")
elim_fields = PETSc.Options().getString(pc.getOptionsPrefix()
+ "pc_sc_eliminate_fields",
None)
if elim_fields:
elim_fields = [int(i) for i in elim_fields.split(',')]
else:
# By default, we condense down to the last field in the
# mixed space.
elim_fields = [i for i in range(0, len(W) - 1)]
condensed_fields = list(set(range(len(W))) - set(elim_fields))
if len(condensed_fields) != 1:
raise NotImplementedError("Cannot condense to more than one field")
c_field, = condensed_fields
# Need to duplicate a space which is NOT
# associated with a subspace of a mixed space.
Vc = FunctionSpace(W.mesh(), W[c_field].ufl_element())
bcs = []
cxt_bcs = self.cxt.row_bcs
for bc in cxt_bcs:
if bc.function_space().index != c_field:
raise NotImplementedError("Strong BC set on unsupported space")
if isinstance(bc.function_arg, Function):
bc_arg = interpolate(bc.function_arg, Vc)
else:
# Constants don't need to be interpolated
bc_arg = bc.function_arg
bcs.append(DirichletBC(Vc, bc_arg, bc.sub_domain))
mat_type = PETSc.Options().getString(prefix + "mat_type", "aij")
self.c_field = c_field
self.condensed_rhs = Function(Vc)
self.residual = Function(W)
self.solution = Function(W)
# Get expressions for the condensed linear system
A_tensor = Tensor(self.bilinear_form)
reduced_sys = self.condensed_system(A_tensor, self.residual, elim_fields)
S_expr = reduced_sys.lhs
r_expr = reduced_sys.rhs
# Construct the condensed right-hand side
self._assemble_Srhs = create_assembly_callable(
r_expr,
tensor=self.condensed_rhs,
form_compiler_parameters=self.cxt.fc_params)
# Allocate and set the condensed operator
self.S = allocate_matrix(S_expr,
bcs=bcs,
form_compiler_parameters=self.cxt.fc_params,
mat_type=mat_type,
options_prefix=prefix,
appctx=self.get_appctx(pc))
self._assemble_S = create_assembly_callable(
S_expr,
tensor=self.S,
bcs=bcs,
form_compiler_parameters=self.cxt.fc_params,
mat_type=mat_type)
self._assemble_S()
Smat = self.S.petscmat
# If a different matrix is used for preconditioning,
# assemble this as well
if A != P:
self.cxt_pc = P.getPythonContext()
P_tensor = Tensor(self.cxt_pc.a)
P_reduced_sys = self.condensed_system(P_tensor,
self.residual,
elim_fields)
S_pc_expr = P_reduced_sys.lhs
self.S_pc_expr = S_pc_expr
# Allocate and set the condensed operator
self.S_pc = allocate_matrix(S_expr,
bcs=bcs,
form_compiler_parameters=self.cxt.fc_params,
mat_type=mat_type,
options_prefix=prefix,
appctx=self.get_appctx(pc))
self._assemble_S_pc = create_assembly_callable(
S_pc_expr,
tensor=self.S_pc,
bcs=bcs,
form_compiler_parameters=self.cxt.fc_params,
mat_type=mat_type)
self._assemble_S_pc()
Smat_pc = self.S_pc.petscmat
else:
self.S_pc_expr = S_expr
Smat_pc = Smat
# Get nullspace for the condensed operator (if any).
# This is provided as a user-specified callback which
# returns the basis for the nullspace.
nullspace = self.cxt.appctx.get("condensed_field_nullspace", None)
if nullspace is not None:
nsp = nullspace(Vc)
Smat.setNullSpace(nsp.nullspace(comm=pc.comm))
# Create a SNESContext for the DM associated with the trace problem
self._ctx_ref = self.new_snes_ctx(pc,
S_expr,
bcs,
mat_type,
self.cxt.fc_params,
options_prefix=prefix)
# Push new context onto the dm associated with the condensed problem
c_dm = Vc.dm
# Set up ksp for the condensed problem
c_ksp = PETSc.KSP().create(comm=pc.comm)
c_ksp.incrementTabLevel(1, parent=pc)
# Set the dm for the condensed solver
c_ksp.setDM(c_dm)
c_ksp.setDMActive(False)
c_ksp.setOptionsPrefix(prefix)
c_ksp.setOperators(A=Smat, P=Smat_pc)
self.condensed_ksp = c_ksp
with dmhooks.add_hooks(c_dm, self,
appctx=self._ctx_ref,
save=False):
c_ksp.setFromOptions()
# Set up local solvers for backwards substitution
self.local_solvers = self.local_solver_calls(A_tensor,
self.residual,
self.solution,
elim_fields)
