def simple_mpi_communication_entrypoint():
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue)
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
from meshmode.mesh import BTAG_ALL
from mpi4py import MPI
comm = MPI.COMM_WORLD
num_parts = comm.Get_size()
mesh_dist = MPIMeshDistributor(comm)
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-1, ) * 2,
b=(1, ) * 2,
nelements_per_axis=(2, ) * 2)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element,
num_parts)
else:
local_mesh = mesh_dist.receive_mesh_part()
vol_discr = DiscretizationCollection(actx,
local_mesh,
order=5,
mpi_communicator=comm)
sym_x = sym.nodes(local_mesh.dim)
myfunc_symb = sym.sin(np.dot(sym_x, [2, 3]))
myfunc = bind(vol_discr, myfunc_symb)(actx)
sym_all_faces_func = sym.cse(
sym.project("vol", "all_faces")(sym.var("myfunc")))
sym_int_faces_func = sym.cse(
sym.project("vol", "int_faces")(sym.var("myfunc")))
sym_bdry_faces_func = sym.cse(
sym.project(BTAG_ALL,
"all_faces")(sym.project("vol",
BTAG_ALL)(sym.var("myfunc"))))
bound_face_swap = bind(
vol_discr,
sym.project("int_faces", "all_faces")(
sym.OppositeInteriorFaceSwap("int_faces")(sym_int_faces_func)) -
(sym_all_faces_func - sym_bdry_faces_func))
hopefully_zero = bound_face_swap(myfunc=myfunc)
error = actx.np.linalg.norm(hopefully_zero, ord=np.inf)
print(__file__)
with np.printoptions(threshold=100000000, suppress=True):
logger.debug(hopefully_zero)
logger.info("error: %.5e", error)
assert error < 1e-14
def absorbing_bc(self, w):
"""Construct part of the flux operator template for 1st order
absorbing boundary conditions.
"""
absorb_normal = sym.normal(self.absorb_tag, self.dimensions)
e, h = self.split_eh(w)
if self.fixed_material:
epsilon = self.epsilon
mu = self.mu
absorb_Z = (mu/epsilon)**0.5 # noqa: N806
absorb_Y = 1/absorb_Z # noqa: N806
absorb_e = sym.cse(sym.project("vol", self.absorb_tag)(e))
absorb_h = sym.cse(sym.project("vol", self.absorb_tag)(h))
bc = flat_obj_array(
absorb_e + 1/2*(self.space_cross_h(absorb_normal, self.space_cross_e(
absorb_normal, absorb_e))
- absorb_Z*self.space_cross_h(absorb_normal, absorb_h)),
absorb_h + 1/2*(
self.space_cross_e(absorb_normal, self.space_cross_h(
absorb_normal, absorb_h))
+ absorb_Y*self.space_cross_e(absorb_normal, absorb_e)))
return bc
def simple_mpi_communication_entrypoint():
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
from mpi4py import MPI
comm = MPI.COMM_WORLD
num_parts = comm.Get_size()
mesh_dist = MPIMeshDistributor(comm)
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-1, ) * 2,
b=(1, ) * 2,
n=(3, ) * 2)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element,
num_parts)
else:
local_mesh = mesh_dist.receive_mesh_part()
vol_discr = DGDiscretizationWithBoundaries(cl_ctx,
local_mesh,
order=5,
mpi_communicator=comm)
sym_x = sym.nodes(local_mesh.dim)
myfunc_symb = sym.sin(np.dot(sym_x, [2, 3]))
myfunc = bind(vol_discr, myfunc_symb)(queue)
sym_all_faces_func = sym.cse(
sym.interp("vol", "all_faces")(sym.var("myfunc")))
sym_int_faces_func = sym.cse(
sym.interp("vol", "int_faces")(sym.var("myfunc")))
sym_bdry_faces_func = sym.cse(
sym.interp(sym.BTAG_ALL,
"all_faces")(sym.interp("vol",
sym.BTAG_ALL)(sym.var("myfunc"))))
bound_face_swap = bind(
vol_discr,
sym.interp("int_faces", "all_faces")(
sym.OppositeInteriorFaceSwap("int_faces")(sym_int_faces_func)) -
(sym_all_faces_func - sym_bdry_faces_func))
# print(bound_face_swap)
# 1/0
hopefully_zero = bound_face_swap(queue, myfunc=myfunc)
import numpy.linalg as la
error = la.norm(hopefully_zero.get())
np.set_printoptions(threshold=100000000, suppress=True)
print(hopefully_zero)
print(error)
assert error < 1e-14
def pmc_bc(self, w):
"Construct part of the flux operator template for PMC boundary conditions"
e, h = self.split_eh(w)
pmc_e = sym.cse(sym.interp("vol", self.pmc_tag)(e))
pmc_h = sym.cse(sym.interp("vol", self.pmc_tag)(h))
return join_fields(pmc_e, -pmc_h)
def pmc_bc(self, w):
"Construct part of the flux operator template for PMC boundary conditions"
e, h = self.split_eh(w)
pmc_e = sym.cse(sym.project("vol", self.pmc_tag)(e))
pmc_h = sym.cse(sym.project("vol", self.pmc_tag)(h))
return flat_obj_array(pmc_e, -pmc_h)
def sym_operator(self):
d = self.ambient_dim
w = sym.make_sym_array("w", d + 1)
u = w[0]
v = w[1:]
# boundary conditions -------------------------------------------------
# dirichlet BCs -------------------------------------------------------
dir_u = sym.cse(sym.interp("vol", self.dirichlet_tag)(u))
dir_v = sym.cse(sym.interp("vol", self.dirichlet_tag)(v))
if self.dirichlet_bc_f:
# FIXME
from warnings import warn
warn("Inhomogeneous Dirichlet conditions on the wave equation "
"are still having issues.")
dir_g = sym.Field("dir_bc_u")
dir_bc = join_fields(2 * dir_g - dir_u, dir_v)
else:
dir_bc = join_fields(-dir_u, dir_v)
dir_bc = sym.cse(dir_bc, "dir_bc")
# neumann BCs ---------------------------------------------------------
neu_u = sym.cse(sym.interp("vol", self.neumann_tag)(u))
neu_v = sym.cse(sym.interp("vol", self.neumann_tag)(v))
neu_bc = sym.cse(join_fields(neu_u, -neu_v), "neu_bc")
# radiation BCs -------------------------------------------------------
rad_normal = sym.normal(self.radiation_tag, d)
rad_u = sym.cse(sym.interp("vol", self.radiation_tag)(u))
rad_v = sym.cse(sym.interp("vol", self.radiation_tag)(v))
rad_bc = sym.cse(
join_fields(
0.5 * (rad_u - self.sign * np.dot(rad_normal, rad_v)), 0.5 *
rad_normal * (np.dot(rad_normal, rad_v) - self.sign * rad_u)),
"rad_bc")
# entire operator -----------------------------------------------------
def flux(pair):
return sym.interp(pair.dd, "all_faces")(self.flux(pair))
result = sym.InverseMassOperator()(
join_fields(-self.c *
np.dot(sym.stiffness_t(self.ambient_dim), v), -self.c *
(sym.stiffness_t(self.ambient_dim) * u)) -
sym.FaceMassOperator()
(flux(sym.int_tpair(w)) +
flux(sym.bv_tpair(self.dirichlet_tag, w, dir_bc)) +
flux(sym.bv_tpair(self.neumann_tag, w, neu_bc)) +
flux(sym.bv_tpair(self.radiation_tag, w, rad_bc))))
result[0] += self.source_f
return result
def get_strong_wave_op_with_discr_direct(cl_ctx, dims=2, order=4):
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * dims,
b=(0.5, ) * dims,
n=(16, ) * dims)
logger.debug("%d elements", mesh.nelements)
discr = DGDiscretizationWithBoundaries(cl_ctx, mesh, order=order)
source_center = np.array([0.1, 0.22, 0.33])[:dims]
source_width = 0.05
source_omega = 3
sym_x = sym.nodes(mesh.dim)
sym_source_center_dist = sym_x - source_center
sym_t = sym.ScalarVariable("t")
from meshmode.mesh import BTAG_ALL
c = -0.1
sign = -1
w = sym.make_sym_array("w", dims + 1)
u = w[0]
v = w[1:]
source_f = (
sym.sin(source_omega * sym_t) *
sym.exp(-np.dot(sym_source_center_dist, sym_source_center_dist) /
source_width**2))
rad_normal = sym.normal(BTAG_ALL, dims)
rad_u = sym.cse(sym.interp("vol", BTAG_ALL)(u))
rad_v = sym.cse(sym.interp("vol", BTAG_ALL)(v))
rad_bc = sym.cse(
sym.join_fields(
0.5 * (rad_u - sign * np.dot(rad_normal, rad_v)),
0.5 * rad_normal * (np.dot(rad_normal, rad_v) - sign * rad_u)),
"rad_bc")
sym_operator = (
-sym.join_fields(-c * np.dot(sym.nabla(dims), v) - source_f, -c *
(sym.nabla(dims) * u)) + sym.InverseMassOperator()(
sym.FaceMassOperator()
(dg_flux(c, sym.int_tpair(w)) +
dg_flux(c, sym.bv_tpair(BTAG_ALL, w, rad_bc)))))
return (sym_operator, discr)
def test_bessel(ctx_factory):
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue)
dims = 2
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(0.1, ) * dims,
b=(1.0, ) * dims,
n=(8, ) * dims)
discr = DGDiscretizationWithBoundaries(actx, mesh, order=3)
nodes = sym.nodes(dims)
r = sym.cse(sym.sqrt(nodes[0]**2 + nodes[1]**2))
# https://dlmf.nist.gov/10.6.1
n = 3
bessel_zero = (sym.bessel_j(n + 1, r) + sym.bessel_j(n - 1, r) -
2 * n / r * sym.bessel_j(n, r))
z = bind(discr, sym.norm(2, bessel_zero))(actx)
assert z < 1e-15
def integral(arg, dd=None):
sym = _sym()
if dd is None:
dd = sym.DD_VOLUME
dd = sym.as_dofdesc(dd)
return sym.NodalSum(dd)(
arg * sym.cse(
sym.MassOperator(dd_in=dd)(sym.Ones(dd)),
"mass_quad_weights",
sym.cse_scope.DISCRETIZATION))
def surface_advection_weak_flux(flux_type, u, velocity):
v_dot_n = v_dot_n_tpair(velocity, dd=u.dd)
# NOTE: the normals in v_dot_n point to the exterior of their respective
# elements, so this is actually just an average
v_dot_n = sym.cse(0.5 * (v_dot_n.int - v_dot_n.ext), "v_dot_normal")
flux_type = flux_type.lower()
if flux_type == "central":
return u.avg * v_dot_n
elif flux_type == "lf":
return u.avg * v_dot_n + 0.5 * sym.fabs(v_dot_n) * (u.int - u.ext)
else:
raise ValueError(f"flux '{flux_type}' is not implemented")
def __init__(self,
queue,
discr,
field_var_name,
grudge_bound_op,
num_fields,
component_getter,
exec_mapper_factory=ExecutionMapper):
"""Arguments:
field_var_name: The name of the simulation variable
grudge_bound_op: The BoundExpression for the right-hand side
num_fields: The number of components in the simulation variable
component_getter: A function, which, given an object array
representing the simulation variable, splits the array into
its components
"""
super().__init__(queue, component_getter)
# Construct sym_rhs to have the effect of replacing the RHS calls in the
# dagrt code with calls of the grudge operator.
from grudge.symbolic.primitives import FunctionSymbol, Variable
call = sym.cse(
FunctionSymbol("grudge_op")(
*((Variable("t", dd=sym.DD_SCALAR), ) + tuple(
Variable(field_var_name, dd=sym.DD_VOLUME)[i]
for i in range(num_fields)))))
from pytools.obj_array import join_fields
sym_rhs = join_fields(*(call[i] for i in range(num_fields)))
self.queue = queue
self.grudge_bound_op = grudge_bound_op
from grudge.function_registry import register_external_function
freg = register_external_function(base_function_registry,
"grudge_op",
implementation=self._bound_op,
dd=sym.DD_VOLUME)
self.set_up_stepper(discr, field_var_name, sym_rhs, num_fields, freg,
exec_mapper_factory)
self.component_getter = component_getter
def incident_bc(self, w):
"""Flux terms for incident boundary conditions"""
# NOTE: Untested for inhomogeneous materials, but would usually be
# physically meaningless anyway (are there exceptions to this?)
e, h = self.split_eh(w)
from grudge.tools import count_subset
fld_cnt = count_subset(self.get_eh_subset())
from grudge.tools import is_zero
incident_bc_data = self.incident_bc_data(self, e, h)
if is_zero(incident_bc_data):
return make_obj_array([0]*fld_cnt)
else:
return sym.cse(-incident_bc_data)
def test_incorrect_assignment_aggregation(actx_factory, ambient_dim):
"""Tests that the greedy assignemnt aggregation code works on a non-trivial
expression (on which it didn't work at the time of writing).
"""
actx = actx_factory()
target_order = 4
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * ambient_dim,
b=(0.5, ) * ambient_dim,
n=(8, ) * ambient_dim,
order=1)
discr = DiscretizationCollection(actx, mesh, order=target_order)
# {{{ test with a relative norm
from grudge.dof_desc import DD_VOLUME
dd = DD_VOLUME
sym_x = sym.make_sym_array("y", ambient_dim, dd=dd)
sym_y = sym.make_sym_array("y", ambient_dim, dd=dd)
sym_norm_y = sym.norm(2, sym_y, dd=dd)
sym_norm_d = sym.norm(2, sym_x - sym_y, dd=dd)
sym_op = sym_norm_d / sym_norm_y
logger.info("%s", sym.pretty(sym_op))
# FIXME: this shouldn't raise a RuntimeError
with pytest.raises(RuntimeError):
bind(discr, sym_op)(actx, x=1.0, y=discr.discr_from_dd(dd).nodes())
# }}}
# {{{ test with repeated mass inverses
sym_minv_y = sym.cse(sym.InverseMassOperator()(sym_y), "minv_y")
sym_u = make_obj_array([0.5 * sym.Ones(dd), 0.0, 0.0])[:ambient_dim]
sym_div_u = sum(d(u) for d, u in zip(sym.nabla(ambient_dim), sym_u))
sym_op = sym.MassOperator(dd)(sym_u) \
+ sym.MassOperator(dd)(sym_minv_y * sym_div_u)
logger.info("%s", sym.pretty(sym_op))
# FIXME: this shouldn't raise a RuntimeError either
bind(discr, sym_op)(actx, y=discr.discr_from_dd(dd).nodes())
def weak_flux(self, u):
normal = sym.normal(u. dd, self.ambient_dim)
v_dot_normal = sym.cse(self.v.dot(normal), "v_dot_normal")
norm_v = sym.sqrt((self.v**2).sum())
if self.flux_type == "central":
return u.avg*v_dot_normal
elif self.flux_type == "lf":
return u.avg*v_dot_normal + 0.5*norm_v*(u.int - u.ext)
elif self.flux_type == "upwind":
return (
v_dot_normal * sym.If(
sym.Comparison(v_dot_normal, ">", 0),
u.int, # outflow
u.ext, # inflow
))
else:
raise ValueError("invalid flux type")
def advection_weak_flux(flux_type, u, velocity):
normal = sym.normal(u.dd, len(velocity))
v_dot_n = sym.cse(velocity.dot(normal), "v_dot_normal")
flux_type = flux_type.lower()
if flux_type == "central":
return u.avg * v_dot_n
elif flux_type == "lf":
norm_v = sym.sqrt((velocity**2).sum())
return u.avg * v_dot_n + 0.5 * norm_v * (u.int - u.ext)
elif flux_type == "upwind":
u_upwind = sym.If(
sym.Comparison(v_dot_n, ">", 0),
u.int, # outflow
u.ext # inflow
)
return u_upwind * v_dot_n
else:
raise ValueError("flux `{}` is not implemented".format(flux_type))
def test_bessel(actx_factory):
actx = actx_factory()
dims = 2
mesh = mgen.generate_regular_rect_mesh(a=(0.1, ) * dims,
b=(1.0, ) * dims,
nelements_per_axis=(8, ) * dims)
discr = DiscretizationCollection(actx, mesh, order=3)
nodes = sym.nodes(dims)
r = sym.cse(sym.sqrt(nodes[0]**2 + nodes[1]**2))
# https://dlmf.nist.gov/10.6.1
n = 3
bessel_zero = (sym.bessel_j(n + 1, r) + sym.bessel_j(n - 1, r) -
2 * n / r * sym.bessel_j(n, r))
z = bind(discr, sym.norm(2, bessel_zero))(actx)
assert z < 1e-15
def __call__(self, expr):
# Put the result expressions into variables as well.
expr = sym.cse(expr, "_result")
# from grudge.symbolic.mappers.type_inference import TypeInferrer
# self.typedict = TypeInferrer()(expr)
# Used for diff batching
self.diff_ops = self.collect_diff_ops(expr)
codegen_state = CodeGenerationState(generating_discr_code=False)
# Finally, walk the expression and build the code.
result = super().__call__(expr, codegen_state)
eval_code = self.eval_code
del self.eval_code
discr_code = self.discr_code
del self.discr_code
from grudge.symbolic.dofdesc_inference import DOFDescInferenceMapper
inf_mapper = DOFDescInferenceMapper(
discr_code + eval_code, self.function_registry)
eval_code = aggregate_assignments(
inf_mapper, eval_code, result, self.max_vectors_in_batch_expr)
discr_code = rewrite_insn_to_loopy_insns(inf_mapper, discr_code)
eval_code = rewrite_insn_to_loopy_insns(inf_mapper, eval_code)
from pytools.obj_array import make_obj_array
return (
Code(discr_code,
make_obj_array(
[Variable(name)
for name in self.discr_scope_names_copied_to_eval])),
Code(eval_code, result))
def transcribe_phase(dag, field_var_name, field_components, phase_name,
sym_operator):
"""Generate a Grudge operator for a Dagrt time integrator phase.
Arguments:
dag: The Dagrt code object for the time integrator
field_var_name: The name of the simulation variable
field_components: The number of components (fields) in the variable
phase_name: The name of the phase to transcribe
sym_operator: The Grudge symbolic expression to substitue for the
right-hand side evaluation in the Dagrt code
"""
sym_operator = gmap.OperatorBinder()(sym_operator)
phase = dag.phases[phase_name]
ctx = {
"<t>": sym.var("input_t", dof_desc.DD_SCALAR),
"<dt>": sym.var("input_dt", dof_desc.DD_SCALAR),
f"<state>{field_var_name}": sym.make_sym_array(
f"input_{field_var_name}", field_components),
"<p>residual": sym.make_sym_array(
"input_residual", field_components),
}
rhs_name = f"<func>{field_var_name}"
output_vars = [v for v in ctx]
yielded_states = []
ordered_stmts = topological_sort(
isolate_function_calls_in_phase(
phase,
dag.get_stmt_id_generator(),
dag.get_var_name_generator()).statements,
phase.depends_on)
for stmt in ordered_stmts:
if stmt.condition is not True:
raise NotImplementedError(
"non-True condition (in statement '%s') not supported"
% stmt.id)
if isinstance(stmt, lang.Nop):
pass
elif isinstance(stmt, lang.Assign):
if not isinstance(stmt.lhs, p.Variable):
raise NotImplementedError("lhs of statement %s is not a variable: %s"
% (stmt.id, stmt.lhs))
ctx[stmt.lhs.name] = sym.cse(
DagrtToGrudgeRewriter(ctx)(stmt.rhs),
(
stmt.lhs.name
.replace("<", "")
.replace(">", "")))
elif isinstance(stmt, lang.AssignFunctionCall):
if stmt.function_id != rhs_name:
raise NotImplementedError(
"statement '%s' calls unsupported function '%s'"
% (stmt.id, stmt.function_id))
if stmt.parameters:
raise NotImplementedError(
"statement '%s' calls function '%s' with positional arguments"
% (stmt.id, stmt.function_id))
kwargs = {name: sym.cse(DagrtToGrudgeRewriter(ctx)(arg))
for name, arg in stmt.kw_parameters.items()}
if len(stmt.assignees) != 1:
raise NotImplementedError(
"statement '%s' calls function '%s' "
"with more than one LHS"
% (stmt.id, stmt.function_id))
assignee, = stmt.assignees
ctx[assignee] = GrudgeArgSubstitutor(kwargs)(sym_operator)
elif isinstance(stmt, lang.YieldState):
d2g = DagrtToGrudgeRewriter(ctx)
yielded_states.append(
(
stmt.time_id,
d2g(stmt.time),
stmt.component_id,
d2g(stmt.expression)))
else:
raise NotImplementedError("statement %s is of unsupported type ''%s'"
% (stmt.id, type(stmt).__name__))
return output_vars, [ctx[ov] for ov in output_vars], yielded_states
def flux(self, u):
normal = sym.normal(u.dd, self.ambient_dim)
v_dot_normal = sym.cse(self.v.dot(normal), "v_dot_normal")
return u.int * v_dot_normal - self.weak_flux(u)