def complete_setup(self):
"""
Returns the tuple ``remote_to_local_bdry_conn``
where `remote_to_local_bdry_conn` is a
:class:`DirectDiscretizationConnection` that gives the connection that
performs data exchange across faces from partition `i_remote_part` to the
local mesh.
"""
remote_bdry_mesh, remote_group_infos = self.mpi_comm.recv(
source=self.i_remote_part, tag=TAG_SEND_BOUNDARY)
logger.debug("rank %d: Received rank %d data", self.i_local_part,
self.i_remote_part)
from meshmode.discretization import Discretization
# Connect local_mesh to remote_mesh
from meshmode.discretization.connection import make_partition_connection
remote_to_local_bdry_conn = make_partition_connection(
self.array_context,
local_bdry_conn=self.local_bdry_conn,
i_local_part=self.i_local_part,
remote_bdry_discr=Discretization(self.array_context,
remote_bdry_mesh,
self.bdry_grp_factory),
remote_group_infos=remote_group_infos)
self.send_req.wait()
return remote_to_local_bdry_conn
def complete_setup(self):
"""
Returns the tuple ``remote_to_local_bdry_conn``
where `remote_to_local_bdry_conn` is a
:class:`DirectDiscretizationConnection` that gives the connection that
performs data exchange across faces from partition `i_remote_part` to the
local mesh.
"""
remote_data = self.mpi_comm.recv(source=self.i_remote_part,
tag=TAG_SEND_BOUNDARY)
logger.debug('rank %d: Received rank %d data', self.i_local_part,
self.i_remote_part)
from meshmode.discretization import Discretization
remote_bdry_mesh = remote_data['bdry_mesh']
remote_bdry = Discretization(self.queue.context, remote_bdry_mesh,
self.bdry_grp_factory)
remote_adj_groups = remote_data['adj']
remote_to_elem_faces = remote_data['to_elem_faces']
remote_to_elem_indices = remote_data['to_elem_indices']
# Connect local_mesh to remote_mesh
from meshmode.discretization.connection import make_partition_connection
remote_to_local_bdry_conn = make_partition_connection(
self.local_bdry_conn, self.i_local_part, remote_bdry,
remote_adj_groups, remote_to_elem_faces, remote_to_elem_indices)
self.send_req.wait()
return remote_to_local_bdry_conn
def complete_setup(self):
"""
Returns the tuple ``remote_to_local_bdry_conn``
where `remote_to_local_bdry_conn` is a
:class:`DirectDiscretizationConnection` that gives the connection that
performs data exchange across faces from partition `i_remote_part` to the
local mesh.
"""
remote_data = self.mpi_comm.recv(
source=self.i_remote_part,
tag=TAG_SEND_BOUNDARY)
logger.debug('rank %d: Received rank %d data',
self.i_local_part, self.i_remote_part)
from meshmode.discretization import Discretization
remote_bdry_mesh = remote_data['bdry_mesh']
remote_bdry = Discretization(self.queue.context, remote_bdry_mesh,
self.bdry_grp_factory)
remote_adj_groups = remote_data['adj']
remote_to_elem_faces = remote_data['to_elem_faces']
remote_to_elem_indices = remote_data['to_elem_indices']
# Connect local_mesh to remote_mesh
from meshmode.discretization.connection import make_partition_connection
remote_to_local_bdry_conn = make_partition_connection(self.local_bdry_conn,
self.i_local_part,
remote_bdry,
remote_adj_groups,
remote_to_elem_faces,
remote_to_elem_indices)
self.send_req.wait()
return remote_to_local_bdry_conn
def complete_some(self):
"""
Returns a :class:`dict` mapping a subset of remote partitions to
remote-to-local boundary connections, where a remote-to-local boundary
connection is a
:class:`~meshmode.discretization.connection.DirectDiscretizationConnection`
that performs data exchange across faces from partition `i_remote_part`
to the local mesh. When an empty dictionary is returned, setup is
complete.
"""
from mpi4py import MPI
if not self.pending_recvs:
# Already completed, nothing more to do
return {}
status = MPI.Status()
# Wait for any receive
data = [self._internal_mpi_comm.recv(status=status)]
parts = [status.source]
# Complete any other available receives while we're at it
while self._internal_mpi_comm.iprobe():
data.append(self._internal_mpi_comm.recv(status=status))
parts.append(status.source)
remote_to_local_bdry_conns = {}
for i_remote_part, (remote_bdry_mesh, remote_group_infos) in zip(parts,
data):
logger.debug("rank %d: Received rank %d data",
self.i_local_part, i_remote_part)
# Connect local_mesh to remote_mesh
from meshmode.discretization.connection import make_partition_connection
local_bdry_conn = self.local_bdry_conns[i_remote_part]
remote_to_local_bdry_conns[i_remote_part] = make_partition_connection(
self.array_context,
local_bdry_conn=local_bdry_conn,
i_local_part=self.i_local_part,
remote_bdry_discr=local_bdry_conn.to_discr.copy(
actx=self.array_context,
mesh=remote_bdry_mesh,
group_factory=self.bdry_grp_factory),
remote_group_infos=remote_group_infos)
self.pending_recvs.remove(i_remote_part)
if not self.pending_recvs:
MPI.Request.waitall(self.send_reqs)
logger.info("bdry comm rank %d comm end", self.i_local_part)
return remote_to_local_bdry_conns
def test_partition_interpolation(actx_factory, dim, mesh_pars, num_parts,
num_groups, part_method):
np.random.seed(42)
group_factory = PolynomialWarpAndBlendGroupFactory
actx = actx_factory()
order = 4
def f(x):
return 10. * actx.np.sin(50. * x)
for n in mesh_pars:
from meshmode.mesh.generation import generate_warped_rect_mesh
base_mesh = generate_warped_rect_mesh(dim, order=order, n=n)
if num_groups > 1:
from meshmode.mesh.processing import split_mesh_groups
# Group every Nth element
element_flags = np.arange(
base_mesh.nelements,
dtype=base_mesh.element_id_dtype) % num_groups
mesh = split_mesh_groups(base_mesh, element_flags)
else:
mesh = base_mesh
if part_method == "random":
part_per_element = np.random.randint(num_parts,
size=mesh.nelements)
else:
pytest.importorskip("pymetis")
from meshmode.distributed import get_partition_by_pymetis
part_per_element = get_partition_by_pymetis(
mesh, num_parts, connectivity=part_method)
from meshmode.mesh.processing import partition_mesh
part_meshes = [
partition_mesh(mesh, part_per_element, i)[0]
for i in range(num_parts)
]
connected_parts = set()
for i_local_part, part_mesh in enumerate(part_meshes):
from meshmode.distributed import get_connected_partitions
neighbors = get_connected_partitions(part_mesh)
for i_remote_part in neighbors:
connected_parts.add((i_local_part, i_remote_part))
from meshmode.discretization import Discretization
vol_discrs = [
Discretization(actx, part_meshes[i], group_factory(order))
for i in range(num_parts)
]
from meshmode.mesh import BTAG_PARTITION
from meshmode.discretization.connection import (
make_face_restriction, make_partition_connection, check_connection)
for i_local_part, i_remote_part in connected_parts:
# Mark faces within local_mesh that are connected to remote_mesh
local_bdry_conn = make_face_restriction(
actx, vol_discrs[i_local_part], group_factory(order),
BTAG_PARTITION(i_remote_part))
# Mark faces within remote_mesh that are connected to local_mesh
remote_bdry_conn = make_face_restriction(
actx, vol_discrs[i_remote_part], group_factory(order),
BTAG_PARTITION(i_local_part))
bdry_nelements = sum(grp.nelements
for grp in local_bdry_conn.to_discr.groups)
remote_bdry_nelements = sum(
grp.nelements for grp in remote_bdry_conn.to_discr.groups)
assert bdry_nelements == remote_bdry_nelements, \
"partitions do not have the same number of connected elements"
local_bdry = local_bdry_conn.to_discr
remote_bdry = remote_bdry_conn.to_discr
from meshmode.distributed import make_remote_group_infos
remote_to_local_conn = make_partition_connection(
actx,
local_bdry_conn=local_bdry_conn,
i_local_part=i_local_part,
remote_bdry_discr=remote_bdry,
remote_group_infos=make_remote_group_infos(
actx, remote_bdry_conn))
# Connect from local mesh to remote mesh
local_to_remote_conn = make_partition_connection(
actx,
local_bdry_conn=remote_bdry_conn,
i_local_part=i_remote_part,
remote_bdry_discr=local_bdry,
remote_group_infos=make_remote_group_infos(
actx, local_bdry_conn))
check_connection(actx, remote_to_local_conn)
check_connection(actx, local_to_remote_conn)
true_local_points = f(thaw(actx, local_bdry.nodes()[0]))
remote_points = local_to_remote_conn(true_local_points)
local_points = remote_to_local_conn(remote_points)
err = actx.np.linalg.norm(true_local_points - local_points, np.inf)
# Can't currently expect exact results due to limitations of
# interpolation "snapping" in DirectDiscretizationConnection's
# _resample_point_pick_indices
assert err < 1e-11
def test_partition_interpolation(ctx_factory, dim, mesh_pars, num_parts,
num_groups, scramble_partitions):
np.random.seed(42)
group_factory = PolynomialWarpAndBlendGroupFactory
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
order = 4
from pytools.convergence import EOCRecorder
eoc_rec = dict()
for i in range(num_parts):
for j in range(num_parts):
if i == j:
continue
eoc_rec[i, j] = EOCRecorder()
def f(x):
return 10. * cl.clmath.sin(50. * x)
for n in mesh_pars:
from meshmode.mesh.generation import generate_warped_rect_mesh
meshes = [
generate_warped_rect_mesh(dim, order=order, n=n)
for _ in range(num_groups)
]
if num_groups > 1:
from meshmode.mesh.processing import merge_disjoint_meshes
mesh = merge_disjoint_meshes(meshes)
else:
mesh = meshes[0]
if scramble_partitions:
part_per_element = np.random.randint(num_parts,
size=mesh.nelements)
else:
from pymetis import part_graph
_, p = part_graph(
num_parts,
xadj=mesh.nodal_adjacency.neighbors_starts.tolist(),
adjncy=mesh.nodal_adjacency.neighbors.tolist())
part_per_element = np.array(p)
from meshmode.mesh.processing import partition_mesh
part_meshes = [
partition_mesh(mesh, part_per_element, i)[0]
for i in range(num_parts)
]
from meshmode.discretization import Discretization
vol_discrs = [
Discretization(cl_ctx, part_meshes[i], group_factory(order))
for i in range(num_parts)
]
from meshmode.mesh import BTAG_PARTITION
from meshmode.discretization.connection import (
make_face_restriction, make_partition_connection, check_connection)
for i_local_part, i_remote_part in eoc_rec.keys():
if eoc_rec[i_local_part, i_remote_part] is None:
continue
# Mark faces within local_mesh that are connected to remote_mesh
local_bdry_conn = make_face_restriction(
vol_discrs[i_local_part], group_factory(order),
BTAG_PARTITION(i_remote_part))
# If these parts are not connected, don't bother checking the error
bdry_nodes = local_bdry_conn.to_discr.nodes()
if bdry_nodes.size == 0:
eoc_rec[i_local_part, i_remote_part] = None
continue
# Mark faces within remote_mesh that are connected to local_mesh
remote_bdry_conn = make_face_restriction(
vol_discrs[i_remote_part], group_factory(order),
BTAG_PARTITION(i_local_part))
assert bdry_nodes.size == remote_bdry_conn.to_discr.nodes().size, \
"partitions do not have the same number of connected nodes"
# Gather just enough information for the connection
local_bdry = local_bdry_conn.to_discr
local_mesh = part_meshes[i_local_part]
local_adj_groups = [
local_mesh.facial_adjacency_groups[i][None]
for i in range(len(local_mesh.groups))
]
local_batches = [
local_bdry_conn.groups[i].batches
for i in range(len(local_mesh.groups))
]
local_from_elem_faces = [[
batch.to_element_face for batch in grp_batches
] for grp_batches in local_batches]
local_from_elem_indices = [[
batch.to_element_indices.get(queue=queue)
for batch in grp_batches
] for grp_batches in local_batches]
remote_bdry = remote_bdry_conn.to_discr
remote_mesh = part_meshes[i_remote_part]
remote_adj_groups = [
remote_mesh.facial_adjacency_groups[i][None]
for i in range(len(remote_mesh.groups))
]
remote_batches = [
remote_bdry_conn.groups[i].batches
for i in range(len(remote_mesh.groups))
]
remote_from_elem_faces = [[
batch.to_element_face for batch in grp_batches
] for grp_batches in remote_batches]
remote_from_elem_indices = [[
batch.to_element_indices.get(queue=queue)
for batch in grp_batches
] for grp_batches in remote_batches]
# Connect from remote_mesh to local_mesh
remote_to_local_conn = make_partition_connection(
local_bdry_conn, i_local_part, remote_bdry, remote_adj_groups,
remote_from_elem_faces, remote_from_elem_indices)
# Connect from local mesh to remote mesh
local_to_remote_conn = make_partition_connection(
remote_bdry_conn, i_remote_part, local_bdry, local_adj_groups,
local_from_elem_faces, local_from_elem_indices)
check_connection(remote_to_local_conn)
check_connection(local_to_remote_conn)
true_local_points = f(local_bdry.nodes()[0].with_queue(queue))
remote_points = local_to_remote_conn(queue, true_local_points)
local_points = remote_to_local_conn(queue, remote_points)
err = la.norm((true_local_points - local_points).get(), np.inf)
eoc_rec[i_local_part, i_remote_part].add_data_point(1. / n, err)
for (i, j), e in eoc_rec.items():
if e is not None:
print("Error of connection from part %i to part %i." % (i, j))
print(e)
assert (e.order_estimate() >= order - 0.5 or e.max_error() < 1e-11)
def test_partition_interpolation(ctx_factory, dim, mesh_pars,
num_parts, num_groups, part_method):
np.random.seed(42)
group_factory = PolynomialWarpAndBlendGroupFactory
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue)
order = 4
def f(x):
return 10.*actx.np.sin(50.*x)
for n in mesh_pars:
from meshmode.mesh.generation import generate_warped_rect_mesh
base_mesh = generate_warped_rect_mesh(dim, order=order, n=n)
if num_groups > 1:
from meshmode.mesh.processing import split_mesh_groups
# Group every Nth element
element_flags = np.arange(base_mesh.nelements,
dtype=base_mesh.element_id_dtype) % num_groups
mesh = split_mesh_groups(base_mesh, element_flags)
else:
mesh = base_mesh
if part_method == "random":
part_per_element = np.random.randint(num_parts, size=mesh.nelements)
else:
pytest.importorskip('pymetis')
from meshmode.distributed import get_partition_by_pymetis
part_per_element = get_partition_by_pymetis(mesh, num_parts,
connectivity=part_method)
from meshmode.mesh.processing import partition_mesh
part_meshes = [
partition_mesh(mesh, part_per_element, i)[0] for i in range(num_parts)]
connected_parts = set()
for i_local_part, part_mesh in enumerate(part_meshes):
from meshmode.distributed import get_connected_partitions
neighbors = get_connected_partitions(part_mesh)
for i_remote_part in neighbors:
connected_parts.add((i_local_part, i_remote_part))
from meshmode.discretization import Discretization
vol_discrs = [Discretization(actx, part_meshes[i], group_factory(order))
for i in range(num_parts)]
from meshmode.mesh import BTAG_PARTITION
from meshmode.discretization.connection import (make_face_restriction,
make_partition_connection,
check_connection)
for i_local_part, i_remote_part in connected_parts:
# Mark faces within local_mesh that are connected to remote_mesh
local_bdry_conn = make_face_restriction(actx, vol_discrs[i_local_part],
group_factory(order),
BTAG_PARTITION(i_remote_part))
# Mark faces within remote_mesh that are connected to local_mesh
remote_bdry_conn = make_face_restriction(actx, vol_discrs[i_remote_part],
group_factory(order),
BTAG_PARTITION(i_local_part))
bdry_nelements = sum(
grp.nelements for grp in local_bdry_conn.to_discr.groups)
remote_bdry_nelements = sum(
grp.nelements for grp in remote_bdry_conn.to_discr.groups)
assert bdry_nelements == remote_bdry_nelements, \
"partitions do not have the same number of connected elements"
# Gather just enough information for the connection
local_bdry = local_bdry_conn.to_discr
local_mesh = part_meshes[i_local_part]
local_adj_groups = [local_mesh.facial_adjacency_groups[i][None]
for i in range(len(local_mesh.groups))]
local_batches = [local_bdry_conn.groups[i].batches
for i in range(len(local_mesh.groups))]
local_from_elem_faces = [[batch.to_element_face
for batch in grp_batches]
for grp_batches in local_batches]
local_from_elem_indices = [[batch.to_element_indices.get(queue=queue)
for batch in grp_batches]
for grp_batches in local_batches]
remote_bdry = remote_bdry_conn.to_discr
remote_mesh = part_meshes[i_remote_part]
remote_adj_groups = [remote_mesh.facial_adjacency_groups[i][None]
for i in range(len(remote_mesh.groups))]
remote_batches = [remote_bdry_conn.groups[i].batches
for i in range(len(remote_mesh.groups))]
remote_from_elem_faces = [[batch.to_element_face
for batch in grp_batches]
for grp_batches in remote_batches]
remote_from_elem_indices = [[batch.to_element_indices.get(queue=queue)
for batch in grp_batches]
for grp_batches in remote_batches]
# Connect from remote_mesh to local_mesh
remote_to_local_conn = make_partition_connection(
actx, local_bdry_conn, i_local_part, remote_bdry,
remote_adj_groups, remote_from_elem_faces,
remote_from_elem_indices)
# Connect from local mesh to remote mesh
local_to_remote_conn = make_partition_connection(
actx, remote_bdry_conn, i_remote_part, local_bdry,
local_adj_groups, local_from_elem_faces,
local_from_elem_indices)
check_connection(actx, remote_to_local_conn)
check_connection(actx, local_to_remote_conn)
true_local_points = f(thaw(actx, local_bdry.nodes()[0]))
remote_points = local_to_remote_conn(true_local_points)
local_points = remote_to_local_conn(remote_points)
err = flat_norm(true_local_points - local_points, np.inf)
# Can't currently expect exact results due to limitations of
# interpolation 'snapping' in DirectDiscretizationConnection's
# _resample_point_pick_indices
assert err < 1e-11