def solve_system(N):
fenics_mesh = dolfinx.UnitCubeMesh(fenicsx_comm, N, N, N)
fenics_space = dolfinx.FunctionSpace(fenics_mesh, ("CG", 1))
u = ufl.TrialFunction(fenics_space)
v = ufl.TestFunction(fenics_space)
k = 2
# print(u*v*ufl.ds)
form = (ufl.inner(ufl.grad(u), ufl.grad(v)) -
k**2 * ufl.inner(u, v)) * ufl.dx
# locate facets on the cube boundary
facets = locate_entities_boundary(
fenics_mesh, 2, lambda x: np.logical_or(
np.logical_or(
np.logical_or(np.isclose(x[2], 0.0), np.isclose(x[2], 1.0)),
np.logical_or(np.isclose(x[1], 0.0), np.isclose(x[1], 1.0))),
np.logical_or(np.isclose(x[0], 0.0), np.isclose(x[0], 1.0))))
facets.sort()
# alternative - more general approach
boundary = entities_to_geometry(
fenics_mesh,
fenics_mesh.topology.dim - 1,
exterior_facet_indices(fenics_mesh),
True,
)
# print(len(facets)
assert len(facets) == len(exterior_facet_indices(fenics_mesh))
u0 = fem.Function(fenics_space)
with u0.vector.localForm() as u0_loc:
u0_loc.set(0)
# solution vector
bc = DirichletBC(u0, locate_dofs_topological(fenics_space, 2, facets))
A = 1 + 1j
f = Function(fenics_space)
f.interpolate(lambda x: A * k**2 * np.cos(k * x[0]) * np.cos(k * x[1]))
L = ufl.inner(f, v) * ufl.dx
u0.name = "u"
problem = fem.LinearProblem(form,
L,
u=u0,
petsc_options={
"ksp_type": "preonly",
"pc_type": "lu"
})
# problem = fem.LinearProblem(form, L, bcs=[bc], u=u0, petsc_options={"ksp_type": "preonly", "pc_type": "lu"})
start_time = time.time()
soln = problem.solve()
if world_rank == 0:
print("--- fenics solve done in %s seconds ---" %
(time.time() - start_time))
def boundary_grid_from_fenics_mesh(fenics_mesh):
"""
Create a Bempp boundary grid from a FEniCS Mesh.
Return the Bempp grid and a map from the node numberings of the FEniCS
mesh to the node numbers of the boundary grid.
"""
import bempp.api
import numpy as np
from dolfinx.cpp.mesh import entities_to_geometry, exterior_facet_indices
boundary = entities_to_geometry(
fenics_mesh,
fenics_mesh.topology.dim - 1,
exterior_facet_indices(fenics_mesh),
True,
)
bm_nodes = set()
for tri in boundary:
for node in tri:
bm_nodes.add(node)
bm_nodes = list(bm_nodes)
bm_cells = np.array([[bm_nodes.index(i) for i in tri] for tri in boundary])
bm_coords = fenics_mesh.geometry.x[bm_nodes]
bempp_boundary_grid = bempp.api.Grid(bm_coords.transpose(),
bm_cells.transpose())
return bempp_boundary_grid, bm_nodes
def bm_from_fenics_mesh(fenics_mesh):
boundary = entities_to_geometry(
fenics_mesh,
fenics_mesh.topology.dim - 1,
exterior_facet_indices(fenics_mesh),
True,
)
# print("number of facets ", len(exterior_facet_indices(fenics_mesh)))
bm_nodes = set()
for tri in boundary:
for node in tri:
bm_nodes.add(node)
bm_nodes = list(bm_nodes)
# bm_cells - remap cell indices between 0-len(bm_nodes)
bm_cells = np.array([[bm_nodes.index(i) for i in tri] for tri in boundary])
bm_coords = fenics_mesh.geometry.x[bm_nodes]
# print(boundary)
# print("fenics mesh dofs\n", fm_dofs)
# # print("type of bm_coords ", type(bm_nodes), len(bm_nodes))
# print('shape bm_cells ', bm_cells.shape)
# print('type bm_cells ', type(bm_cells))
# print('bm_cells \n', bm_cells)
# print('bm_nodes \n', bm_nodes)
return bm_nodes, bm_cells, bm_coords
def get_num_bdry_verts(fenics_mesh):
num_fenics_vertices = fenics_mesh.topology.connectivity(0, 0).num_nodes
fenics_mesh.topology.create_connectivity(2, 0)
tris = fenics_mesh.topology.connectivity(2, 0)
local2global_nodes = fenics_mesh.topology.index_map(0).global_indices(
False)
exterior_facets = exterior_facet_indices(fenics_mesh)
exterior_nodes = set()
for i in exterior_facets:
for j in tris.links(i):
exterior_nodes.add(local2global_nodes[j])
root = 0
rank = comm.Get_rank()
ghosts = fenics_mesh.topology.index_map(0).ghosts
ghost_owner = fenics_mesh.topology.index_map(0).ghost_owner_rank()
all_global = fenics_mesh.topology.index_map(0).global_indices(False)
all_indices = fenics_mesh.topology.index_map(0).indices(False)
# print(sendbuf_vertices)
ownership = np.ones_like(list(exterior_nodes), dtype=np.int32) * comm.rank
ext_nodes_list = list(exterior_nodes)
ghosts_list = list(ghosts)
for i in range(len(ext_nodes_list)):
if ext_nodes_list[i] in ghosts_list:
index = ghosts_list.index(ext_nodes_list[i])
ownership[i] = ghost_owner[index]
# print(np.sum(ownership == rank))
# print(sendbuf_vertices[ownership == rank])
# print("\n")
# print("exterior nodes length (global)", len(exterior_nodes))
sendbuf_vertices = np.asarray(list(exterior_nodes), dtype=np.int64)
print("ownership ", ownership)
print("external nodes from exterior_facet_indices ", ext_nodes_list)
print("size local ", fenics_mesh.topology.index_map(0).size_local)
print("all indices ", all_indices)
print("RANK {} \n".format(rank))
sendbuf_vertices = sendbuf_vertices[ownership == rank]
# print(sum(sendbuf_vertices))
sendcounts = np.array(comm.gather(len(sendbuf_vertices), root))
# print("ghosts", fenics_mesh.topology.index_map(0).ghosts)
# print("ghost owner rank ", fenics_mesh.topology.index_map(0).ghost_owner_rank())
print("sendcounts ", sendcounts)
if rank == root:
# print("sendcounts: {}, total: {}".format(sendcounts, sum(sendcounts)))
recvbuf = np.empty(sum(sendcounts), dtype=np.int64)
else:
recvbuf = None
comm.Gatherv(sendbuf=sendbuf_vertices,
recvbuf=(recvbuf, sendcounts),
root=root)
if rank == root:
bdry_vertices = sorted(np.unique(recvbuf))
# print("Gathered array: {}, unique length: {}".format(bdry_vertices, len(bdry_vertices)))
# print("unique length: {}".format(len(bdry_vertices)))
return len(bdry_vertices)
return 0
def test_mpi():
N = 2
comm = MPI.COMM_WORLD
world_rank = comm.Get_rank()
world_size = comm.Get_size()
print("world rank ", world_rank)
if world_rank == 1:
print(MPI.COMM_SELF)
fenics_mesh = dolfinx.UnitCubeMesh(MPI.COMM_SELF, N, N, N)
facet_indices = exterior_facet_indices(fenics_mesh)
print(facet_indices)
def bm_from_fenics_mesh_mpi(fenics_mesh, fenics_space):
boundary = entities_to_geometry(
fenics_mesh,
fenics_mesh.topology.dim - 1,
exterior_facet_indices(fenics_mesh),
True,
)
dofmap = fenics_space.dofmap.index_map.global_indices(False)
geom_map = fenics_mesh.geometry.index_map().global_indices(False)
dofmap_mesh = fenics_mesh.geometry.dofmap
assert dofmap == geom_map
# print("dofmap ", dofmap)
# print("geometry map ", geom_map)
# print("dofmap mesh", dofmap_mesh)
# print("number of facets ", len(exterior_facet_indices(fenics_mesh)))
bm_nodes = set()
for i, tri in enumerate(boundary):
for j, node in enumerate(tri):
# print(node, boundary[i][j])
glob_geom_node = geom_map[node]
boundary[i][j] = glob_geom_node
bm_nodes.add(node)
bm_nodes_global = [geom_map[i] for i in bm_nodes]
bm_nodes = list(bm_nodes)
bm_coords = fenics_mesh.geometry.x[bm_nodes]
# bm_cells - remap cell indices between 0-len(bm_nodes)
# bm_cells = np.array([[bm_nodes.index(i) for i in tri] for tri in boundary])
# print("bm_coords\n", bm_coords)
# print("bm_nodes\n", bm_nodes)
# print("boundary\n", boundary)
# # print("type of bm_coords ", type(bm_nodes), len(bm_nodes))
# print('shape bm_cells ', bm_cells.shape)
# print('type bm_cells ', type(bm_cells))
# print('bm_cells \n', bm_cells)
# print("dofmap ", fenics_mesh.geometry.dofmap)
return bm_nodes_global, bm_coords, boundary
def create_boundary_mesh(mesh, comm, orient=False):
"""
Create a mesh consisting of all exterior facets of a mesh
Input:
mesh - The mesh
comm - The MPI communicator
orient - Boolean flag for reorientation of facets to have
consistent outwards-pointing normal (default: True)
Output:
bmesh - The boundary mesh
bmesh_to_geometry - Map from cells of the boundary mesh
to the geometry of the original mesh
"""
ext_facets = cmesh.exterior_facet_indices(mesh)
boundary_geometry = cmesh.entities_to_geometry(mesh, mesh.topology.dim - 1,
ext_facets, orient)
facet_type = dolfinx.cpp.mesh.to_string(
cmesh.cell_entity_type(mesh.topology.cell_type, mesh.topology.dim - 1))
facet_cell = ufl.Cell(facet_type, geometric_dimension=mesh.geometry.dim)
degree = mesh.ufl_domain().ufl_coordinate_element().degree()
ufl_domain = ufl.Mesh(ufl.VectorElement("Lagrange", facet_cell, degree))
bmesh = dolfinx.mesh.create_mesh(comm, boundary_geometry, mesh.geometry.x,
ufl_domain)
return bmesh, boundary_geometry
tris = mesh.topology.connectivity(2, 0)
mesh.topology.create_connectivity(2, 3)
tri_to_tet = mesh.topology.connectivity(2, 3)
surface_tris = []
for i in range(tris.num_nodes):
if (len(tri_to_tet.links(i)) == 1):
surface_tris += [i]
surface_tris = np.array(surface_tris)
len(surface_tris)
print("Surface tris done")
triangles = surface_tris
# print(surface_tris)
facets = exterior_facet_indices(mesh)
# print('facets ', facets)
# now apply boundary conditions
V = FunctionSpace(mesh, ("Lagrange", 1))
u0 = Function(V)
u0.vector.set(0.0)
bc = DirichletBC(u0, locate_dofs_topological(V, 2, facets))
# Define variational problem
u = ufl.TrialFunction(V)
v = ufl.TestFunction(V)
x = ufl.SpatialCoordinate(mesh)
f = 10 * ufl.exp(-((x[0] - 30)**2 + (x[1] + 50)**2) / 200)
g = ufl.sin(5 * x[0])
def get_num_bdry_dofs(fenics_mesh):
exterior_facets = exterior_facet_indices(fenics_mesh)
boundary = entities_to_geometry(
fenics_mesh,
fenics_mesh.topology.dim - 1,
exterior_facet_indices(fenics_mesh),
True,
)
bm_nodes = set()
fenics_space = dolfinx.FunctionSpace(fenics_mesh, ("CG", 1))
global_dofs_map = fenics_space.dofmap.index_map.global_indices(False)
# print(fenics_space.dofmap.index_map.ghosts)
mapping, dof_map = vertex_dofmap(fenics_mesh)
# value of 9
vertex_err_val = 9
if vertex_err_val in mapping.keys():
print("value of corresponding dof is ", mapping[9])
for i, tri in enumerate(boundary):
for j, node in enumerate(tri):
# print(node, boundary[i][j])
glob_geom_node = global_dofs_map[node]
# glob_geom_node = geom_map[node]
boundary[i][j] = glob_geom_node
bm_nodes.add(node)
bm_nodes_global = [global_dofs_map[i] for i in bm_nodes]
ghosts = fenics_space.dofmap.index_map.ghosts
ghosts_list = list(ghosts)
ghost_owner = fenics_space.dofmap.index_map.ghost_owner_rank()
ownership = np.ones_like(bm_nodes_global, dtype=np.int32) * comm.rank
for i in range(len(bm_nodes_global)):
if bm_nodes_global[i] in ghosts_list:
index = ghosts_list.index(bm_nodes_global[i])
ownership[i] = ghost_owner[index]
print(ownership)
print(bm_nodes_global)
print("RANK {} \n".format(comm.Get_rank()))
root = 0
rank = comm.Get_rank()
sendbuf = np.array(bm_nodes_global)
# print("sendbuf ", sendbuf)
sendcounts = np.array(comm.gather(len(sendbuf), root))
print("sendcounts ", sendcounts)
if rank == root:
print("sendcounts: {}, total: {}".format(sendcounts, sum(sendcounts)))
recvbuf = np.empty(sum(sendcounts), dtype=np.int64)
else:
recvbuf = None
comm.Gatherv(sendbuf=sendbuf, recvbuf=(recvbuf, sendcounts), root=root)
if rank == root:
bm_nodes_unique = sorted(np.unique(recvbuf))
print("Gathered array: {}, unique length: {}".format(
bm_nodes_unique, len(bm_nodes_unique)))
return len(bm_nodes_unique)
return 0
def play(fenics_mesh):
exterior_facets = exterior_facet_indices(fenics_mesh)
num_fenics_vertices = fenics_mesh.topology.connectivity(0, 0).num_nodes
tets = fenics_mesh.topology.connectivity(3, 0)
fenics_mesh.topology.create_connectivity(2, 0)
tris = fenics_mesh.topology.connectivity(2, 0)
fenics_mesh.topology.create_connectivity(2, 3)
tri_to_tet = fenics_mesh.topology.connectivity(2, 3)
exterior_nodes = set()
local2global_nodes = fenics_mesh.topology.index_map(0).global_indices(
False)
# from exterior facets get all nodes (global) on bdry
# is the bdry triangle index the same as in tris?
for i in exterior_facets:
for j in tris.links(i):
exterior_nodes.add(local2global_nodes[j])
# print("ghost owner rank ", fenics_mesh.topology.index_map(0))
# print("index_map ", dir(fenics_mesh.topology.index_map(0)))
ghosts = fenics_mesh.topology.index_map(0).ghosts
# print("l2g ", local2global_nodes)
# ghosts_global = [i for i in ghosts]
print("ghosts", fenics_mesh.topology.index_map(0).ghosts)
# print("ghosts_global", fenics_mesh.topology.index_map(0).ghosts)
print("ghost owner rank ",
fenics_mesh.topology.index_map(0).ghost_owner_rank())
print("all global indices on this process",
fenics_mesh.topology.index_map(0).indices(True))
# print(dir(fenics_mesh.topology.index_map(0)))
size_local = fenics_mesh.topology.index_map(0).size_local
size_global = fenics_mesh.topology.index_map(0).size_global
# shared = fenics_mesh.topology.index_map(0).shared_indices
print("size local ", size_local, " size global ", size_global)
print("exterior nodes (global)", exterior_nodes)
print("exterior nodes length (global)", len(exterior_nodes))
sendbuf_nodes = np.asarray(list(exterior_nodes), dtype=np.int64)
print("length of sendbuf ", len(sendbuf_nodes))
# print("exterior facets ", exterior_facets)
# print("exterior facet 0", [ local2global_nodes[i] for i in tris.links(0)])
print("RANK ", comm.Get_rank())
# print(local2global_nodes)
boundary = entities_to_geometry(
fenics_mesh,
fenics_mesh.topology.dim - 1,
exterior_facet_indices(fenics_mesh),
True,
)
bm_nodes = set()
# this map is wrong:
geom_map = fenics_mesh.geometry.index_map().global_indices(False)
# use this instead:
fenics_space = dolfinx.FunctionSpace(fenics_mesh, ("CG", 1))
global_dofs_map = fenics_space.dofmap.index_map.global_indices(False)
mapping_space = space_dofmap(fenics_mesh, fenics_space)
# print("mapping space ", mapping_space)
# print(mapping)
exterior_dofs = []
for i in exterior_nodes:
exterior_dofs.append(mapping_space[i])
print("exterior dofs from vertices", exterior_dofs)
for i, tri in enumerate(boundary):
for j, node in enumerate(tri):
# print(node, boundary[i][j])
glob_geom_node = global_dofs_map[node]
# glob_geom_node = geom_map[node]
boundary[i][j] = glob_geom_node
bm_nodes.add(node)
bm_nodes_global = [geom_map[i] for i in bm_nodes]
print("bm nodes ", bm_nodes_global)
mapping, dof_map = vertex_dofmap(fenics_mesh)
# print("geom nodes ", bm_nodes_global)
# print(exterior_facets)
# get mapping between dofs and vertices
mesh_dofmap = fenics_mesh.geometry.dofmap
# mapping = vertex_dofmap(fenics_mesh)
# print(mapping[13])
mesh_dofs = []
for i in exterior_nodes:
# print(i)
mesh_dofs.append(mapping[i])
# print("mapping ", mapping)
# print("dof_map ", dof_map)
ma = [dof_map[i] for i in bm_nodes_global]
print(ma)
print("\n")
# print("bm_nodes global ", sorted(bm_nodes_global))
print("do a gather")
recvbuf_nodes = None
if comm.rank == 0:
info = MPI.Status()
recvbuf_nodes = np.empty(comm.Get_size() * len(exterior_nodes),
dtype=np.int64)
comm.Gather(sendbuf_nodes, recvbuf_nodes, root=0)
print("received nodes ", len(np.unique(recvbuf_nodes)))
exit(0)
def bm_from_fenics_mesh(comm, fenics_comm, fenics_mesh, fenics_space):
"""
Create a Bempp boundary grid from a FEniCS Mesh.
Return the Bempp grid and a map from the node numberings of the FEniCS
mesh to the node numbers of the boundary grid.
"""
from dolfinx.cpp.mesh import entities_to_geometry, exterior_facet_indices
boundary = entities_to_geometry(
fenics_mesh,
fenics_mesh.topology.dim - 1,
exterior_facet_indices(fenics_mesh),
True,
)
dofmap = fenics_space.dofmap.index_map.global_indices(False)
geom_map = fenics_mesh.geometry.index_map().global_indices(False)
dofmap_mesh = fenics_mesh.geometry.dofmap
# assert dofmap == geom_map
# print("number of facets ", len(exterior_facet_indices(fenics_mesh)))
bm_nodes = set()
for i, tri in enumerate(boundary):
for j, node in enumerate(tri):
# print(node, boundary[i][j])
glob_dof_node = dofmap[node]
boundary[i][j] = glob_dof_node
bm_nodes.add(node)
bm_nodes_global = [dofmap[i] for i in bm_nodes]
bm_nodes = list(bm_nodes)
bm_coords = fenics_mesh.geometry.x[bm_nodes]
# bm_cells - remap cell indices between 0-len(bm_nodes)
# bm_cells = np.array([[bm_nodes.index(i) for i in tri] for tri in boundary])
# print('shape bm_cells ', bm_cells.shape)
# print('bm_cells \n', bm_cells)
# print('bm_coords len ', len(bm_coords))
# print('bm_coords type ', type(bm_coords[0][0]))
# print("RANK ", fenics_comm.rank)
gathered_bm_coords = gather(fenics_comm, bm_coords, 3, np.float64)
gathered_bm_tris = gather(fenics_comm, boundary, 3, np.int32)
gathered_bm_nodes = gather(fenics_comm,
np.asarray(bm_nodes_global, np.int32), 1,
np.int32)
global_alldofs = np.asarray(
fenics_space.dofmap.index_map.global_indices(False), dtype=np.int32)
gathered_global_alldofs = gather(fenics_comm, global_alldofs, 1, np.int32)
if fenics_comm.rank == 0:
all_bm_coords = gathered_bm_coords.reshape(
int(len(gathered_bm_coords) / 3), 3)
all_bm_tris = gathered_bm_tris.reshape(int(len(gathered_bm_tris) / 3),
3)
all_bm_nodes = gathered_bm_nodes
# sort gathered nodes and remove repetitions (ghosts on bdry)
sorted_indices = all_bm_nodes.argsort()
all_bm_nodes_sorted = all_bm_nodes[sorted_indices]
all_bm_coords_sorted = all_bm_coords[sorted_indices]
# print("sorted indices, ", sorted_indices)
all_bm_nodes, unique = np.unique(all_bm_nodes_sorted,
return_index=True)
all_bm_coords = all_bm_coords_sorted[unique]
all_bm_nodes_list = list(all_bm_nodes)
# bm_cells - remap boundary triangle indices between 0-len(bm_nodes) - this can be improved
all_bm_cells = np.array([[all_bm_nodes_list.index(i) for i in tri]
for tri in all_bm_tris],
dtype=np.int32)
all_bm_nodes = np.asarray(all_bm_nodes_list, dtype=np.int32)
# send to Bempp process
send(comm, all_bm_coords, MPI.DOUBLE, 100)
send(comm, all_bm_cells, MPI.INT, 101)
send(comm, all_bm_nodes, MPI.INT, 102)
# print("all_bm_cells ", type(all_bm_cells))
num_fenics_vertices = len(np.unique(np.sort(gathered_global_alldofs)))
# hack - to change
comm.send(num_fenics_vertices, dest=0, tag=103)