def create_submesh(mesh, markers): mpi_comm = mesh.mpi_comm() if not has_pybind11(): mpi_comm = mpi_comm.tompi4py() assert isinstance(markers, MeshFunctionBool) assert markers.dim() == mesh.topology().dim() marker_id = True # == 1. Extract marked cells == # # Dolfin does not support a distributed mesh that is empty on some processes. # cbcpost gets around this by moving a single cell from the a non-empty processor to an empty one. # Note that, however, this cannot work if the number of marked cell is less than the number of processors. # In the interest of considering this case, we enable at least one cell (arbitrarily) on each processor. # We find this solution acceptable for our purposes, despite the increase of the reduced mesh size, # since we are never actually interested in solving a PDE on the reduced mesh, but rather only in # assemblying tensors on it and extract their values at some locations. backup_first_marker_id = None if marker_id not in markers.array(): backup_first_marker_id = markers.array()[0] markers.array()[0] = marker_id assert marker_id in markers.array() # == 2. Create submesh == # submesh = Mesh(mesh.mpi_comm()) mesh_editor = MeshEditor() mesh_editor.open(submesh, mesh.ufl_cell().cellname(), mesh.ufl_cell().topological_dimension(), mesh.ufl_cell().geometric_dimension()) # Extract cells from mesh with specified marker_id mesh_cell_indices = where(markers.array() == marker_id)[0] mesh_cells = mesh.cells()[mesh_cell_indices] mesh_global_cell_indices = sorted([mesh.topology().global_indices(mesh.topology().dim())[cell_index] for cell_index in mesh_cell_indices]) # Get vertices of extracted cells mesh_vertex_indices = unique(mesh_cells.flatten()) mesh_global_vertex_indices = sorted([mesh.topology().global_indices(0)[vertex_index] for vertex_index in mesh_vertex_indices]) # Number vertices in a way which is independent from the number of processors. To do so ... # ... first of all collect all vertices from all processors allgathered_mesh_global_vertex_indices__non_empty_processors = list() allgathered_mesh_global_vertex_indices__empty_processors = list() for r in range(mpi_comm.size): backup_first_marker_id_r = mpi_comm.bcast(backup_first_marker_id, root=r) if backup_first_marker_id_r is None: allgathered_mesh_global_vertex_indices__non_empty_processors.extend(mpi_comm.bcast(mesh_global_vertex_indices, root=r)) else: allgathered_mesh_global_vertex_indices__empty_processors.extend(mpi_comm.bcast(mesh_global_vertex_indices, root=r)) allgathered_mesh_global_vertex_indices__non_empty_processors = sorted(unique(allgathered_mesh_global_vertex_indices__non_empty_processors)) allgathered_mesh_global_vertex_indices__empty_processors = sorted(unique(allgathered_mesh_global_vertex_indices__empty_processors)) # ... then create a dict that will contain the map from mesh global vertex index to submesh global vertex index. # ... Here make sure to number first "real" vertices (those coming from non empty processors), since the other ones # ... are just a side effect of the current partitioning! allgathered_mesh_to_submesh_vertex_global_indices = dict() _submesh_vertex_global_index = 0 for mesh_vertex_global_index in allgathered_mesh_global_vertex_indices__non_empty_processors: assert mesh_vertex_global_index not in allgathered_mesh_to_submesh_vertex_global_indices allgathered_mesh_to_submesh_vertex_global_indices[mesh_vertex_global_index] = _submesh_vertex_global_index _submesh_vertex_global_index += 1 for mesh_vertex_global_index in allgathered_mesh_global_vertex_indices__empty_processors: if mesh_vertex_global_index not in allgathered_mesh_to_submesh_vertex_global_indices: allgathered_mesh_to_submesh_vertex_global_indices[mesh_vertex_global_index] = _submesh_vertex_global_index _submesh_vertex_global_index += 1 # Number cells in a way which is independent from the number of processors. To do so ... # ... first of all collect all cells from all processors allgathered_mesh_global_cell_indices__non_empty_processors = list() allgathered_mesh_global_cell_indices__empty_processors = list() for r in range(mpi_comm.size): backup_first_marker_id_r = mpi_comm.bcast(backup_first_marker_id, root=r) if backup_first_marker_id_r is None: allgathered_mesh_global_cell_indices__non_empty_processors.extend(mpi_comm.bcast(mesh_global_cell_indices, root=r)) else: allgathered_mesh_global_cell_indices__empty_processors.extend(mpi_comm.bcast(mesh_global_cell_indices, root=r)) allgathered_mesh_global_cell_indices__non_empty_processors = sorted(unique(allgathered_mesh_global_cell_indices__non_empty_processors)) allgathered_mesh_global_cell_indices__empty_processors = sorted(unique(allgathered_mesh_global_cell_indices__empty_processors)) # ... then create a dict that will contain the map from mesh global cell index to submesh global cell index. # ... Here make sure to number first "real" vertices (those coming from non empty processors), since the other ones # ... are just a side effect of the current partitioning! allgathered_mesh_to_submesh_cell_global_indices = dict() _submesh_cell_global_index = 0 for mesh_cell_global_index in allgathered_mesh_global_cell_indices__non_empty_processors: assert mesh_cell_global_index not in allgathered_mesh_to_submesh_cell_global_indices allgathered_mesh_to_submesh_cell_global_indices[mesh_cell_global_index] = _submesh_cell_global_index _submesh_cell_global_index += 1 for mesh_cell_global_index in allgathered_mesh_global_cell_indices__empty_processors: assert mesh_cell_global_index not in allgathered_mesh_to_submesh_cell_global_indices allgathered_mesh_to_submesh_cell_global_indices[mesh_cell_global_index] = _submesh_cell_global_index _submesh_cell_global_index += 1 # Also create a mapping from mesh local vertex index to submesh local vertex index. mesh_to_submesh_vertex_local_indices = dict(zip(mesh_vertex_indices, list(range(len(mesh_vertex_indices))))) # Also create a mapping from mesh local cell index to submesh local cell index. mesh_to_submesh_cell_local_indices = dict(zip(mesh_cell_indices, list(range(len(mesh_cell_indices))))) # Now, define submesh cells submesh_cells = list() for i, c in enumerate(mesh_cells): submesh_cells.append([mesh_to_submesh_vertex_local_indices[j] for j in c]) # Store vertices as submesh_vertices[local_index] = (global_index, coordinates) submesh_vertices = dict() for mesh_vertex_local_index, submesh_vertex_local_index in mesh_to_submesh_vertex_local_indices.items(): submesh_vertices[submesh_vertex_local_index] = ( allgathered_mesh_to_submesh_vertex_global_indices[mesh.topology().global_indices(0)[mesh_vertex_local_index]], mesh.coordinates()[mesh_vertex_local_index] ) # Collect the global number of vertices and cells global_num_cells = mpi_comm.allreduce(len(submesh_cells), op=SUM) global_num_vertices = len(allgathered_mesh_to_submesh_vertex_global_indices) # Fill in mesh_editor mesh_editor.init_vertices_global(len(submesh_vertices), global_num_vertices) mesh_editor.init_cells_global(len(submesh_cells), global_num_cells) for local_index, cell_vertices in enumerate(submesh_cells): if has_pybind11(): mesh_editor.add_cell(local_index, cell_vertices) else: mesh_editor.add_cell(local_index, *cell_vertices) for local_index, (global_index, coordinates) in submesh_vertices.items(): mesh_editor.add_vertex_global(local_index, global_index, coordinates) mesh_editor.close() # Initialize topology submesh.topology().init(0, len(submesh_vertices), global_num_vertices) submesh.topology().init(mesh.ufl_cell().topological_dimension(), len(submesh_cells), global_num_cells) # Correct the global index of cells for local_index in range(len(submesh_cells)): submesh.topology().set_global_index( submesh.topology().dim(), local_index, allgathered_mesh_to_submesh_cell_global_indices[mesh_global_cell_indices[local_index]] ) # == 3. Store (local) mesh to/from submesh map for cells, facets and vertices == # # Cells submesh.mesh_to_submesh_cell_local_indices = mesh_to_submesh_cell_local_indices submesh.submesh_to_mesh_cell_local_indices = mesh_cell_indices # Vertices submesh.mesh_to_submesh_vertex_local_indices = mesh_to_submesh_vertex_local_indices submesh.submesh_to_mesh_vertex_local_indices = mesh_vertex_indices # Facets mesh_vertices_to_mesh_facets = dict() mesh_facets_to_mesh_vertices = dict() for mesh_cell_index in mesh_cell_indices: mesh_cell = Cell(mesh, mesh_cell_index) for mesh_facet in facets(mesh_cell): mesh_facet_vertices = list() for mesh_facet_vertex in vertices(mesh_facet): mesh_facet_vertices.append(mesh_facet_vertex.index()) mesh_facet_vertices = tuple(sorted(mesh_facet_vertices)) if mesh_facet_vertices in mesh_vertices_to_mesh_facets: assert mesh_vertices_to_mesh_facets[mesh_facet_vertices] == mesh_facet.index() else: mesh_vertices_to_mesh_facets[mesh_facet_vertices] = mesh_facet.index() if mesh_facet.index() in mesh_facets_to_mesh_vertices: assert mesh_facets_to_mesh_vertices[mesh_facet.index()] == mesh_facet_vertices else: mesh_facets_to_mesh_vertices[mesh_facet.index()] = mesh_facet_vertices submesh_vertices_to_submesh_facets = dict() submesh_facets_to_submesh_vertices = dict() for submesh_facet in facets(submesh): submesh_facet_vertices = list() for submesh_facet_vertex in vertices(submesh_facet): submesh_facet_vertices.append(submesh_facet_vertex.index()) submesh_facet_vertices = tuple(sorted(submesh_facet_vertices)) assert submesh_facet_vertices not in submesh_vertices_to_submesh_facets submesh_vertices_to_submesh_facets[submesh_facet_vertices] = submesh_facet.index() assert submesh_facet.index() not in submesh_facets_to_submesh_vertices submesh_facets_to_submesh_vertices[submesh_facet.index()] = submesh_facet_vertices mesh_to_submesh_facets_local_indices = dict() for (mesh_facet_index, mesh_vertices) in mesh_facets_to_mesh_vertices.items(): submesh_vertices = tuple(sorted([submesh.mesh_to_submesh_vertex_local_indices[mesh_vertex] for mesh_vertex in mesh_vertices])) submesh_facet_index = submesh_vertices_to_submesh_facets[submesh_vertices] mesh_to_submesh_facets_local_indices[mesh_facet_index] = submesh_facet_index submesh_to_mesh_facets_local_indices = dict() for (submesh_facet_index, submesh_vertices) in submesh_facets_to_submesh_vertices.items(): mesh_vertices = tuple(sorted([submesh.submesh_to_mesh_vertex_local_indices[submesh_vertex] for submesh_vertex in submesh_vertices])) mesh_facet_index = mesh_vertices_to_mesh_facets[mesh_vertices] submesh_to_mesh_facets_local_indices[submesh_facet_index] = mesh_facet_index submesh.mesh_to_submesh_facet_local_indices = mesh_to_submesh_facets_local_indices submesh.submesh_to_mesh_facet_local_indices = list() assert min(submesh_to_mesh_facets_local_indices.keys()) == 0 assert max(submesh_to_mesh_facets_local_indices.keys()) == len(submesh_to_mesh_facets_local_indices.keys()) - 1 for submesh_facet_index in range(len(submesh_to_mesh_facets_local_indices)): submesh.submesh_to_mesh_facet_local_indices.append(submesh_to_mesh_facets_local_indices[submesh_facet_index]) # == 3bis. Prepare (temporary) global indices of facets == # # Wrapper to DistributedMeshTools::number_entities if has_pybind11(): cpp_code = """ #include <pybind11/pybind11.h> #include <dolfin/mesh/DistributedMeshTools.h> #include <dolfin/mesh/Mesh.h> void initialize_global_indices(std::shared_ptr<dolfin::Mesh> mesh, std::size_t dim) { dolfin::DistributedMeshTools::number_entities(*mesh, dim); } PYBIND11_MODULE(SIGNATURE, m) { m.def("initialize_global_indices", &initialize_global_indices); } """ initialize_global_indices = compile_cpp_code(cpp_code).initialize_global_indices else: cpp_code = """ void initialize_global_indices(Mesh & mesh, std::size_t dim) { DistributedMeshTools::number_entities(mesh, dim); } """ initialize_global_indices = compile_extension_module(cpp_code, additional_system_headers=["dolfin/mesh/DistributedMeshTools.h"]).initialize_global_indices initialize_global_indices(mesh, mesh.topology().dim() - 1) # Prepare global indices of facets mesh_facets_local_to_global_indices = dict() for mesh_cell_index in mesh_cell_indices: mesh_cell = Cell(mesh, mesh_cell_index) for mesh_facet in facets(mesh_cell): mesh_facets_local_to_global_indices[mesh_facet.index()] = mesh_facet.global_index() mesh_facets_global_indices_in_submesh = list() for mesh_facet_local_index in mesh_to_submesh_facets_local_indices.keys(): mesh_facets_global_indices_in_submesh.append(mesh_facets_local_to_global_indices[mesh_facet_local_index]) allgathered__mesh_facets_global_indices_in_submesh = list() for r in range(mpi_comm.size): allgathered__mesh_facets_global_indices_in_submesh.extend(mpi_comm.bcast(mesh_facets_global_indices_in_submesh, root=r)) allgathered__mesh_facets_global_indices_in_submesh = sorted(set(allgathered__mesh_facets_global_indices_in_submesh)) mesh_to_submesh_facets_global_indices = dict() for (submesh_facet_global_index, mesh_facet_global_index) in enumerate(allgathered__mesh_facets_global_indices_in_submesh): mesh_to_submesh_facets_global_indices[mesh_facet_global_index] = submesh_facet_global_index submesh_facets_local_to_global_indices = dict() for (submesh_facet_local_index, mesh_facet_local_index) in submesh_to_mesh_facets_local_indices.items(): submesh_facets_local_to_global_indices[submesh_facet_local_index] = mesh_to_submesh_facets_global_indices[mesh_facets_local_to_global_indices[mesh_facet_local_index]] # == 4. Assign shared vertices == # shared_entities_dimensions = { "vertex": 0, "facet": submesh.topology().dim() - 1, "cell": submesh.topology().dim() } shared_entities_class = { "vertex": Vertex, "facet": Facet, "cell": Cell } shared_entities_iterator = { "vertex": vertices, "facet": facets, "cell": cells } shared_entities_submesh_global_index_getter = { "vertex": lambda entity: entity.global_index(), "facet": lambda entity: submesh_facets_local_to_global_indices[entity.index()], "cell": lambda entity: entity.global_index() } for entity_type in ["vertex", "facet", "cell"]: # do not use .keys() because the order is important dim = shared_entities_dimensions[entity_type] class_ = shared_entities_class[entity_type] iterator = shared_entities_iterator[entity_type] submesh_global_index_getter = shared_entities_submesh_global_index_getter[entity_type] # Get shared entities from mesh. A subset of these will end being shared entities also the submesh # (thanks to the fact that we do not redistribute cells from one processor to another) if mpi_comm.size > 1: # some entities may not be initialized in serial, since they are not needed assert mesh.topology().have_shared_entities(dim), "Mesh shared entities have not been initialized for dimension " + str(dim) if mesh.topology().have_shared_entities(dim): # always true in parallel (when really needed) # However, it may happen that an entity which has been selected is not shared anymore because only one of # the sharing processes has it in the submesh. For instance, consider the case # of two cells across the interface (located on a facet f) between two processors. It may happen that # only one of the two cells is selected: the facet f and its vertices are not shared anymore! # For this reason, we create a new dict from global entity index to processors sharing them. Thus ... # ... first of all get global indices corresponding to local entities if entity_type in ["vertex", "cell"]: assert submesh.topology().have_global_indices(dim), "Submesh global indices have not been initialized for dimension " + str(dim) submesh_local_entities_global_index = list() submesh_local_entities_global_to_local_index = dict() for entity in iterator(submesh): local_entity_index = entity.index() global_entity_index = submesh_global_index_getter(entity) submesh_local_entities_global_index.append(global_entity_index) submesh_local_entities_global_to_local_index[global_entity_index] = local_entity_index # ... then gather all global indices from all processors gathered__submesh_local_entities_global_index = list() # over processor id for r in range(mpi_comm.size): gathered__submesh_local_entities_global_index.append(mpi_comm.bcast(submesh_local_entities_global_index, root=r)) # ... then create dict from global index to processors sharing it submesh_shared_entities__global = dict() for r in range(mpi_comm.size): for global_entity_index in gathered__submesh_local_entities_global_index[r]: if global_entity_index not in submesh_shared_entities__global: submesh_shared_entities__global[global_entity_index] = list() submesh_shared_entities__global[global_entity_index].append(r) # ... and finally popuplate shared entities dict, which is the same as the dict above except that # the current processor rank is removed and a local indexing is used submesh_shared_entities = dict() # from local index to list of integers for (global_entity_index, processors) in submesh_shared_entities__global.items(): if ( mpi_comm.rank in processors # only local entities and len(processors) > 1 # it was still shared after submesh extraction ): other_processors_list = list(processors) other_processors_list.remove(mpi_comm.rank) other_processors = array(other_processors_list, dtype=uintp) submesh_shared_entities[submesh_local_entities_global_to_local_index[global_entity_index]] = other_processors # Need an extension module to populate shared_entities because in python each call to shared_entities # returns a temporary. if has_pybind11(): cpp_code = """ #include <Eigen/Core> #include <pybind11/pybind11.h> #include <pybind11/eigen.h> #include <dolfin/mesh/Mesh.h> using OtherProcesses = Eigen::Ref<const Eigen::Matrix<std::size_t, Eigen::Dynamic, 1>>; void set_shared_entities(std::shared_ptr<dolfin::Mesh> submesh, std::size_t idx, const OtherProcesses other_processes, std::size_t dim) { std::set<unsigned int> set_other_processes; for (std::size_t i(0); i < other_processes.size(); i++) set_other_processes.insert(other_processes[i]); submesh->topology().shared_entities(dim)[idx] = set_other_processes; } PYBIND11_MODULE(SIGNATURE, m) { m.def("set_shared_entities", &set_shared_entities); } """ set_shared_entities = compile_cpp_code(cpp_code).set_shared_entities else: cpp_code = """ void set_shared_entities(Mesh & submesh, std::size_t idx, const Array<std::size_t>& other_processes, std::size_t dim) { std::set<unsigned int> set_other_processes; for (std::size_t i(0); i < other_processes.size(); i++) set_other_processes.insert(other_processes[i]); submesh.topology().shared_entities(dim)[idx] = set_other_processes; } """ set_shared_entities = compile_extension_module(cpp_code).set_shared_entities for (submesh_entity_local_index, other_processors) in submesh_shared_entities.items(): set_shared_entities(submesh, submesh_entity_local_index, other_processors, dim) log(DEBUG, "Local indices of shared entities for dimension " + str(dim) + ": " + str(list(submesh.topology().shared_entities(0).keys()))) log(DEBUG, "Global indices of shared entities for dimension " + str(dim) + ": " + str([class_(submesh, local_index).global_index() for local_index in submesh.topology().shared_entities(dim).keys()])) # == 5. Also initialize submesh facets global indices, now that shared facets have been computed == # initialize_global_indices(submesh, submesh.topology().dim() - 1) # note that DOLFIN might change the numbering when compared to the one at 3bis # == 6. Restore backup_first_marker_id and return == # if backup_first_marker_id is not None: markers.array()[0] = backup_first_marker_id return submesh