def create_task_dof_maps(field,
cell_tasks,
inter_facets,
is_overlap=True,
use_expand_dofs=False,
save_inter_regions=False,
output_dir=None):
"""
For each task list its inner and interface DOFs of the given field and
create PETSc numbering that is consecutive in each subdomain.
For each task, the DOF map has the following structure::
[inner,
[own_inter1, own_inter2, ...],
[overlap_cells1, overlap_cells2, ...],
n_task_total, task_offset]
The overlapping cells are defined so that the system matrix corresponding
to each task can be assembled independently, see [1]. TODO: Some "corner"
cells may be added even if not needed - filter them out by using the PETSc
DOFs range.
When debugging domain partitioning problems, it is advisable to set
`save_inter_regions` to True to save the task interfaces as meshes as well
as vertex-based markers - to be used only with moderate problems and small
numbers of tasks.
[1] J. Sistek and F. Cirak. Parallel iterative solution of the
incompressible Navier-Stokes equations with application to rotating
wings. Submitted for publication, 2015
"""
domain = field.domain
cmesh = domain.cmesh
if use_expand_dofs:
id_map = nm.zeros(field.n_nod * field.n_components, dtype=nm.uint32)
else:
id_map = nm.zeros(field.n_nod, dtype=nm.uint32)
def _get_dofs_region(field, region):
dofs = field.get_dofs_in_region(region)
if use_expand_dofs:
dofs = expand_dofs(dofs, field.n_components)
return dofs
def _get_dofs_conn(field, conn):
dofs = nm.unique(conn)
if use_expand_dofs:
dofs = expand_dofs(dofs, field.n_components)
return dofs
dof_maps = {}
count = 0
inter_count = 0
ocs = nm.zeros(0, dtype=nm.int32)
cell_parts = []
for ir, ntasks in ordered_iteritems(inter_facets):
cells = nm.where(cell_tasks == ir)[0].astype(nm.int32)
cell_parts.append(cells)
cregion = Region.from_cells(cells, domain, name='task_%d' % ir)
domain.regions.append(cregion)
dofs = _get_dofs_region(field, cregion)
rdof_map = dof_maps.setdefault(ir, [None, [], [], 0, 0])
inter_dofs = []
for ic, facets in ordered_iteritems(ntasks):
cdof_map = dof_maps.setdefault(ic, [None, [], [], 0, 0])
name = 'inter_%d_%d' % (ir, ic)
ii = ir
region = Region.from_facets(facets,
domain,
name,
parent=cregion.name)
region.update_shape()
if save_inter_regions:
output_dir = output_dir if output_dir is not None else '.'
filename = os.path.join(output_dir, '%s.mesh' % name)
aux = domain.mesh.from_region(region,
domain.mesh,
is_surface=True)
aux.write(filename, io='auto')
mask = nm.zeros((domain.mesh.n_nod, 1), dtype=nm.float64)
mask[region.vertices] = 1
out = {name: Struct(name=name, mode='vertex', data=mask)}
filename = os.path.join(output_dir, '%s.h5' % name)
domain.mesh.write(filename, out=out, io='auto')
inter_dofs.append(_get_dofs_region(field, region))
sd = FESurface('surface_data_%s' % region.name, region,
field.efaces, field.econn, field.region)
econn = sd.get_connectivity()
n_facet = econn.shape[0]
ii2 = max(int(n_facet / 2), 1)
dr = _get_dofs_conn(field, econn[:ii2])
ii = nm.where((id_map[dr] == 0))[0]
n_new = len(ii)
if n_new:
rdof_map[1].append(dr[ii])
rdof_map[3] += n_new
id_map[dr[ii]] = 1
inter_count += n_new
count += n_new
if is_overlap:
ovs = cmesh.get_incident(0, region.facets[:ii2],
cmesh.tdim - 1)
ocs = cmesh.get_incident(cmesh.tdim, ovs, 0)
rdof_map[2].append(ocs.astype(nm.int32))
dc = _get_dofs_conn(field, econn[ii2:])
ii = nm.where((id_map[dc] == 0))[0]
n_new = len(ii)
if n_new:
cdof_map[1].append(dc[ii])
cdof_map[3] += n_new
id_map[dc[ii]] = 1
inter_count += n_new
count += n_new
if is_overlap:
ovs = cmesh.get_incident(0, region.facets[ii2:],
cmesh.tdim - 1)
ocs = cmesh.get_incident(cmesh.tdim, ovs, 0)
cdof_map[2].append(ocs.astype(nm.int32))
domain.regions.pop() # Remove the cell region.
inner_dofs = nm.setdiff1d(dofs, nm.concatenate(inter_dofs))
n_inner = len(inner_dofs)
rdof_map[3] += n_inner
assert_(nm.all(id_map[inner_dofs] == 0))
id_map[inner_dofs] = 1
count += n_inner
rdof_map[0] = inner_dofs
offset = 0
overlap_cells = []
for ir, dof_map in ordered_iteritems(dof_maps):
n_owned = dof_map[3]
i0 = len(dof_map[0])
id_map[dof_map[0]] = nm.arange(offset, offset + i0, dtype=nm.uint32)
for aux in dof_map[1]:
i1 = len(aux)
id_map[aux] = nm.arange(offset + i0,
offset + i0 + i1,
dtype=nm.uint32)
i0 += i1
if len(dof_map[2]):
ocs = nm.unique(nm.concatenate(dof_map[2]))
else:
ocs = nm.zeros(0, dtype=nm.int32)
overlap_cells.append(ocs)
assert_(i0 == n_owned)
dof_map[4] = offset
offset += n_owned
if not len(dof_maps):
dofs = _get_dofs_region(field, field.region)
dof_maps[0] = [dofs, [], [], len(dofs), 0]
id_map[:] = nm.arange(len(dofs), dtype=nm.uint32)
if not len(cell_parts):
cell_parts.append(nm.arange(len(cell_tasks), dtype=nm.int32))
if not len(overlap_cells):
overlap_cells.append(nm.zeros(0, dtype=nm.int32))
return dof_maps, id_map, cell_parts, overlap_cells
def create_task_dof_maps(field, cell_tasks, inter_facets, is_overlap=True,
use_expand_dofs=False, save_inter_regions=False,
output_dir=None):
"""
For each task list its inner and interface DOFs of the given field and
create PETSc numbering that is consecutive in each subdomain.
For each task, the DOF map has the following structure::
[inner,
[own_inter1, own_inter2, ...],
[overlap_cells1, overlap_cells2, ...],
n_task_total, task_offset]
The overlapping cells are defined so that the system matrix corresponding
to each task can be assembled independently, see [1]. TODO: Some "corner"
cells may be added even if not needed - filter them out by using the PETSc
DOFs range.
When debugging domain partitioning problems, it is advisable to set
`save_inter_regions` to True to save the task interfaces as meshes as well
as vertex-based markers - to be used only with moderate problems and small
numbers of tasks.
[1] J. Sistek and F. Cirak. Parallel iterative solution of the
incompressible Navier-Stokes equations with application to rotating
wings. Submitted for publication, 2015
"""
domain = field.domain
cmesh = domain.cmesh
if use_expand_dofs:
id_map = nm.zeros(field.n_nod * field.n_components, dtype=nm.uint32)
else:
id_map = nm.zeros(field.n_nod, dtype=nm.uint32)
def _get_dofs_region(field, region):
dofs = field.get_dofs_in_region(region)
if use_expand_dofs:
dofs = expand_dofs(dofs, field.n_components)
return dofs
def _get_dofs_conn(field, conn):
dofs = nm.unique(conn)
if use_expand_dofs:
dofs = expand_dofs(dofs, field.n_components)
return dofs
dof_maps = {}
count = 0
inter_count = 0
ocs = nm.zeros(0, dtype=nm.int32)
cell_parts = []
for ir, ntasks in ordered_iteritems(inter_facets):
cells = nm.where(cell_tasks == ir)[0].astype(nm.int32)
cell_parts.append(cells)
cregion = Region.from_cells(cells, domain, name='task_%d' % ir)
domain.regions.append(cregion)
dofs = _get_dofs_region(field, cregion)
rdof_map = dof_maps.setdefault(ir, [None, [], [], 0, 0])
inter_dofs = []
for ic, facets in ordered_iteritems(ntasks):
cdof_map = dof_maps.setdefault(ic, [None, [], [], 0, 0])
name = 'inter_%d_%d' % (ir, ic)
ii = ir
region = Region.from_facets(facets, domain, name,
parent=cregion.name)
region.update_shape()
if save_inter_regions:
output_dir = output_dir if output_dir is not None else '.'
filename = os.path.join(output_dir, '%s.mesh' % name)
aux = domain.mesh.from_region(region, domain.mesh,
is_surface=True)
aux.write(filename, io='auto')
mask = nm.zeros((domain.mesh.n_nod, 1), dtype=nm.float64)
mask[region.vertices] = 1
out = {name : Struct(name=name, mode='vertex', data=mask)}
filename = os.path.join(output_dir, '%s.h5' % name)
domain.mesh.write(filename, out=out, io='auto')
inter_dofs.append(_get_dofs_region(field, region))
sd = FESurface('surface_data_%s' % region.name, region,
field.efaces, field.econn, field.region)
econn = sd.get_connectivity()
n_facet = econn.shape[0]
ii2 = max(int(n_facet / 2), 1)
dr = _get_dofs_conn(field, econn[:ii2])
ii = nm.where((id_map[dr] == 0))[0]
n_new = len(ii)
if n_new:
rdof_map[1].append(dr[ii])
rdof_map[3] += n_new
id_map[dr[ii]] = 1
inter_count += n_new
count += n_new
if is_overlap:
ovs = cmesh.get_incident(0, region.facets[:ii2],
cmesh.tdim - 1)
ocs = cmesh.get_incident(cmesh.tdim, ovs, 0)
rdof_map[2].append(ocs.astype(nm.int32))
dc = _get_dofs_conn(field, econn[ii2:])
ii = nm.where((id_map[dc] == 0))[0]
n_new = len(ii)
if n_new:
cdof_map[1].append(dc[ii])
cdof_map[3] += n_new
id_map[dc[ii]] = 1
inter_count += n_new
count += n_new
if is_overlap:
ovs = cmesh.get_incident(0, region.facets[ii2:],
cmesh.tdim - 1)
ocs = cmesh.get_incident(cmesh.tdim, ovs, 0)
cdof_map[2].append(ocs.astype(nm.int32))
domain.regions.pop() # Remove the cell region.
inner_dofs = nm.setdiff1d(dofs, nm.concatenate(inter_dofs))
n_inner = len(inner_dofs)
rdof_map[3] += n_inner
assert_(nm.all(id_map[inner_dofs] == 0))
id_map[inner_dofs] = 1
count += n_inner
rdof_map[0] = inner_dofs
offset = 0
overlap_cells = []
for ir, dof_map in ordered_iteritems(dof_maps):
n_owned = dof_map[3]
i0 = len(dof_map[0])
id_map[dof_map[0]] = nm.arange(offset, offset + i0, dtype=nm.uint32)
for aux in dof_map[1]:
i1 = len(aux)
id_map[aux] = nm.arange(offset + i0, offset + i0 + i1,
dtype=nm.uint32)
i0 += i1
if len(dof_map[2]):
ocs = nm.unique(nm.concatenate(dof_map[2]))
else:
ocs = nm.zeros(0, dtype=nm.int32)
overlap_cells.append(ocs)
assert_(i0 == n_owned)
dof_map[4] = offset
offset += n_owned
if not len(dof_maps):
dofs = _get_dofs_region(field, field.region)
dof_maps[0] = [dofs, [], [], len(dofs), 0]
id_map[:] = nm.arange(len(dofs), dtype=nm.uint32)
if not len(cell_parts):
cell_parts.append(nm.arange(len(cell_tasks), dtype=nm.int32))
if not len(overlap_cells):
overlap_cells.append(nm.zeros(0, dtype=nm.int32))
return dof_maps, id_map, cell_parts, overlap_cells
