def solve_problem(mesh_filename, options, comm): order_u = options.order_u order_p = options.order_p rank, size = comm.Get_rank(), comm.Get_size() output('rank', rank, 'of', size) stats = Struct() timer = Timer('solve_timer') timer.start() mesh = Mesh.from_file(mesh_filename) stats.t_read_mesh = timer.stop() timer.start() if rank == 0: cell_tasks = pl.partition_mesh(mesh, size, use_metis=options.metis, verbose=True) else: cell_tasks = None stats.t_partition_mesh = timer.stop() output('creating global domain and fields...') timer.start() domain = FEDomain('domain', mesh) omega = domain.create_region('Omega', 'all') field1 = Field.from_args('fu', nm.float64, mesh.dim, omega, approx_order=order_u) field2 = Field.from_args('fp', nm.float64, 1, omega, approx_order=order_p) fields = [field1, field2] stats.t_create_global_fields = timer.stop() output('...done in', timer.dt) output('distributing fields...') timer.start() distribute = pl.distribute_fields_dofs lfds, gfds = distribute(fields, cell_tasks, is_overlap=True, use_expand_dofs=True, save_inter_regions=options.save_inter_regions, output_dir=options.output_dir, comm=comm, verbose=True) stats.t_distribute_fields_dofs = timer.stop() output('...done in', timer.dt) output('creating local problem...') timer.start() cells = lfds[0].cells omega_gi = Region.from_cells(cells, domain) omega_gi.finalize() omega_gi.update_shape() pb = create_local_problem(omega_gi, [order_u, order_p]) variables = pb.get_variables() state = State(variables) state.fill(0.0) state.apply_ebc() stats.t_create_local_problem = timer.stop() output('...done in', timer.dt) output('allocating global system...') timer.start() sizes, drange, pdofs = pl.setup_composite_dofs(lfds, fields, variables, verbose=True) pmtx, psol, prhs = pl.create_petsc_system(pb.mtx_a, sizes, pdofs, drange, is_overlap=True, comm=comm, verbose=True) stats.t_allocate_global_system = timer.stop() output('...done in', timer.dt) output('creating solver...') timer.start() conf = Struct(method='bcgsl', precond='jacobi', sub_precond='none', i_max=10000, eps_a=1e-50, eps_r=1e-6, eps_d=1e4, verbose=True) status = {} ls = PETScKrylovSolver(conf, comm=comm, mtx=pmtx, status=status) field_ranges = {} for ii, variable in enumerate(variables.iter_state(ordered=True)): field_ranges[variable.name] = lfds[ii].petsc_dofs_range ls.set_field_split(field_ranges, comm=comm) ev = PETScParallelEvaluator(pb, pdofs, drange, True, psol, comm, verbose=True) nls_status = {} conf = Struct(method='newtonls', i_max=5, eps_a=0, eps_r=1e-5, eps_s=0.0, verbose=True) nls = PETScNonlinearSolver(conf, pmtx=pmtx, prhs=prhs, comm=comm, fun=ev.eval_residual, fun_grad=ev.eval_tangent_matrix, lin_solver=ls, status=nls_status) stats.t_create_solver = timer.stop() output('...done in', timer.dt) output('solving...') timer.start() state = pb.create_state() state.apply_ebc() ev.psol_i[...] = state() ev.gather(psol, ev.psol_i) psol = nls(psol) ev.scatter(ev.psol_i, psol) sol0_i = ev.psol_i[...] stats.t_solve = timer.stop() output('...done in', timer.dt) output('saving solution...') timer.start() state.set_full(sol0_i) out = state.create_output_dict() filename = os.path.join(options.output_dir, 'sol_%02d.h5' % comm.rank) pb.domain.mesh.write(filename, io='auto', out=out) gather_to_zero = pl.create_gather_to_zero(psol) psol_full = gather_to_zero(psol) if comm.rank == 0: sol = psol_full[...].copy() u = FieldVariable('u', 'parameter', field1, primary_var_name='(set-to-None)') remap = gfds[0].id_map ug = sol[remap] p = FieldVariable('p', 'parameter', field2, primary_var_name='(set-to-None)') remap = gfds[1].id_map pg = sol[remap] if (((order_u == 1) and (order_p == 1)) or (options.linearization == 'strip')): out = u.create_output(ug) out.update(p.create_output(pg)) filename = os.path.join(options.output_dir, 'sol.h5') mesh.write(filename, io='auto', out=out) else: out = u.create_output(ug, linearization=Struct(kind='adaptive', min_level=0, max_level=order_u, eps=1e-3)) filename = os.path.join(options.output_dir, 'sol_u.h5') out['u'].mesh.write(filename, io='auto', out=out) out = p.create_output(pg, linearization=Struct(kind='adaptive', min_level=0, max_level=order_p, eps=1e-3)) filename = os.path.join(options.output_dir, 'sol_p.h5') out['p'].mesh.write(filename, io='auto', out=out) stats.t_save_solution = timer.stop() output('...done in', timer.dt) stats.t_total = timer.total stats.n_dof = sizes[1] stats.n_dof_local = sizes[0] stats.n_cell = omega.shape.n_cell stats.n_cell_local = omega_gi.shape.n_cell return stats
def solve_problem(mesh_filename, options, comm): order_u = options.order_u order_p = options.order_p rank, size = comm.Get_rank(), comm.Get_size() output('rank', rank, 'of', size) mesh = Mesh.from_file(mesh_filename) if rank == 0: cell_tasks = pl.partition_mesh(mesh, size, use_metis=options.metis, verbose=True) else: cell_tasks = None domain = FEDomain('domain', mesh) omega = domain.create_region('Omega', 'all') field1 = Field.from_args('fu', nm.float64, mesh.dim, omega, approx_order=order_u) field2 = Field.from_args('fp', nm.float64, 1, omega, approx_order=order_p) fields = [field1, field2] output('distributing fields...') tt = time.clock() lfds, gfds = pl.distribute_fields_dofs(fields, cell_tasks, is_overlap=True, use_expand_dofs=True, comm=comm, verbose=True) output('...done in', time.clock() - tt) output('creating local problem...') tt = time.clock() cells = lfds[0].cells omega_gi = Region.from_cells(cells, domain) omega_gi.finalize() omega_gi.update_shape() pb = create_local_problem(omega_gi, [order_u, order_p]) variables = pb.get_variables() state = State(variables) state.fill(0.0) state.apply_ebc() output('...done in', time.clock() - tt) output('allocating global system...') tt = time.clock() sizes, drange, pdofs = pl.setup_composite_dofs(lfds, fields, variables, verbose=True) pmtx, psol, prhs = pl.create_petsc_system(pb.mtx_a, sizes, pdofs, drange, is_overlap=True, comm=comm, verbose=True) output('...done in', time.clock() - tt) output('creating solver...') tt = time.clock() conf = Struct(method='bcgsl', precond='jacobi', sub_precond=None, i_max=10000, eps_a=1e-50, eps_r=1e-6, eps_d=1e4, verbose=True) status = {} ls = PETScKrylovSolver(conf, comm=comm, mtx=pmtx, status=status) field_ranges = {} for ii, variable in enumerate(variables.iter_state(ordered=True)): field_ranges[variable.name] = lfds[ii].petsc_dofs_range ls.set_field_split(field_ranges, comm=comm) ev = PETScParallelEvaluator(pb, pdofs, drange, True, psol, comm, verbose=True) nls_status = {} conf = Struct(method='newtonls', i_max=5, eps_a=0, eps_r=1e-5, eps_s=0.0, verbose=True) nls = PETScNonlinearSolver(conf, pmtx=pmtx, prhs=prhs, comm=comm, fun=ev.eval_residual, fun_grad=ev.eval_tangent_matrix, lin_solver=ls, status=nls_status) output('...done in', time.clock() - tt) output('solving...') tt = time.clock() state = pb.create_state() state.apply_ebc() ev.psol_i[...] = state() ev.gather(psol, ev.psol_i) psol = nls(psol) ev.scatter(ev.psol_i, psol) sol0_i = ev.psol_i[...] output('...done in', time.clock() - tt) output('saving solution...') tt = time.clock() state.set_full(sol0_i) out = state.create_output_dict() filename = os.path.join(options.output_dir, 'sol_%02d.h5' % comm.rank) pb.domain.mesh.write(filename, io='auto', out=out) gather_to_zero = pl.create_gather_to_zero(psol) psol_full = gather_to_zero(psol) if comm.rank == 0: sol = psol_full[...].copy() u = FieldVariable('u', 'parameter', field1, primary_var_name='(set-to-None)') remap = gfds[0].id_map ug = sol[remap] p = FieldVariable('p', 'parameter', field2, primary_var_name='(set-to-None)') remap = gfds[1].id_map pg = sol[remap] if (((order_u == 1) and (order_p == 1)) or (options.linearization == 'strip')): out = u.create_output(ug) out.update(p.create_output(pg)) filename = os.path.join(options.output_dir, 'sol.h5') mesh.write(filename, io='auto', out=out) else: out = u.create_output(ug, linearization=Struct(kind='adaptive', min_level=0, max_level=order_u, eps=1e-3)) filename = os.path.join(options.output_dir, 'sol_u.h5') out['u'].mesh.write(filename, io='auto', out=out) out = p.create_output(pg, linearization=Struct(kind='adaptive', min_level=0, max_level=order_p, eps=1e-3)) filename = os.path.join(options.output_dir, 'sol_p.h5') out['p'].mesh.write(filename, io='auto', out=out) output('...done in', time.clock() - tt)
def solve_problem(mesh_filename, options, comm): order = options.order rank, size = comm.Get_rank(), comm.Get_size() output('rank', rank, 'of', size) mesh = Mesh.from_file(mesh_filename) if rank == 0: cell_tasks = pl.partition_mesh(mesh, size, use_metis=options.metis, verbose=True) else: cell_tasks = None domain = FEDomain('domain', mesh) omega = domain.create_region('Omega', 'all') field = Field.from_args('fu', nm.float64, 1, omega, approx_order=order) output('distributing field %s...' % field.name) tt = time.clock() distribute = pl.distribute_fields_dofs lfds, gfds = distribute([field], cell_tasks, is_overlap=True, save_inter_regions=options.save_inter_regions, output_dir=options.output_dir, comm=comm, verbose=True) lfd = lfds[0] output('...done in', time.clock() - tt) if rank == 0: dof_maps = gfds[0].dof_maps id_map = gfds[0].id_map if options.verify: verify_save_dof_maps(field, cell_tasks, dof_maps, id_map, options, verbose=True) if options.plot: ppd.plot_partitioning([None, None], field, cell_tasks, gfds[0], options.output_dir, size) output('creating local problem...') tt = time.clock() omega_gi = Region.from_cells(lfd.cells, field.domain) omega_gi.finalize() omega_gi.update_shape() pb = create_local_problem(omega_gi, order) output('...done in', time.clock() - tt) variables = pb.get_variables() eqs = pb.equations u_i = variables['u_i'] field_i = u_i.field if options.plot: ppd.plot_local_dofs([None, None], field, field_i, omega_gi, options.output_dir, rank) output('allocating global system...') tt = time.clock() sizes, drange = pl.get_sizes(lfd.petsc_dofs_range, field.n_nod, 1) output('sizes:', sizes) output('drange:', drange) pdofs = pl.get_local_ordering(field_i, lfd.petsc_dofs_conn) output('pdofs:', pdofs) pmtx, psol, prhs = pl.create_petsc_system(pb.mtx_a, sizes, pdofs, drange, is_overlap=True, comm=comm, verbose=True) output('...done in', time.clock() - tt) output('evaluating local problem...') tt = time.clock() state = State(variables) state.fill(0.0) state.apply_ebc() rhs_i = eqs.eval_residuals(state()) # This must be after pl.create_petsc_system() call! mtx_i = eqs.eval_tangent_matrices(state(), pb.mtx_a) output('...done in', time.clock() - tt) output('assembling global system...') tt = time.clock() pl.apply_ebc_to_matrix(mtx_i, u_i.eq_map.eq_ebc) pl.assemble_rhs_to_petsc(prhs, rhs_i, pdofs, drange, is_overlap=True, comm=comm, verbose=True) pl.assemble_mtx_to_petsc(pmtx, mtx_i, pdofs, drange, is_overlap=True, comm=comm, verbose=True) output('...done in', time.clock() - tt) output('creating solver...') tt = time.clock() conf = Struct(method='cg', precond='gamg', sub_precond=None, i_max=10000, eps_a=1e-50, eps_r=1e-5, eps_d=1e4, verbose=True) status = {} ls = PETScKrylovSolver(conf, comm=comm, mtx=pmtx, status=status) output('...done in', time.clock() - tt) output('solving...') tt = time.clock() psol = ls(prhs, psol, conf) psol_i = pl.create_local_petsc_vector(pdofs) gather, scatter = pl.create_gather_scatter(pdofs, psol_i, psol, comm=comm) scatter(psol_i, psol) sol0_i = state() - psol_i[...] psol_i[...] = sol0_i gather(psol, psol_i) output('...done in', time.clock() - tt) output('saving solution...') tt = time.clock() u_i.set_data(sol0_i) out = u_i.create_output() filename = os.path.join(options.output_dir, 'sol_%02d.h5' % comm.rank) pb.domain.mesh.write(filename, io='auto', out=out) gather_to_zero = pl.create_gather_to_zero(psol) psol_full = gather_to_zero(psol) if comm.rank == 0: sol = psol_full[...].copy()[id_map] u = FieldVariable('u', 'parameter', field, primary_var_name='(set-to-None)') filename = os.path.join(options.output_dir, 'sol.h5') if (order == 1) or (options.linearization == 'strip'): out = u.create_output(sol) mesh.write(filename, io='auto', out=out) else: out = u.create_output(sol, linearization=Struct(kind='adaptive', min_level=0, max_level=order, eps=1e-3)) out['u'].mesh.write(filename, io='auto', out=out) output('...done in', time.clock() - tt) if options.show: plt.show()
def create_task_dof_maps(field, cell_tasks, inter_facets, is_overlap=True, use_expand_dofs=False, save_inter_regions=False, output_dir=None): """ For each task list its inner and interface DOFs of the given field and create PETSc numbering that is consecutive in each subdomain. For each task, the DOF map has the following structure:: [inner, [own_inter1, own_inter2, ...], [overlap_cells1, overlap_cells2, ...], n_task_total, task_offset] The overlapping cells are defined so that the system matrix corresponding to each task can be assembled independently, see [1]. TODO: Some "corner" cells may be added even if not needed - filter them out by using the PETSc DOFs range. When debugging domain partitioning problems, it is advisable to set `save_inter_regions` to True to save the task interfaces as meshes as well as vertex-based markers - to be used only with moderate problems and small numbers of tasks. [1] J. Sistek and F. Cirak. Parallel iterative solution of the incompressible Navier-Stokes equations with application to rotating wings. Submitted for publication, 2015 """ domain = field.domain cmesh = domain.cmesh if use_expand_dofs: id_map = nm.zeros(field.n_nod * field.n_components, dtype=nm.uint32) else: id_map = nm.zeros(field.n_nod, dtype=nm.uint32) def _get_dofs_region(field, region): dofs = field.get_dofs_in_region(region) if use_expand_dofs: dofs = expand_dofs(dofs, field.n_components) return dofs def _get_dofs_conn(field, conn): dofs = nm.unique(conn) if use_expand_dofs: dofs = expand_dofs(dofs, field.n_components) return dofs dof_maps = {} count = 0 inter_count = 0 ocs = nm.zeros(0, dtype=nm.int32) cell_parts = [] for ir, ntasks in ordered_iteritems(inter_facets): cells = nm.where(cell_tasks == ir)[0].astype(nm.int32) cell_parts.append(cells) cregion = Region.from_cells(cells, domain, name='task_%d' % ir) domain.regions.append(cregion) dofs = _get_dofs_region(field, cregion) rdof_map = dof_maps.setdefault(ir, [None, [], [], 0, 0]) inter_dofs = [] for ic, facets in ordered_iteritems(ntasks): cdof_map = dof_maps.setdefault(ic, [None, [], [], 0, 0]) name = 'inter_%d_%d' % (ir, ic) ii = ir region = Region.from_facets(facets, domain, name, parent=cregion.name) region.update_shape() if save_inter_regions: output_dir = output_dir if output_dir is not None else '.' filename = os.path.join(output_dir, '%s.mesh' % name) aux = domain.mesh.from_region(region, domain.mesh, is_surface=True) aux.write(filename, io='auto') mask = nm.zeros((domain.mesh.n_nod, 1), dtype=nm.float64) mask[region.vertices] = 1 out = {name: Struct(name=name, mode='vertex', data=mask)} filename = os.path.join(output_dir, '%s.h5' % name) domain.mesh.write(filename, out=out, io='auto') inter_dofs.append(_get_dofs_region(field, region)) sd = FESurface('surface_data_%s' % region.name, region, field.efaces, field.econn, field.region) econn = sd.get_connectivity() n_facet = econn.shape[0] ii2 = max(int(n_facet / 2), 1) dr = _get_dofs_conn(field, econn[:ii2]) ii = nm.where((id_map[dr] == 0))[0] n_new = len(ii) if n_new: rdof_map[1].append(dr[ii]) rdof_map[3] += n_new id_map[dr[ii]] = 1 inter_count += n_new count += n_new if is_overlap: ovs = cmesh.get_incident(0, region.facets[:ii2], cmesh.tdim - 1) ocs = cmesh.get_incident(cmesh.tdim, ovs, 0) rdof_map[2].append(ocs.astype(nm.int32)) dc = _get_dofs_conn(field, econn[ii2:]) ii = nm.where((id_map[dc] == 0))[0] n_new = len(ii) if n_new: cdof_map[1].append(dc[ii]) cdof_map[3] += n_new id_map[dc[ii]] = 1 inter_count += n_new count += n_new if is_overlap: ovs = cmesh.get_incident(0, region.facets[ii2:], cmesh.tdim - 1) ocs = cmesh.get_incident(cmesh.tdim, ovs, 0) cdof_map[2].append(ocs.astype(nm.int32)) domain.regions.pop() # Remove the cell region. inner_dofs = nm.setdiff1d(dofs, nm.concatenate(inter_dofs)) n_inner = len(inner_dofs) rdof_map[3] += n_inner assert_(nm.all(id_map[inner_dofs] == 0)) id_map[inner_dofs] = 1 count += n_inner rdof_map[0] = inner_dofs offset = 0 overlap_cells = [] for ir, dof_map in ordered_iteritems(dof_maps): n_owned = dof_map[3] i0 = len(dof_map[0]) id_map[dof_map[0]] = nm.arange(offset, offset + i0, dtype=nm.uint32) for aux in dof_map[1]: i1 = len(aux) id_map[aux] = nm.arange(offset + i0, offset + i0 + i1, dtype=nm.uint32) i0 += i1 if len(dof_map[2]): ocs = nm.unique(nm.concatenate(dof_map[2])) else: ocs = nm.zeros(0, dtype=nm.int32) overlap_cells.append(ocs) assert_(i0 == n_owned) dof_map[4] = offset offset += n_owned if not len(dof_maps): dofs = _get_dofs_region(field, field.region) dof_maps[0] = [dofs, [], [], len(dofs), 0] id_map[:] = nm.arange(len(dofs), dtype=nm.uint32) if not len(cell_parts): cell_parts.append(nm.arange(len(cell_tasks), dtype=nm.int32)) if not len(overlap_cells): overlap_cells.append(nm.zeros(0, dtype=nm.int32)) return dof_maps, id_map, cell_parts, overlap_cells
def solve_problem(mesh_filename, options, comm): order = options.order rank, size = comm.Get_rank(), comm.Get_size() output('rank', rank, 'of', size) mesh = Mesh.from_file(mesh_filename) if rank == 0: cell_tasks = pl.partition_mesh(mesh, size, use_metis=options.metis, verbose=True) else: cell_tasks = None output('creating global domain and field...') tt = time.clock() domain = FEDomain('domain', mesh) omega = domain.create_region('Omega', 'all') field = Field.from_args('fu', nm.float64, 1, omega, approx_order=order) output('...done in', time.clock() - tt) output('distributing field %s...' % field.name) tt = time.clock() distribute = pl.distribute_fields_dofs lfds, gfds = distribute([field], cell_tasks, is_overlap=True, save_inter_regions=options.save_inter_regions, output_dir=options.output_dir, comm=comm, verbose=True) lfd = lfds[0] output('...done in', time.clock() - tt) if rank == 0: dof_maps = gfds[0].dof_maps id_map = gfds[0].id_map if options.verify: verify_save_dof_maps(field, cell_tasks, dof_maps, id_map, options, verbose=True) if options.plot: ppd.plot_partitioning([None, None], field, cell_tasks, gfds[0], options.output_dir, size) output('creating local problem...') tt = time.clock() omega_gi = Region.from_cells(lfd.cells, field.domain) omega_gi.finalize() omega_gi.update_shape() pb = create_local_problem(omega_gi, order) output('...done in', time.clock() - tt) variables = pb.get_variables() eqs = pb.equations u_i = variables['u_i'] field_i = u_i.field if options.plot: ppd.plot_local_dofs([None, None], field, field_i, omega_gi, options.output_dir, rank) output('allocating global system...') tt = time.clock() sizes, drange = pl.get_sizes(lfd.petsc_dofs_range, field.n_nod, 1) output('sizes:', sizes) output('drange:', drange) pdofs = pl.get_local_ordering(field_i, lfd.petsc_dofs_conn) output('pdofs:', pdofs) pmtx, psol, prhs = pl.create_petsc_system(pb.mtx_a, sizes, pdofs, drange, is_overlap=True, comm=comm, verbose=True) output('...done in', time.clock() - tt) output('evaluating local problem...') tt = time.clock() state = State(variables) state.fill(0.0) state.apply_ebc() rhs_i = eqs.eval_residuals(state()) # This must be after pl.create_petsc_system() call! mtx_i = eqs.eval_tangent_matrices(state(), pb.mtx_a) output('...done in', time.clock() - tt) output('assembling global system...') tt = time.clock() apply_ebc_to_matrix(mtx_i, u_i.eq_map.eq_ebc) pl.assemble_rhs_to_petsc(prhs, rhs_i, pdofs, drange, is_overlap=True, comm=comm, verbose=True) pl.assemble_mtx_to_petsc(pmtx, mtx_i, pdofs, drange, is_overlap=True, comm=comm, verbose=True) output('...done in', time.clock() - tt) output('creating solver...') tt = time.clock() conf = Struct(method='cg', precond='gamg', sub_precond='none', i_max=10000, eps_a=1e-50, eps_r=1e-5, eps_d=1e4, verbose=True) status = {} ls = PETScKrylovSolver(conf, comm=comm, mtx=pmtx, status=status) output('...done in', time.clock() - tt) output('solving...') tt = time.clock() psol = ls(prhs, psol) psol_i = pl.create_local_petsc_vector(pdofs) gather, scatter = pl.create_gather_scatter(pdofs, psol_i, psol, comm=comm) scatter(psol_i, psol) sol0_i = state() - psol_i[...] psol_i[...] = sol0_i gather(psol, psol_i) output('...done in', time.clock() - tt) output('saving solution...') tt = time.clock() u_i.set_data(sol0_i) out = u_i.create_output() filename = os.path.join(options.output_dir, 'sol_%02d.h5' % comm.rank) pb.domain.mesh.write(filename, io='auto', out=out) gather_to_zero = pl.create_gather_to_zero(psol) psol_full = gather_to_zero(psol) if comm.rank == 0: sol = psol_full[...].copy()[id_map] u = FieldVariable('u', 'parameter', field, primary_var_name='(set-to-None)') filename = os.path.join(options.output_dir, 'sol.h5') if (order == 1) or (options.linearization == 'strip'): out = u.create_output(sol) mesh.write(filename, io='auto', out=out) else: out = u.create_output(sol, linearization=Struct(kind='adaptive', min_level=0, max_level=order, eps=1e-3)) out['u'].mesh.write(filename, io='auto', out=out) output('...done in', time.clock() - tt) if options.show: plt.show()
def create_task_dof_maps(field, cell_tasks, inter_facets, is_overlap=True, use_expand_dofs=False, save_inter_regions=False, output_dir=None): """ For each task list its inner and interface DOFs of the given field and create PETSc numbering that is consecutive in each subdomain. For each task, the DOF map has the following structure:: [inner, [own_inter1, own_inter2, ...], [overlap_cells1, overlap_cells2, ...], n_task_total, task_offset] The overlapping cells are defined so that the system matrix corresponding to each task can be assembled independently, see [1]. TODO: Some "corner" cells may be added even if not needed - filter them out by using the PETSc DOFs range. When debugging domain partitioning problems, it is advisable to set `save_inter_regions` to True to save the task interfaces as meshes as well as vertex-based markers - to be used only with moderate problems and small numbers of tasks. [1] J. Sistek and F. Cirak. Parallel iterative solution of the incompressible Navier-Stokes equations with application to rotating wings. Submitted for publication, 2015 """ domain = field.domain cmesh = domain.cmesh if use_expand_dofs: id_map = nm.zeros(field.n_nod * field.n_components, dtype=nm.uint32) else: id_map = nm.zeros(field.n_nod, dtype=nm.uint32) def _get_dofs_region(field, region): dofs = field.get_dofs_in_region(region) if use_expand_dofs: dofs = expand_dofs(dofs, field.n_components) return dofs def _get_dofs_conn(field, conn): dofs = nm.unique(conn) if use_expand_dofs: dofs = expand_dofs(dofs, field.n_components) return dofs dof_maps = {} count = 0 inter_count = 0 ocs = nm.zeros(0, dtype=nm.int32) cell_parts = [] for ir, ntasks in ordered_iteritems(inter_facets): cells = nm.where(cell_tasks == ir)[0].astype(nm.int32) cell_parts.append(cells) cregion = Region.from_cells(cells, domain, name='task_%d' % ir) domain.regions.append(cregion) dofs = _get_dofs_region(field, cregion) rdof_map = dof_maps.setdefault(ir, [None, [], [], 0, 0]) inter_dofs = [] for ic, facets in ordered_iteritems(ntasks): cdof_map = dof_maps.setdefault(ic, [None, [], [], 0, 0]) name = 'inter_%d_%d' % (ir, ic) ii = ir region = Region.from_facets(facets, domain, name, parent=cregion.name) region.update_shape() if save_inter_regions: output_dir = output_dir if output_dir is not None else '.' filename = os.path.join(output_dir, '%s.mesh' % name) aux = domain.mesh.from_region(region, domain.mesh, is_surface=True) aux.write(filename, io='auto') mask = nm.zeros((domain.mesh.n_nod, 1), dtype=nm.float64) mask[region.vertices] = 1 out = {name : Struct(name=name, mode='vertex', data=mask)} filename = os.path.join(output_dir, '%s.h5' % name) domain.mesh.write(filename, out=out, io='auto') inter_dofs.append(_get_dofs_region(field, region)) sd = FESurface('surface_data_%s' % region.name, region, field.efaces, field.econn, field.region) econn = sd.get_connectivity() n_facet = econn.shape[0] ii2 = max(int(n_facet / 2), 1) dr = _get_dofs_conn(field, econn[:ii2]) ii = nm.where((id_map[dr] == 0))[0] n_new = len(ii) if n_new: rdof_map[1].append(dr[ii]) rdof_map[3] += n_new id_map[dr[ii]] = 1 inter_count += n_new count += n_new if is_overlap: ovs = cmesh.get_incident(0, region.facets[:ii2], cmesh.tdim - 1) ocs = cmesh.get_incident(cmesh.tdim, ovs, 0) rdof_map[2].append(ocs.astype(nm.int32)) dc = _get_dofs_conn(field, econn[ii2:]) ii = nm.where((id_map[dc] == 0))[0] n_new = len(ii) if n_new: cdof_map[1].append(dc[ii]) cdof_map[3] += n_new id_map[dc[ii]] = 1 inter_count += n_new count += n_new if is_overlap: ovs = cmesh.get_incident(0, region.facets[ii2:], cmesh.tdim - 1) ocs = cmesh.get_incident(cmesh.tdim, ovs, 0) cdof_map[2].append(ocs.astype(nm.int32)) domain.regions.pop() # Remove the cell region. inner_dofs = nm.setdiff1d(dofs, nm.concatenate(inter_dofs)) n_inner = len(inner_dofs) rdof_map[3] += n_inner assert_(nm.all(id_map[inner_dofs] == 0)) id_map[inner_dofs] = 1 count += n_inner rdof_map[0] = inner_dofs offset = 0 overlap_cells = [] for ir, dof_map in ordered_iteritems(dof_maps): n_owned = dof_map[3] i0 = len(dof_map[0]) id_map[dof_map[0]] = nm.arange(offset, offset + i0, dtype=nm.uint32) for aux in dof_map[1]: i1 = len(aux) id_map[aux] = nm.arange(offset + i0, offset + i0 + i1, dtype=nm.uint32) i0 += i1 if len(dof_map[2]): ocs = nm.unique(nm.concatenate(dof_map[2])) else: ocs = nm.zeros(0, dtype=nm.int32) overlap_cells.append(ocs) assert_(i0 == n_owned) dof_map[4] = offset offset += n_owned if not len(dof_maps): dofs = _get_dofs_region(field, field.region) dof_maps[0] = [dofs, [], [], len(dofs), 0] id_map[:] = nm.arange(len(dofs), dtype=nm.uint32) if not len(cell_parts): cell_parts.append(nm.arange(len(cell_tasks), dtype=nm.int32)) if not len(overlap_cells): overlap_cells.append(nm.zeros(0, dtype=nm.int32)) return dof_maps, id_map, cell_parts, overlap_cells