def _region_leaf(level, op):
token, details = op['token'], op['orig']
if token != 'KW_Region':
parse_def = token + '<' + ' '.join(details) + '>'
region = Region('leaf', rdef, domain, parse_def=parse_def)
if token == 'KW_Region':
details = details[1][2:]
aux = regions.find(details)
if not aux:
raise ValueError('region %s does not exist' % details)
else:
if rdef[:4] == 'copy':
region = aux.copy()
else:
region = aux
elif token == 'KW_All':
region.vertices = nm.arange(n_coor, dtype=nm.uint32)
elif token == 'E_VIR':
where = details[2]
if where[0] == '[':
vertices = nm.array(eval(where), dtype=nm.uint32)
assert_(nm.amin(vertices) >= 0)
assert_(nm.amax(vertices) < n_coor)
else:
coors = domain.cmesh.coors
y = z = None
x = coors[:, 0]
if dim > 1:
y = coors[:, 1]
if dim > 2:
z = coors[:, 2]
coor_dict = {'x': x, 'y': y, 'z': z}
vertices = nm.where(eval(where, {}, coor_dict))[0]
region.vertices = vertices
elif token == 'E_VOS':
facets = domain.cmesh.get_surface_facets()
region.set_kind('facet')
region.facets = facets
elif token == 'E_VBF':
where = details[2]
coors = domain.cmesh.coors
fun = functions[where]
vertices = fun(coors, domain=domain)
region.vertices = vertices
elif token == 'E_CBF':
where = details[2]
coors = domain.get_centroids(dim)
fun = functions[where]
cells = fun(coors, domain=domain)
region.cells = cells
elif token == 'E_COG':
group = int(details[3])
region.cells = nm.where(domain.cmesh.cell_groups == group)[0]
elif token == 'E_COSET':
raise NotImplementedError('element sets not implemented!')
elif token == 'E_VOG':
group = int(details[3])
region.vertices = nm.where(domain.cmesh.vertex_groups == group)[0]
elif token == 'E_VOSET':
try:
vertices = domain.vertex_set_bcs[details[3]]
except KeyError:
msg = 'undefined vertex set! (%s)' % details[3]
raise ValueError(msg)
region.vertices = vertices
elif token == 'E_OVIR':
aux = regions[details[3][2:]]
region.vertices = aux.vertices[0:1]
elif token == 'E_VI':
region.vertices = nm.array([int(ii) for ii in details[1:]],
dtype=nm.uint32)
elif token == 'E_CI':
region.cells = nm.array([int(ii) for ii in details[1:]],
dtype=nm.uint32)
else:
output('token "%s" unkown - check regions!' % token)
raise NotImplementedError
return region
def _region_leaf(level, op):
token, details = op['token'], op['orig']
if token != 'KW_Region':
parse_def = token + '<' + ' '.join(details) + '>'
region = Region('leaf', rdef, domain, parse_def=parse_def)
if token == 'KW_Region':
details = details[1][2:]
aux = regions.find(details)
if not aux:
raise ValueError, 'region %s does not exist' % details
else:
if rdef[:4] == 'copy':
region = aux.copy()
else:
region = aux
elif token == 'KW_All':
region.vertices = nm.arange(n_coor, dtype=nm.uint32)
elif token == 'E_VIR':
where = details[2]
if where[0] == '[':
vertices = nm.array(eval(where), dtype=nm.uint32)
assert_(nm.amin(vertices) >= 0)
assert_(nm.amax(vertices) < n_coor)
else:
coors = domain.cmesh.coors
y = z = None
x = coors[:,0]
if dim > 1:
y = coors[:,1]
if dim > 2:
z = coors[:,2]
coor_dict = {'x' : x, 'y' : y, 'z': z}
vertices = nm.where(eval(where, {}, coor_dict))[0]
region.vertices = vertices
elif token == 'E_VOS':
facets = domain.cmesh.get_surface_facets()
region.set_kind('facet')
region.facets = facets
elif token == 'E_VBF':
where = details[2]
coors = domain.cmesh.coors
fun = functions[where]
vertices = fun(coors, domain=domain)
region.vertices = vertices
elif token == 'E_CBF':
where = details[2]
coors = domain.get_centroids(dim)
fun = functions[where]
cells = fun(coors, domain=domain)
region.cells = cells
elif token == 'E_COG':
group = int(details[3])
region.cells = nm.where(domain.cmesh.cell_groups == group)[0]
elif token == 'E_COSET':
raise NotImplementedError('element sets not implemented!')
elif token == 'E_VOG':
group = int(details[3])
region.vertices = nm.where(domain.cmesh.vertex_groups == group)[0]
elif token == 'E_VOSET':
try:
vertices = domain.vertex_set_bcs[details[3]]
except KeyError:
msg = 'undefined vertex set! (%s)' % details[3]
raise ValueError(msg)
region.vertices = vertices
elif token == 'E_OVIR':
aux = regions[details[3][2:]]
region.vertices = aux.vertices[0:1]
elif token == 'E_VI':
region.vertices = nm.array([int(ii) for ii in details[1:]],
dtype=nm.uint32)
elif token == 'E_CI':
region.cells = nm.array([int(ii) for ii in details[1:]],
dtype=nm.uint32)
else:
output('token "%s" unkown - check regions!' % token)
raise NotImplementedError
return region
def solve_problem(mesh_filename, options, comm):
order_u = options.order_u
order_p = options.order_p
rank, size = comm.Get_rank(), comm.Get_size()
output('rank', rank, 'of', size)
mesh = Mesh.from_file(mesh_filename)
if rank == 0:
cell_tasks = pl.partition_mesh(mesh, size, use_metis=options.metis,
verbose=True)
else:
cell_tasks = None
domain = FEDomain('domain', mesh)
omega = domain.create_region('Omega', 'all')
field1 = Field.from_args('fu', nm.float64, mesh.dim, omega,
approx_order=order_u)
field2 = Field.from_args('fp', nm.float64, 1, omega,
approx_order=order_p)
fields = [field1, field2]
output('distributing fields...')
tt = time.clock()
lfds, gfds = pl.distribute_fields_dofs(fields, cell_tasks,
is_overlap=True,
use_expand_dofs=True,
comm=comm, verbose=True)
output('...done in', time.clock() - tt)
output('creating local problem...')
tt = time.clock()
cells = lfds[0].cells
omega_gi = Region.from_cells(cells, domain)
omega_gi.finalize()
omega_gi.update_shape()
pb = create_local_problem(omega_gi, [order_u, order_p])
variables = pb.get_variables()
state = State(variables)
state.fill(0.0)
state.apply_ebc()
output('...done in', time.clock() - tt)
output('allocating global system...')
tt = time.clock()
sizes, drange, pdofs = pl.setup_composite_dofs(lfds, fields, variables,
verbose=True)
pmtx, psol, prhs = pl.create_petsc_system(pb.mtx_a, sizes, pdofs, drange,
is_overlap=True, comm=comm,
verbose=True)
output('...done in', time.clock() - tt)
output('creating solver...')
tt = time.clock()
conf = Struct(method='bcgsl', precond='jacobi', sub_precond=None,
i_max=10000, eps_a=1e-50, eps_r=1e-6, eps_d=1e4,
verbose=True)
status = {}
ls = PETScKrylovSolver(conf, comm=comm, mtx=pmtx, status=status)
field_ranges = {}
for ii, variable in enumerate(variables.iter_state(ordered=True)):
field_ranges[variable.name] = lfds[ii].petsc_dofs_range
ls.set_field_split(field_ranges, comm=comm)
ev = PETScParallelEvaluator(pb, pdofs, drange, True,
psol, comm, verbose=True)
nls_status = {}
conf = Struct(method='newtonls',
i_max=5, eps_a=0, eps_r=1e-5, eps_s=0.0,
verbose=True)
nls = PETScNonlinearSolver(conf, pmtx=pmtx, prhs=prhs, comm=comm,
fun=ev.eval_residual,
fun_grad=ev.eval_tangent_matrix,
lin_solver=ls, status=nls_status)
output('...done in', time.clock() - tt)
output('solving...')
tt = time.clock()
state = pb.create_state()
state.apply_ebc()
ev.psol_i[...] = state()
ev.gather(psol, ev.psol_i)
psol = nls(psol)
ev.scatter(ev.psol_i, psol)
sol0_i = ev.psol_i[...]
output('...done in', time.clock() - tt)
output('saving solution...')
tt = time.clock()
state.set_full(sol0_i)
out = state.create_output_dict()
filename = os.path.join(options.output_dir, 'sol_%02d.h5' % comm.rank)
pb.domain.mesh.write(filename, io='auto', out=out)
gather_to_zero = pl.create_gather_to_zero(psol)
psol_full = gather_to_zero(psol)
if comm.rank == 0:
sol = psol_full[...].copy()
u = FieldVariable('u', 'parameter', field1,
primary_var_name='(set-to-None)')
remap = gfds[0].id_map
ug = sol[remap]
p = FieldVariable('p', 'parameter', field2,
primary_var_name='(set-to-None)')
remap = gfds[1].id_map
pg = sol[remap]
if (((order_u == 1) and (order_p == 1))
or (options.linearization == 'strip')):
out = u.create_output(ug)
out.update(p.create_output(pg))
filename = os.path.join(options.output_dir, 'sol.h5')
mesh.write(filename, io='auto', out=out)
else:
out = u.create_output(ug, linearization=Struct(kind='adaptive',
min_level=0,
max_level=order_u,
eps=1e-3))
filename = os.path.join(options.output_dir, 'sol_u.h5')
out['u'].mesh.write(filename, io='auto', out=out)
out = p.create_output(pg, linearization=Struct(kind='adaptive',
min_level=0,
max_level=order_p,
eps=1e-3))
filename = os.path.join(options.output_dir, 'sol_p.h5')
out['p'].mesh.write(filename, io='auto', out=out)
output('...done in', time.clock() - tt)
def solve_problem(mesh_filename, options, comm):
order_u = options.order_u
order_p = options.order_p
rank, size = comm.Get_rank(), comm.Get_size()
output('rank', rank, 'of', size)
stats = Struct()
timer = Timer('solve_timer')
timer.start()
mesh = Mesh.from_file(mesh_filename)
stats.t_read_mesh = timer.stop()
timer.start()
if rank == 0:
cell_tasks = pl.partition_mesh(mesh, size, use_metis=options.metis,
verbose=True)
else:
cell_tasks = None
stats.t_partition_mesh = timer.stop()
output('creating global domain and fields...')
timer.start()
domain = FEDomain('domain', mesh)
omega = domain.create_region('Omega', 'all')
field1 = Field.from_args('fu', nm.float64, mesh.dim, omega,
approx_order=order_u)
field2 = Field.from_args('fp', nm.float64, 1, omega,
approx_order=order_p)
fields = [field1, field2]
stats.t_create_global_fields = timer.stop()
output('...done in', timer.dt)
output('distributing fields...')
timer.start()
distribute = pl.distribute_fields_dofs
lfds, gfds = distribute(fields, cell_tasks,
is_overlap=True,
use_expand_dofs=True,
save_inter_regions=options.save_inter_regions,
output_dir=options.output_dir,
comm=comm, verbose=True)
stats.t_distribute_fields_dofs = timer.stop()
output('...done in', timer.dt)
output('creating local problem...')
timer.start()
cells = lfds[0].cells
omega_gi = Region.from_cells(cells, domain)
omega_gi.finalize()
omega_gi.update_shape()
pb = create_local_problem(omega_gi, [order_u, order_p])
variables = pb.get_variables()
state = State(variables)
state.fill(0.0)
state.apply_ebc()
stats.t_create_local_problem = timer.stop()
output('...done in', timer.dt)
output('allocating global system...')
timer.start()
sizes, drange, pdofs = pl.setup_composite_dofs(lfds, fields, variables,
verbose=True)
pmtx, psol, prhs = pl.create_petsc_system(pb.mtx_a, sizes, pdofs, drange,
is_overlap=True, comm=comm,
verbose=True)
stats.t_allocate_global_system = timer.stop()
output('...done in', timer.dt)
output('creating solver...')
timer.start()
conf = Struct(method='bcgsl', precond='jacobi', sub_precond='none',
i_max=10000, eps_a=1e-50, eps_r=1e-6, eps_d=1e4,
verbose=True)
status = {}
ls = PETScKrylovSolver(conf, comm=comm, mtx=pmtx, status=status)
field_ranges = {}
for ii, variable in enumerate(variables.iter_state(ordered=True)):
field_ranges[variable.name] = lfds[ii].petsc_dofs_range
ls.set_field_split(field_ranges, comm=comm)
ev = PETScParallelEvaluator(pb, pdofs, drange, True,
psol, comm, verbose=True)
nls_status = {}
conf = Struct(method='newtonls',
i_max=5, eps_a=0, eps_r=1e-5, eps_s=0.0,
verbose=True)
nls = PETScNonlinearSolver(conf, pmtx=pmtx, prhs=prhs, comm=comm,
fun=ev.eval_residual,
fun_grad=ev.eval_tangent_matrix,
lin_solver=ls, status=nls_status)
stats.t_create_solver = timer.stop()
output('...done in', timer.dt)
output('solving...')
timer.start()
state = pb.create_state()
state.apply_ebc()
ev.psol_i[...] = state()
ev.gather(psol, ev.psol_i)
psol = nls(psol)
ev.scatter(ev.psol_i, psol)
sol0_i = ev.psol_i[...]
stats.t_solve = timer.stop()
output('...done in', timer.dt)
output('saving solution...')
timer.start()
state.set_full(sol0_i)
out = state.create_output_dict()
filename = os.path.join(options.output_dir, 'sol_%02d.h5' % comm.rank)
pb.domain.mesh.write(filename, io='auto', out=out)
gather_to_zero = pl.create_gather_to_zero(psol)
psol_full = gather_to_zero(psol)
if comm.rank == 0:
sol = psol_full[...].copy()
u = FieldVariable('u', 'parameter', field1,
primary_var_name='(set-to-None)')
remap = gfds[0].id_map
ug = sol[remap]
p = FieldVariable('p', 'parameter', field2,
primary_var_name='(set-to-None)')
remap = gfds[1].id_map
pg = sol[remap]
if (((order_u == 1) and (order_p == 1))
or (options.linearization == 'strip')):
out = u.create_output(ug)
out.update(p.create_output(pg))
filename = os.path.join(options.output_dir, 'sol.h5')
mesh.write(filename, io='auto', out=out)
else:
out = u.create_output(ug, linearization=Struct(kind='adaptive',
min_level=0,
max_level=order_u,
eps=1e-3))
filename = os.path.join(options.output_dir, 'sol_u.h5')
out['u'].mesh.write(filename, io='auto', out=out)
out = p.create_output(pg, linearization=Struct(kind='adaptive',
min_level=0,
max_level=order_p,
eps=1e-3))
filename = os.path.join(options.output_dir, 'sol_p.h5')
out['p'].mesh.write(filename, io='auto', out=out)
stats.t_save_solution = timer.stop()
output('...done in', timer.dt)
stats.t_total = timer.total
stats.n_dof = sizes[1]
stats.n_dof_local = sizes[0]
stats.n_cell = omega.shape.n_cell
stats.n_cell_local = omega_gi.shape.n_cell
return stats
def solve_problem(mesh_filename, options, comm):
order = options.order
rank, size = comm.Get_rank(), comm.Get_size()
output('rank', rank, 'of', size)
mesh = Mesh.from_file(mesh_filename)
if rank == 0:
cell_tasks = pl.partition_mesh(mesh, size, use_metis=options.metis,
verbose=True)
else:
cell_tasks = None
domain = FEDomain('domain', mesh)
omega = domain.create_region('Omega', 'all')
field = Field.from_args('fu', nm.float64, 1, omega, approx_order=order)
output('distributing field %s...' % field.name)
tt = time.clock()
distribute = pl.distribute_fields_dofs
lfds, gfds = distribute([field], cell_tasks,
is_overlap=True,
save_inter_regions=options.save_inter_regions,
output_dir=options.output_dir,
comm=comm, verbose=True)
lfd = lfds[0]
output('...done in', time.clock() - tt)
if rank == 0:
dof_maps = gfds[0].dof_maps
id_map = gfds[0].id_map
if options.verify:
verify_save_dof_maps(field, cell_tasks,
dof_maps, id_map, options, verbose=True)
if options.plot:
ppd.plot_partitioning([None, None], field, cell_tasks, gfds[0],
options.output_dir, size)
output('creating local problem...')
tt = time.clock()
omega_gi = Region.from_cells(lfd.cells, field.domain)
omega_gi.finalize()
omega_gi.update_shape()
pb = create_local_problem(omega_gi, order)
output('...done in', time.clock() - tt)
variables = pb.get_variables()
eqs = pb.equations
u_i = variables['u_i']
field_i = u_i.field
if options.plot:
ppd.plot_local_dofs([None, None], field, field_i, omega_gi,
options.output_dir, rank)
output('allocating global system...')
tt = time.clock()
sizes, drange = pl.get_sizes(lfd.petsc_dofs_range, field.n_nod, 1)
output('sizes:', sizes)
output('drange:', drange)
pdofs = pl.get_local_ordering(field_i, lfd.petsc_dofs_conn)
output('pdofs:', pdofs)
pmtx, psol, prhs = pl.create_petsc_system(pb.mtx_a, sizes, pdofs, drange,
is_overlap=True, comm=comm,
verbose=True)
output('...done in', time.clock() - tt)
output('evaluating local problem...')
tt = time.clock()
state = State(variables)
state.fill(0.0)
state.apply_ebc()
rhs_i = eqs.eval_residuals(state())
# This must be after pl.create_petsc_system() call!
mtx_i = eqs.eval_tangent_matrices(state(), pb.mtx_a)
output('...done in', time.clock() - tt)
output('assembling global system...')
tt = time.clock()
pl.apply_ebc_to_matrix(mtx_i, u_i.eq_map.eq_ebc)
pl.assemble_rhs_to_petsc(prhs, rhs_i, pdofs, drange, is_overlap=True,
comm=comm, verbose=True)
pl.assemble_mtx_to_petsc(pmtx, mtx_i, pdofs, drange, is_overlap=True,
comm=comm, verbose=True)
output('...done in', time.clock() - tt)
output('creating solver...')
tt = time.clock()
conf = Struct(method='cg', precond='gamg', sub_precond=None,
i_max=10000, eps_a=1e-50, eps_r=1e-5, eps_d=1e4, verbose=True)
status = {}
ls = PETScKrylovSolver(conf, comm=comm, mtx=pmtx, status=status)
output('...done in', time.clock() - tt)
output('solving...')
tt = time.clock()
psol = ls(prhs, psol, conf)
psol_i = pl.create_local_petsc_vector(pdofs)
gather, scatter = pl.create_gather_scatter(pdofs, psol_i, psol, comm=comm)
scatter(psol_i, psol)
sol0_i = state() - psol_i[...]
psol_i[...] = sol0_i
gather(psol, psol_i)
output('...done in', time.clock() - tt)
output('saving solution...')
tt = time.clock()
u_i.set_data(sol0_i)
out = u_i.create_output()
filename = os.path.join(options.output_dir, 'sol_%02d.h5' % comm.rank)
pb.domain.mesh.write(filename, io='auto', out=out)
gather_to_zero = pl.create_gather_to_zero(psol)
psol_full = gather_to_zero(psol)
if comm.rank == 0:
sol = psol_full[...].copy()[id_map]
u = FieldVariable('u', 'parameter', field,
primary_var_name='(set-to-None)')
filename = os.path.join(options.output_dir, 'sol.h5')
if (order == 1) or (options.linearization == 'strip'):
out = u.create_output(sol)
mesh.write(filename, io='auto', out=out)
else:
out = u.create_output(sol, linearization=Struct(kind='adaptive',
min_level=0,
max_level=order,
eps=1e-3))
out['u'].mesh.write(filename, io='auto', out=out)
output('...done in', time.clock() - tt)
if options.show:
plt.show()
def create_task_dof_maps(field,
cell_tasks,
inter_facets,
is_overlap=True,
use_expand_dofs=False,
save_inter_regions=False,
output_dir=None):
"""
For each task list its inner and interface DOFs of the given field and
create PETSc numbering that is consecutive in each subdomain.
For each task, the DOF map has the following structure::
[inner,
[own_inter1, own_inter2, ...],
[overlap_cells1, overlap_cells2, ...],
n_task_total, task_offset]
The overlapping cells are defined so that the system matrix corresponding
to each task can be assembled independently, see [1]. TODO: Some "corner"
cells may be added even if not needed - filter them out by using the PETSc
DOFs range.
When debugging domain partitioning problems, it is advisable to set
`save_inter_regions` to True to save the task interfaces as meshes as well
as vertex-based markers - to be used only with moderate problems and small
numbers of tasks.
[1] J. Sistek and F. Cirak. Parallel iterative solution of the
incompressible Navier-Stokes equations with application to rotating
wings. Submitted for publication, 2015
"""
domain = field.domain
cmesh = domain.cmesh
if use_expand_dofs:
id_map = nm.zeros(field.n_nod * field.n_components, dtype=nm.uint32)
else:
id_map = nm.zeros(field.n_nod, dtype=nm.uint32)
def _get_dofs_region(field, region):
dofs = field.get_dofs_in_region(region)
if use_expand_dofs:
dofs = expand_dofs(dofs, field.n_components)
return dofs
def _get_dofs_conn(field, conn):
dofs = nm.unique(conn)
if use_expand_dofs:
dofs = expand_dofs(dofs, field.n_components)
return dofs
dof_maps = {}
count = 0
inter_count = 0
ocs = nm.zeros(0, dtype=nm.int32)
cell_parts = []
for ir, ntasks in ordered_iteritems(inter_facets):
cells = nm.where(cell_tasks == ir)[0].astype(nm.int32)
cell_parts.append(cells)
cregion = Region.from_cells(cells, domain, name='task_%d' % ir)
domain.regions.append(cregion)
dofs = _get_dofs_region(field, cregion)
rdof_map = dof_maps.setdefault(ir, [None, [], [], 0, 0])
inter_dofs = []
for ic, facets in ordered_iteritems(ntasks):
cdof_map = dof_maps.setdefault(ic, [None, [], [], 0, 0])
name = 'inter_%d_%d' % (ir, ic)
ii = ir
region = Region.from_facets(facets,
domain,
name,
parent=cregion.name)
region.update_shape()
if save_inter_regions:
output_dir = output_dir if output_dir is not None else '.'
filename = os.path.join(output_dir, '%s.mesh' % name)
aux = domain.mesh.from_region(region,
domain.mesh,
is_surface=True)
aux.write(filename, io='auto')
mask = nm.zeros((domain.mesh.n_nod, 1), dtype=nm.float64)
mask[region.vertices] = 1
out = {name: Struct(name=name, mode='vertex', data=mask)}
filename = os.path.join(output_dir, '%s.h5' % name)
domain.mesh.write(filename, out=out, io='auto')
inter_dofs.append(_get_dofs_region(field, region))
sd = FESurface('surface_data_%s' % region.name, region,
field.efaces, field.econn, field.region)
econn = sd.get_connectivity()
n_facet = econn.shape[0]
ii2 = max(int(n_facet / 2), 1)
dr = _get_dofs_conn(field, econn[:ii2])
ii = nm.where((id_map[dr] == 0))[0]
n_new = len(ii)
if n_new:
rdof_map[1].append(dr[ii])
rdof_map[3] += n_new
id_map[dr[ii]] = 1
inter_count += n_new
count += n_new
if is_overlap:
ovs = cmesh.get_incident(0, region.facets[:ii2],
cmesh.tdim - 1)
ocs = cmesh.get_incident(cmesh.tdim, ovs, 0)
rdof_map[2].append(ocs.astype(nm.int32))
dc = _get_dofs_conn(field, econn[ii2:])
ii = nm.where((id_map[dc] == 0))[0]
n_new = len(ii)
if n_new:
cdof_map[1].append(dc[ii])
cdof_map[3] += n_new
id_map[dc[ii]] = 1
inter_count += n_new
count += n_new
if is_overlap:
ovs = cmesh.get_incident(0, region.facets[ii2:],
cmesh.tdim - 1)
ocs = cmesh.get_incident(cmesh.tdim, ovs, 0)
cdof_map[2].append(ocs.astype(nm.int32))
domain.regions.pop() # Remove the cell region.
inner_dofs = nm.setdiff1d(dofs, nm.concatenate(inter_dofs))
n_inner = len(inner_dofs)
rdof_map[3] += n_inner
assert_(nm.all(id_map[inner_dofs] == 0))
id_map[inner_dofs] = 1
count += n_inner
rdof_map[0] = inner_dofs
offset = 0
overlap_cells = []
for ir, dof_map in ordered_iteritems(dof_maps):
n_owned = dof_map[3]
i0 = len(dof_map[0])
id_map[dof_map[0]] = nm.arange(offset, offset + i0, dtype=nm.uint32)
for aux in dof_map[1]:
i1 = len(aux)
id_map[aux] = nm.arange(offset + i0,
offset + i0 + i1,
dtype=nm.uint32)
i0 += i1
if len(dof_map[2]):
ocs = nm.unique(nm.concatenate(dof_map[2]))
else:
ocs = nm.zeros(0, dtype=nm.int32)
overlap_cells.append(ocs)
assert_(i0 == n_owned)
dof_map[4] = offset
offset += n_owned
if not len(dof_maps):
dofs = _get_dofs_region(field, field.region)
dof_maps[0] = [dofs, [], [], len(dofs), 0]
id_map[:] = nm.arange(len(dofs), dtype=nm.uint32)
if not len(cell_parts):
cell_parts.append(nm.arange(len(cell_tasks), dtype=nm.int32))
if not len(overlap_cells):
overlap_cells.append(nm.zeros(0, dtype=nm.int32))
return dof_maps, id_map, cell_parts, overlap_cells
def solve_problem(mesh_filename, options, comm):
order = options.order
rank, size = comm.Get_rank(), comm.Get_size()
output('rank', rank, 'of', size)
mesh = Mesh.from_file(mesh_filename)
if rank == 0:
cell_tasks = pl.partition_mesh(mesh,
size,
use_metis=options.metis,
verbose=True)
else:
cell_tasks = None
output('creating global domain and field...')
tt = time.clock()
domain = FEDomain('domain', mesh)
omega = domain.create_region('Omega', 'all')
field = Field.from_args('fu', nm.float64, 1, omega, approx_order=order)
output('...done in', time.clock() - tt)
output('distributing field %s...' % field.name)
tt = time.clock()
distribute = pl.distribute_fields_dofs
lfds, gfds = distribute([field],
cell_tasks,
is_overlap=True,
save_inter_regions=options.save_inter_regions,
output_dir=options.output_dir,
comm=comm,
verbose=True)
lfd = lfds[0]
output('...done in', time.clock() - tt)
if rank == 0:
dof_maps = gfds[0].dof_maps
id_map = gfds[0].id_map
if options.verify:
verify_save_dof_maps(field,
cell_tasks,
dof_maps,
id_map,
options,
verbose=True)
if options.plot:
ppd.plot_partitioning([None, None], field, cell_tasks, gfds[0],
options.output_dir, size)
output('creating local problem...')
tt = time.clock()
omega_gi = Region.from_cells(lfd.cells, field.domain)
omega_gi.finalize()
omega_gi.update_shape()
pb = create_local_problem(omega_gi, order)
output('...done in', time.clock() - tt)
variables = pb.get_variables()
eqs = pb.equations
u_i = variables['u_i']
field_i = u_i.field
if options.plot:
ppd.plot_local_dofs([None, None], field, field_i, omega_gi,
options.output_dir, rank)
output('allocating global system...')
tt = time.clock()
sizes, drange = pl.get_sizes(lfd.petsc_dofs_range, field.n_nod, 1)
output('sizes:', sizes)
output('drange:', drange)
pdofs = pl.get_local_ordering(field_i, lfd.petsc_dofs_conn)
output('pdofs:', pdofs)
pmtx, psol, prhs = pl.create_petsc_system(pb.mtx_a,
sizes,
pdofs,
drange,
is_overlap=True,
comm=comm,
verbose=True)
output('...done in', time.clock() - tt)
output('evaluating local problem...')
tt = time.clock()
state = State(variables)
state.fill(0.0)
state.apply_ebc()
rhs_i = eqs.eval_residuals(state())
# This must be after pl.create_petsc_system() call!
mtx_i = eqs.eval_tangent_matrices(state(), pb.mtx_a)
output('...done in', time.clock() - tt)
output('assembling global system...')
tt = time.clock()
apply_ebc_to_matrix(mtx_i, u_i.eq_map.eq_ebc)
pl.assemble_rhs_to_petsc(prhs,
rhs_i,
pdofs,
drange,
is_overlap=True,
comm=comm,
verbose=True)
pl.assemble_mtx_to_petsc(pmtx,
mtx_i,
pdofs,
drange,
is_overlap=True,
comm=comm,
verbose=True)
output('...done in', time.clock() - tt)
output('creating solver...')
tt = time.clock()
conf = Struct(method='cg',
precond='gamg',
sub_precond='none',
i_max=10000,
eps_a=1e-50,
eps_r=1e-5,
eps_d=1e4,
verbose=True)
status = {}
ls = PETScKrylovSolver(conf, comm=comm, mtx=pmtx, status=status)
output('...done in', time.clock() - tt)
output('solving...')
tt = time.clock()
psol = ls(prhs, psol)
psol_i = pl.create_local_petsc_vector(pdofs)
gather, scatter = pl.create_gather_scatter(pdofs, psol_i, psol, comm=comm)
scatter(psol_i, psol)
sol0_i = state() - psol_i[...]
psol_i[...] = sol0_i
gather(psol, psol_i)
output('...done in', time.clock() - tt)
output('saving solution...')
tt = time.clock()
u_i.set_data(sol0_i)
out = u_i.create_output()
filename = os.path.join(options.output_dir, 'sol_%02d.h5' % comm.rank)
pb.domain.mesh.write(filename, io='auto', out=out)
gather_to_zero = pl.create_gather_to_zero(psol)
psol_full = gather_to_zero(psol)
if comm.rank == 0:
sol = psol_full[...].copy()[id_map]
u = FieldVariable('u',
'parameter',
field,
primary_var_name='(set-to-None)')
filename = os.path.join(options.output_dir, 'sol.h5')
if (order == 1) or (options.linearization == 'strip'):
out = u.create_output(sol)
mesh.write(filename, io='auto', out=out)
else:
out = u.create_output(sol,
linearization=Struct(kind='adaptive',
min_level=0,
max_level=order,
eps=1e-3))
out['u'].mesh.write(filename, io='auto', out=out)
output('...done in', time.clock() - tt)
if options.show:
plt.show()
def create_task_dof_maps(field, cell_tasks, inter_facets, is_overlap=True,
use_expand_dofs=False, save_inter_regions=False,
output_dir=None):
"""
For each task list its inner and interface DOFs of the given field and
create PETSc numbering that is consecutive in each subdomain.
For each task, the DOF map has the following structure::
[inner,
[own_inter1, own_inter2, ...],
[overlap_cells1, overlap_cells2, ...],
n_task_total, task_offset]
The overlapping cells are defined so that the system matrix corresponding
to each task can be assembled independently, see [1]. TODO: Some "corner"
cells may be added even if not needed - filter them out by using the PETSc
DOFs range.
When debugging domain partitioning problems, it is advisable to set
`save_inter_regions` to True to save the task interfaces as meshes as well
as vertex-based markers - to be used only with moderate problems and small
numbers of tasks.
[1] J. Sistek and F. Cirak. Parallel iterative solution of the
incompressible Navier-Stokes equations with application to rotating
wings. Submitted for publication, 2015
"""
domain = field.domain
cmesh = domain.cmesh
if use_expand_dofs:
id_map = nm.zeros(field.n_nod * field.n_components, dtype=nm.uint32)
else:
id_map = nm.zeros(field.n_nod, dtype=nm.uint32)
def _get_dofs_region(field, region):
dofs = field.get_dofs_in_region(region)
if use_expand_dofs:
dofs = expand_dofs(dofs, field.n_components)
return dofs
def _get_dofs_conn(field, conn):
dofs = nm.unique(conn)
if use_expand_dofs:
dofs = expand_dofs(dofs, field.n_components)
return dofs
dof_maps = {}
count = 0
inter_count = 0
ocs = nm.zeros(0, dtype=nm.int32)
cell_parts = []
for ir, ntasks in ordered_iteritems(inter_facets):
cells = nm.where(cell_tasks == ir)[0].astype(nm.int32)
cell_parts.append(cells)
cregion = Region.from_cells(cells, domain, name='task_%d' % ir)
domain.regions.append(cregion)
dofs = _get_dofs_region(field, cregion)
rdof_map = dof_maps.setdefault(ir, [None, [], [], 0, 0])
inter_dofs = []
for ic, facets in ordered_iteritems(ntasks):
cdof_map = dof_maps.setdefault(ic, [None, [], [], 0, 0])
name = 'inter_%d_%d' % (ir, ic)
ii = ir
region = Region.from_facets(facets, domain, name,
parent=cregion.name)
region.update_shape()
if save_inter_regions:
output_dir = output_dir if output_dir is not None else '.'
filename = os.path.join(output_dir, '%s.mesh' % name)
aux = domain.mesh.from_region(region, domain.mesh,
is_surface=True)
aux.write(filename, io='auto')
mask = nm.zeros((domain.mesh.n_nod, 1), dtype=nm.float64)
mask[region.vertices] = 1
out = {name : Struct(name=name, mode='vertex', data=mask)}
filename = os.path.join(output_dir, '%s.h5' % name)
domain.mesh.write(filename, out=out, io='auto')
inter_dofs.append(_get_dofs_region(field, region))
sd = FESurface('surface_data_%s' % region.name, region,
field.efaces, field.econn, field.region)
econn = sd.get_connectivity()
n_facet = econn.shape[0]
ii2 = max(int(n_facet / 2), 1)
dr = _get_dofs_conn(field, econn[:ii2])
ii = nm.where((id_map[dr] == 0))[0]
n_new = len(ii)
if n_new:
rdof_map[1].append(dr[ii])
rdof_map[3] += n_new
id_map[dr[ii]] = 1
inter_count += n_new
count += n_new
if is_overlap:
ovs = cmesh.get_incident(0, region.facets[:ii2],
cmesh.tdim - 1)
ocs = cmesh.get_incident(cmesh.tdim, ovs, 0)
rdof_map[2].append(ocs.astype(nm.int32))
dc = _get_dofs_conn(field, econn[ii2:])
ii = nm.where((id_map[dc] == 0))[0]
n_new = len(ii)
if n_new:
cdof_map[1].append(dc[ii])
cdof_map[3] += n_new
id_map[dc[ii]] = 1
inter_count += n_new
count += n_new
if is_overlap:
ovs = cmesh.get_incident(0, region.facets[ii2:],
cmesh.tdim - 1)
ocs = cmesh.get_incident(cmesh.tdim, ovs, 0)
cdof_map[2].append(ocs.astype(nm.int32))
domain.regions.pop() # Remove the cell region.
inner_dofs = nm.setdiff1d(dofs, nm.concatenate(inter_dofs))
n_inner = len(inner_dofs)
rdof_map[3] += n_inner
assert_(nm.all(id_map[inner_dofs] == 0))
id_map[inner_dofs] = 1
count += n_inner
rdof_map[0] = inner_dofs
offset = 0
overlap_cells = []
for ir, dof_map in ordered_iteritems(dof_maps):
n_owned = dof_map[3]
i0 = len(dof_map[0])
id_map[dof_map[0]] = nm.arange(offset, offset + i0, dtype=nm.uint32)
for aux in dof_map[1]:
i1 = len(aux)
id_map[aux] = nm.arange(offset + i0, offset + i0 + i1,
dtype=nm.uint32)
i0 += i1
if len(dof_map[2]):
ocs = nm.unique(nm.concatenate(dof_map[2]))
else:
ocs = nm.zeros(0, dtype=nm.int32)
overlap_cells.append(ocs)
assert_(i0 == n_owned)
dof_map[4] = offset
offset += n_owned
if not len(dof_maps):
dofs = _get_dofs_region(field, field.region)
dof_maps[0] = [dofs, [], [], len(dofs), 0]
id_map[:] = nm.arange(len(dofs), dtype=nm.uint32)
if not len(cell_parts):
cell_parts.append(nm.arange(len(cell_tasks), dtype=nm.int32))
if not len(overlap_cells):
overlap_cells.append(nm.zeros(0, dtype=nm.int32))
return dof_maps, id_map, cell_parts, overlap_cells
def _region_leaf(level, op):
token, details = op['token'], op['orig']
if token != 'KW_Region':
parse_def = token + '<' + ' '.join(details) + '>'
region = Region('leaf', rdef, domain, parse_def=parse_def)
if token == 'KW_Region':
details = details[1][2:]
aux = regions.find(details)
if not aux:
raise ValueError, 'region %s does not exist' % details
else:
if rdef[:4] == 'copy':
region = aux.copy()
else:
region = aux
elif token == 'KW_All':
region.vertices = nm.arange(domain.mesh.n_nod, dtype=nm.uint32)
elif token == 'E_VIR':
where = details[2]
if where[0] == '[':
vertices = nm.array(eval(where), dtype=nm.uint32)
assert_(nm.amin(vertices) >= 0)
assert_(nm.amax(vertices) < domain.mesh.n_nod)
else:
coors = domain.get_mesh_coors()
x = coors[:,0]
y = coors[:,1]
if domain.mesh.dim == 3:
z = coors[:,2]
else:
z = None
coor_dict = {'x' : x, 'y' : y, 'z': z}
vertices = nm.where(eval(where, {}, coor_dict))[0]
region.vertices = vertices
elif token == 'E_VOS':
facets = domain.cmesh.get_surface_facets()
region.set_kind('facet')
region.facets = facets
elif token == 'E_VBF':
where = details[2]
coors = domain.get_mesh_coors()
fun = functions[where]
vertices = fun(coors, domain=domain)
region.vertices = vertices
elif token == 'E_CBF':
where = details[2]
coors = domain.get_centroids(domain.mesh.dim)
fun = functions[where]
cells = fun(coors, domain=domain)
region.cells = cells
elif token == 'E_COG':
group = int(details[3])
ig = domain.mat_ids_to_i_gs[group]
group = domain.groups[ig]
off = domain.mesh.el_offsets[ig]
region.cells = off + nm.arange(group.shape.n_el, dtype=nm.uint32)
elif token == 'E_COSET':
raise NotImplementedError('element sets not implemented!')
elif token == 'E_VOG':
group = int(details[3])
vertices = nm.where(domain.mesh.ngroups == group)[0]
region.vertices = vertices
elif token == 'E_VOSET':
try:
vertices = domain.mesh.nodal_bcs[details[3]]
except KeyError:
msg = 'undefined vertex set! (%s)' % details[3]
raise ValueError(msg)
region.vertices = vertices
elif token == 'E_OVIR':
aux = regions[details[3][2:]]
region.vertices = aux.vertices[0:1]
elif token == 'E_VI':
region.vertices = nm.array([int(ii) for ii in details[1:]],
dtype=nm.uint32)
elif token == 'E_CI1':
region.cells = nm.array([int(ii) for ii in details[1:]],
dtype=nm.uint32)
elif token == 'E_CI2':
num = len(details[1:]) / 2
cells = []
for ii in range(num):
ig, iel = int(details[1+2*ii]), int(details[2+2*ii])
cells.append(iel + domain.mesh.el_offsets[ig])
region.cells = cells
else:
output('token "%s" unkown - check regions!' % token)
raise NotImplementedError
return region
