def add_instrumentation(self, mgr, observer):
Depositor.add_instrumentation(self, mgr, observer)
# instrumentation
from pytools.log import IntervalTimer, EventCounter
self.element_activation_counter = EventCounter(
"n_el_activations",
"#Advective rec. elements activated this timestep")
self.element_kill_counter = EventCounter(
"n_el_kills", "#Advective rec. elements retired this timestep")
self.advective_rhs_timer = IntervalTimer(
"t_advective_rhs", "Time spent evaluating advective RHS")
self.active_elements_log = ActiveAdvectiveElements(observer)
mgr.add_quantity(self.element_activation_counter)
mgr.add_quantity(self.element_kill_counter)
mgr.add_quantity(self.advective_rhs_timer)
mgr.add_quantity(self.active_elements_log)
mgr.set_constant("el_activation_threshold", self.activation_threshold)
mgr.set_constant("el_kill_threshold", self.kill_threshold)
mgr.set_constant("adv_upwind_alpha", self.upwind_alpha)
mgr.set_constant("filter_amp", self.filter_amp)
mgr.set_constant("filter_amp", self.filter_order)
def __init__(self,
dtype=numpy.float64,
rcon=None,
vector_primitive_factory=None):
if vector_primitive_factory is None:
from hedge.vector_primitives import VectorPrimitiveFactory
self.vector_primitive_factory = VectorPrimitiveFactory()
else:
self.vector_primitive_factory = vector_primitive_factory
from pytools.log import IntervalTimer, EventCounter
timer_factory = IntervalTimer
if rcon is not None:
timer_factory = rcon.make_timer
self.timer = timer_factory("t_rk4", "Time spent doing algebra in RK4")
self.flop_counter = EventCounter(
"n_flops_rk4", "Floating point operations performed in RK4")
from pytools import match_precision
self.dtype = numpy.dtype(dtype)
self.scalar_dtype = match_precision(numpy.dtype(numpy.float64),
self.dtype)
self.coeffs = numpy.array([self._RK4A, self._RK4B, self._RK4C],
dtype=self.scalar_dtype).T
def __init__(self,
order,
startup_stepper=None,
dtype=numpy.float64,
rcon=None):
self.f_history = []
from pytools import match_precision
self.dtype = numpy.dtype(dtype)
self.scalar_dtype = match_precision(numpy.dtype(numpy.float64),
self.dtype)
self.coefficients = numpy.asarray(make_ab_coefficients(order),
dtype=self.scalar_dtype)
if startup_stepper is not None:
self.startup_stepper = startup_stepper
else:
from hedge.timestep.runge_kutta import LSRK4TimeStepper
self.startup_stepper = LSRK4TimeStepper(self.dtype)
from pytools.log import IntervalTimer, EventCounter
timer_factory = IntervalTimer
if rcon is not None:
timer_factory = rcon.make_timer
self.timer = timer_factory(
"t_ab", "Time spent doing algebra in Adams-Bashforth")
self.flop_counter = EventCounter(
"n_flops_ab", "Floating point operations performed in AB")
def add_instrumentation(self, mgr, observer):
mgr.set_constant("depositor", self.__class__.__name__)
for key, value in self.log_constants.iteritems():
mgr.set_constant(key, value)
from pytools.log import IntervalTimer, EventCounter,\
time_and_count_function
self.deposit_timer = IntervalTimer("t_deposit",
"Time spent depositing")
self.deposit_counter = EventCounter("n_deposit",
"Number of depositions")
self.deposit_densities = time_and_count_function(
self.deposit_densites, self.deposit_timer, self.deposit_counter,
1 + self.method.dimensions_velocity)
self.deposit_j = time_and_count_function(
self.deposit_j, self.deposit_timer, self.deposit_counter,
self.method.dimensions_velocity)
self.deposit_rho = time_and_count_function(self.deposit_rho,
self.deposit_timer,
self.deposit_counter)
mgr.add_quantity(self.deposit_timer)
mgr.add_quantity(self.deposit_counter)
def add_instrumentation(self, mgr):
self.subdiscr.add_instrumentation(mgr)
from pytools.log import EventCounter
self.comm_flux_counter = EventCounter(
"n_comm_flux", "Number of inner flux communication runs")
mgr.add_quantity(self.comm_flux_counter)
def __init__(self,
discr,
units,
depositor,
pusher,
dimensions_pos,
dimensions_velocity,
debug=set()):
self.units = units
self.discretization = discr
self.debug = debug
self.depositor = depositor
self.pusher = pusher
self.dimensions_mesh = discr.dimensions
self.dimensions_pos = dimensions_pos
self.dimensions_velocity = dimensions_velocity
dims = (dimensions_pos, dimensions_velocity)
self.mesh_data = _internal.MeshData(discr.dimensions)
self.mesh_data.fill_from_hedge(discr)
# subsystem init
self.depositor.initialize(self)
self.pusher.initialize(self)
# instrumentation
from pytools.log import IntervalTimer, EventCounter
self.find_el_timer = IntervalTimer("t_find",
"Time spent finding new elements")
self.find_same_counter = EventCounter(
"n_find_same", "#Particles found in same element")
self.find_by_neighbor_counter = EventCounter(
"n_find_neighbor", "#Particles found through neighbor")
self.find_by_vertex_counter = EventCounter(
"n_find_by_vertex", "#Particles found by vertex")
self.find_global_counter = EventCounter(
"n_find_global", "#Particles found by global search")
def __init__(self,
use_high_order=True,
dtype=numpy.float64,
rcon=None,
vector_primitive_factory=None,
atol=0,
rtol=0,
max_dt_growth=5,
min_dt_shrinkage=0.1,
limiter=None):
if vector_primitive_factory is None:
from hedge.vector_primitives import VectorPrimitiveFactory
self.vector_primitive_factory = VectorPrimitiveFactory()
else:
self.vector_primitive_factory = vector_primitive_factory
from pytools.log import IntervalTimer, EventCounter
timer_factory = IntervalTimer
if rcon is not None:
timer_factory = rcon.make_timer
if limiter is None:
self.limiter = lambda x: x
else:
self.limiter = limiter
self.timer = timer_factory("t_rk",
"Time spent doing algebra in Runge-Kutta")
self.flop_counter = EventCounter(
"n_flops_rk", "Floating point operations performed in Runge-Kutta")
self.use_high_order = use_high_order
self.dtype = numpy.dtype(dtype)
self.adaptive = bool(atol or rtol)
self.atol = atol
self.rtol = rtol
from pytools import match_precision
self.scalar_dtype = match_precision(numpy.dtype(numpy.float64),
self.dtype)
self.max_dt_growth = max_dt_growth
self.min_dt_shrinkage = min_dt_shrinkage
self.linear_combiner_cache = {}
def mpi_communication_entrypoint():
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
from mpi4py import MPI
comm = MPI.COMM_WORLD
i_local_rank = comm.Get_rank()
num_parts = comm.Get_size()
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
dt = 0.04
order = 4
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim,
b=(0.5, ) * dim,
n=(16, ) * dim)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element,
num_parts)
else:
local_mesh = mesh_dist.receive_mesh_part()
vol_discr = DGDiscretizationWithBoundaries(cl_ctx,
local_mesh,
order=order,
mpi_communicator=comm)
source_center = np.array([0.1, 0.22, 0.33])[:local_mesh.dim]
source_width = 0.05
source_omega = 3
sym_x = sym.nodes(local_mesh.dim)
sym_source_center_dist = sym_x - source_center
sym_t = sym.ScalarVariable("t")
from grudge.models.wave import StrongWaveOperator
from meshmode.mesh import BTAG_ALL, BTAG_NONE
op = StrongWaveOperator(
-0.1,
vol_discr.dim,
source_f=(
sym.sin(source_omega * sym_t) *
sym.exp(-np.dot(sym_source_center_dist, sym_source_center_dist) /
source_width**2)),
dirichlet_tag=BTAG_NONE,
neumann_tag=BTAG_NONE,
radiation_tag=BTAG_ALL,
flux_type="upwind")
from pytools.obj_array import join_fields
fields = join_fields(
vol_discr.zeros(queue),
[vol_discr.zeros(queue) for i in range(vol_discr.dim)])
# FIXME
# dt = op.estimate_rk4_timestep(vol_discr, fields=fields)
# FIXME: Should meshmode consider BTAG_PARTITION to be a boundary?
# Fails because: "found faces without boundary conditions"
# op.check_bc_coverage(local_mesh)
from pytools.log import LogManager, \
add_general_quantities, \
add_run_info, \
IntervalTimer, EventCounter
log_filename = None
# NOTE: LogManager hangs when using a file on a shared directory.
# log_filename = 'grudge_log.dat'
logmgr = LogManager(log_filename, "w", comm)
add_run_info(logmgr)
add_general_quantities(logmgr)
log_quantities =\
{"rank_data_swap_timer": IntervalTimer("rank_data_swap_timer",
"Time spent evaluating RankDataSwapAssign"),
"rank_data_swap_counter": EventCounter("rank_data_swap_counter",
"Number of RankDataSwapAssign instructions evaluated"),
"exec_timer": IntervalTimer("exec_timer",
"Total time spent executing instructions"),
"insn_eval_timer": IntervalTimer("insn_eval_timer",
"Time spend evaluating instructions"),
"future_eval_timer": IntervalTimer("future_eval_timer",
"Time spent evaluating futures"),
"busy_wait_timer": IntervalTimer("busy_wait_timer",
"Time wasted doing busy wait")}
for quantity in log_quantities.values():
logmgr.add_quantity(quantity)
# print(sym.pretty(op.sym_operator()))
bound_op = bind(vol_discr, op.sym_operator())
# print(bound_op)
# 1/0
def rhs(t, w):
val, rhs.profile_data = bound_op(queue,
profile_data=rhs.profile_data,
log_quantities=log_quantities,
t=t,
w=w)
return val
rhs.profile_data = {}
dt_stepper = set_up_rk4("w", dt, fields, rhs)
final_t = 4
nsteps = int(final_t / dt)
print("rank=%d dt=%g nsteps=%d" % (i_local_rank, dt, nsteps))
# from grudge.shortcuts import make_visualizer
# vis = make_visualizer(vol_discr, vis_order=order)
step = 0
norm = bind(vol_discr, sym.norm(2, sym.var("u")))
from time import time
t_last_step = time()
logmgr.tick_before()
for event in dt_stepper.run(t_end=final_t):
if isinstance(event, dt_stepper.StateComputed):
assert event.component_id == "w"
step += 1
print(step, event.t, norm(queue, u=event.state_component[0]),
time() - t_last_step)
# if step % 10 == 0:
# vis.write_vtk_file("rank%d-fld-%04d.vtu" % (i_local_rank, step),
# [("u", event.state_component[0]),
# ("v", event.state_component[1:])])
t_last_step = time()
logmgr.tick_after()
logmgr.tick_before()
logmgr.tick_after()
def print_profile_data(data):
print("""execute() for rank %d:
\tInstruction Evaluation: %f%%
\tFuture Evaluation: %f%%
\tBusy Wait: %f%%
\tTotal: %f seconds""" %
(i_local_rank, data['insn_eval_time'] / data['total_time'] * 100,
data['future_eval_time'] / data['total_time'] * 100,
data['busy_wait_time'] / data['total_time'] * 100,
data['total_time']))
print_profile_data(rhs.profile_data)
logmgr.close()
logger.debug("Rank %d exiting", i_local_rank)
def simple_wave_entrypoint(dim=2,
num_elems=256,
order=4,
num_steps=30,
log_filename="grudge.dat"):
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
from mpi4py import MPI
comm = MPI.COMM_WORLD
num_parts = comm.Get_size()
n = int(num_elems**(1. / dim))
from meshmode.distributed import MPIMeshDistributor
mesh_dist = MPIMeshDistributor(comm)
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim,
b=(0.5, ) * dim,
n=(n, ) * dim)
from pymetis import part_graph
_, p = part_graph(num_parts,
xadj=mesh.nodal_adjacency.neighbors_starts.tolist(),
adjncy=mesh.nodal_adjacency.neighbors.tolist())
part_per_element = np.array(p)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element,
num_parts)
else:
local_mesh = mesh_dist.receive_mesh_part()
vol_discr = DGDiscretizationWithBoundaries(cl_ctx,
local_mesh,
order=order,
mpi_communicator=comm)
source_center = np.array([0.1, 0.22, 0.33])[:local_mesh.dim]
source_width = 0.05
source_omega = 3
sym_x = sym.nodes(local_mesh.dim)
sym_source_center_dist = sym_x - source_center
sym_t = sym.ScalarVariable("t")
from grudge.models.wave import StrongWaveOperator
from meshmode.mesh import BTAG_ALL, BTAG_NONE
op = StrongWaveOperator(
-0.1,
vol_discr.dim,
source_f=(
sym.sin(source_omega * sym_t) *
sym.exp(-np.dot(sym_source_center_dist, sym_source_center_dist) /
source_width**2)),
dirichlet_tag=BTAG_NONE,
neumann_tag=BTAG_NONE,
radiation_tag=BTAG_ALL,
flux_type="upwind")
from pytools.obj_array import join_fields
fields = join_fields(
vol_discr.zeros(queue),
[vol_discr.zeros(queue) for i in range(vol_discr.dim)])
from pytools.log import LogManager, \
add_general_quantities, \
add_run_info, \
IntervalTimer, EventCounter
# NOTE: LogManager hangs when using a file on a shared directory.
logmgr = LogManager(log_filename, "w", comm)
add_run_info(logmgr)
add_general_quantities(logmgr)
log_quantities =\
{"rank_data_swap_timer": IntervalTimer("rank_data_swap_timer",
"Time spent evaluating RankDataSwapAssign"),
"rank_data_swap_counter": EventCounter("rank_data_swap_counter",
"Number of RankDataSwapAssign instructions evaluated"),
"exec_timer": IntervalTimer("exec_timer",
"Total time spent executing instructions"),
"insn_eval_timer": IntervalTimer("insn_eval_timer",
"Time spend evaluating instructions"),
"future_eval_timer": IntervalTimer("future_eval_timer",
"Time spent evaluating futures"),
"busy_wait_timer": IntervalTimer("busy_wait_timer",
"Time wasted doing busy wait")}
for quantity in log_quantities.values():
logmgr.add_quantity(quantity)
bound_op = bind(vol_discr, op.sym_operator())
def rhs(t, w):
val, rhs.profile_data = bound_op(queue,
profile_data=rhs.profile_data,
log_quantities=log_quantities,
t=t,
w=w)
return val
rhs.profile_data = {}
dt = 0.04
dt_stepper = set_up_rk4("w", dt, fields, rhs)
logmgr.tick_before()
for event in dt_stepper.run(t_end=dt * num_steps):
if isinstance(event, dt_stepper.StateComputed):
logmgr.tick_after()
logmgr.tick_before()
logmgr.tick_after()
def print_profile_data(data):
print("""execute() for rank %d:
\tInstruction Evaluation: %f%%
\tFuture Evaluation: %f%%
\tBusy Wait: %f%%
\tTotal: %f seconds""" %
(comm.Get_rank(), data['insn_eval_time'] / data['total_time'] *
100, data['future_eval_time'] / data['total_time'] * 100,
data['busy_wait_time'] / data['total_time'] * 100,
data['total_time']))
print_profile_data(rhs.profile_data)
logmgr.close()
