def my_checkpoint(step, t, dt, state):
if check_step(step, nrestart) and step != restart_step:
rst_filename = (restart_path + restart_file_pattern.format(
casename=casename, step=step, rank=rank))
rst_data = {
"local_mesh": local_mesh,
"state": current_state,
"t": t,
"step": step,
"global_nelements": global_nelements,
"num_parts": nproc
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_filename, comm)
# awful - computes potentially expensive viz quantities
# regardless of whether it is time to viz
reaction_rates = eos.get_production_rates(state)
viz_fields = [("reaction_rates", reaction_rates)]
return sim_checkpoint(discr,
visualizer,
eos,
cv=state,
vizname=casename,
step=step,
t=t,
dt=dt,
nstatus=nstatus,
nviz=nviz,
constant_cfl=constant_cfl,
comm=comm,
viz_fields=viz_fields)
def my_write_restart(step, t, state):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname != rst_filename:
rst_data = {
"local_mesh": local_mesh,
"state": state,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": nparts
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_write_restart(step, t, state):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname == rst_filename:
if rank == 0:
logger.info("Skipping overwrite of restart file.")
else:
rst_data = {
"local_mesh": local_mesh,
"state": state,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": nproc
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def test_restart_cv(actx_factory, nspecies):
"""Test that restart can read a CV array container."""
actx = actx_factory()
nel_1d = 4
dim = 3
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim,
b=(0.5, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
order = 3
discr = EagerDGDiscretization(actx, mesh, order=order)
from meshmode.dof_array import thaw
nodes = thaw(actx, discr.nodes())
mass = nodes[0]
energy = nodes[1]
mom = make_obj_array([nodes[2] * (i + 3) for i in range(dim)])
species_mass = None
if nspecies > 0:
mass_fractions = make_obj_array(
[i * nodes[0] for i in range(nspecies)])
species_mass = mass * mass_fractions
rst_filename = f"test_{nspecies}.pkl"
from mirgecom.fluid import make_conserved
test_state = make_conserved(dim,
mass=mass,
energy=energy,
momentum=mom,
species_mass=species_mass)
rst_data = {"state": test_state}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_filename)
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
resid = test_state - restart_data["state"]
assert discr.norm(resid.join(), np.inf) == 0
def my_checkpoint(step, t, dt, state, force=False):
do_health = force or check_step(step, nhealth) and step > 0
do_viz = force or check_step(step, nviz)
do_restart = force or check_step(step, nrestart)
do_status = force or check_step(step, nstatus)
if do_viz or do_health:
dv = eos.dependent_vars(state)
errors = False
if do_health:
health_message = ""
if check_naninf_local(discr, "vol", dv.pressure):
errors = True
health_message += "Invalid pressure data found.\n"
elif check_range_local(discr,
"vol",
dv.pressure,
min_value=1,
max_value=2.e6):
errors = True
health_message += "Pressure data failed health check.\n"
errors = comm.allreduce(errors, MPI.LOR)
if errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
if health_message:
logger.info(f"{rank=}: {health_message}")
#if check_step(step, nrestart) and step != restart_step and not errors:
if do_restart or errors:
filename = restart_path + snapshot_pattern.format(
step=step, rank=rank, casename=casename)
restart_dictionary = {
"local_mesh": local_mesh,
"order": order,
"state": state,
"t": t,
"step": step,
"global_nelements": global_nelements,
"num_parts": nparts
}
write_restart_file(actx, restart_dictionary, filename, comm)
if do_status or do_viz or errors:
local_cfl = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state)
max_cfl = nodal_max(discr, "vol", local_cfl)
log_cfl.set_quantity(max_cfl)
#if ((check_step(step, nviz) and step != restart_step) or errors):
if do_viz or errors:
def loc_fn(t):
return flame_start_loc + flame_speed * t
#exact_soln = PlanarDiscontinuity(dim=dim, disc_location=loc_fn,
#sigma=0.0000001, nspecies=nspecies,
#temperature_left=temp_ignition, temperature_right=temp_unburned,
#pressure_left=pres_burned, pressure_right=pres_unburned,
#velocity_left=vel_burned, velocity_right=vel_unburned,
#species_mass_left=y_burned, species_mass_right=y_unburned)
reaction_rates = eos.get_production_rates(cv=state)
# conserved quantities
viz_fields = [
#("cv", state),
("CV_rho", state.mass),
("CV_rhoU", state.momentum[0]),
("CV_rhoV", state.momentum[1]),
("CV_rhoE", state.energy)
]
# species mass fractions
viz_fields.extend(
("Y_" + species_names[i], state.species_mass[i] / state.mass)
for i in range(nspecies))
# dependent variables
viz_fields.extend([
("DV", eos.dependent_vars(state)),
#("exact_soln", exact_soln),
("reaction_rates", reaction_rates),
("cfl", local_cfl)
])
write_visfile(discr,
viz_fields,
visualizer,
vizname=viz_path + casename,
step=step,
t=t,
overwrite=True,
vis_timer=vis_timer)
if errors:
raise RuntimeError("Error detected by user checkpoint, exiting.")
def main(snapshot_pattern="wave-eager-{step:04d}-{rank:04d}.pkl",
restart_step=None):
"""Drive the example."""
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
num_parts = comm.Get_size()
if restart_step is None:
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
nel_1d = 16
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim,
b=(0.5, ) * dim,
nelements_per_axis=(nel_1d, ) *
dim)
print("%d elements" % mesh.nelements)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element,
num_parts)
del mesh
else:
local_mesh = mesh_dist.receive_mesh_part()
fields = None
else:
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(
actx, snapshot_pattern.format(step=restart_step, rank=rank))
local_mesh = restart_data["local_mesh"]
nel_1d = restart_data["nel_1d"]
assert comm.Get_size() == restart_data["num_parts"]
order = 3
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
if local_mesh.dim == 2:
# no deep meaning here, just a fudge factor
dt = 0.7 / (nel_1d * order**2)
elif dim == 3:
# no deep meaning here, just a fudge factor
dt = 0.4 / (nel_1d * order**2)
else:
raise ValueError("don't have a stable time step guesstimate")
t_final = 3
if restart_step is None:
t = 0
istep = 0
fields = flat_obj_array(bump(actx, discr),
[discr.zeros(actx) for i in range(discr.dim)])
else:
t = restart_data["t"]
istep = restart_step
assert istep == restart_step
restart_fields = restart_data["fields"]
old_order = restart_data["order"]
if old_order != order:
old_discr = EagerDGDiscretization(actx,
local_mesh,
order=old_order,
mpi_communicator=comm)
from meshmode.discretization.connection import make_same_mesh_connection
connection = make_same_mesh_connection(
actx, discr.discr_from_dd("vol"),
old_discr.discr_from_dd("vol"))
fields = connection(restart_fields)
else:
fields = restart_fields
vis = make_visualizer(discr)
def rhs(t, w):
return wave_operator(discr, c=1, w=w)
while t < t_final:
# restart must happen at beginning of step
if istep % 100 == 0 and (
# Do not overwrite the restart file that we just read.
istep != restart_step):
from mirgecom.restart import write_restart_file
write_restart_file(actx,
restart_data={
"local_mesh": local_mesh,
"order": order,
"fields": fields,
"t": t,
"step": istep,
"nel_1d": nel_1d,
"num_parts": num_parts
},
filename=snapshot_pattern.format(step=istep,
rank=rank),
comm=comm)
if istep % 10 == 0:
print(istep, t, discr.norm(fields[0]))
vis.write_parallel_vtk_file(
comm, "fld-wave-eager-mpi-%03d-%04d.vtu" % (rank, istep), [
("u", fields[0]),
("v", fields[1:]),
])
fields = rk4_step(fields, t, dt, rhs)
t += dt
istep += 1
def main(snapshot_pattern="wave-mpi-{step:04d}-{rank:04d}.pkl", restart_step=None,
use_profiling=False, use_logmgr=False, actx_class=PyOpenCLArrayContext):
"""Drive the example."""
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
num_parts = comm.Get_size()
logmgr = initialize_logmgr(use_logmgr,
filename="wave-mpi.sqlite", mode="wu", mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(cl_ctx,
properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
if restart_step is None:
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
nel_1d = 16
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(
a=(-0.5,)*dim, b=(0.5,)*dim,
nelements_per_axis=(nel_1d,)*dim)
print("%d elements" % mesh.nelements)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts)
del mesh
else:
local_mesh = mesh_dist.receive_mesh_part()
fields = None
else:
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(
actx, snapshot_pattern.format(step=restart_step, rank=rank)
)
local_mesh = restart_data["local_mesh"]
nel_1d = restart_data["nel_1d"]
assert comm.Get_size() == restart_data["num_parts"]
order = 3
discr = EagerDGDiscretization(actx, local_mesh, order=order,
mpi_communicator=comm)
current_cfl = 0.485
wave_speed = 1.0
from grudge.dt_utils import characteristic_lengthscales
dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed
from grudge.op import nodal_min
dt = nodal_min(discr, "vol", dt)
t_final = 1
if restart_step is None:
t = 0
istep = 0
fields = flat_obj_array(
bump(actx, discr),
[discr.zeros(actx) for i in range(discr.dim)]
)
else:
t = restart_data["t"]
istep = restart_step
assert istep == restart_step
restart_fields = restart_data["fields"]
old_order = restart_data["order"]
if old_order != order:
old_discr = EagerDGDiscretization(actx, local_mesh, order=old_order,
mpi_communicator=comm)
from meshmode.discretization.connection import make_same_mesh_connection
connection = make_same_mesh_connection(actx, discr.discr_from_dd("vol"),
old_discr.discr_from_dd("vol"))
fields = connection(restart_fields)
else:
fields = restart_fields
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr.add_watches(["step.max", "t_step.max", "t_log.max"])
try:
logmgr.add_watches(["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["multiply_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
vis = make_visualizer(discr)
def rhs(t, w):
return wave_operator(discr, c=wave_speed, w=w)
compiled_rhs = actx.compile(rhs)
while t < t_final:
if logmgr:
logmgr.tick_before()
# restart must happen at beginning of step
if istep % 100 == 0 and (
# Do not overwrite the restart file that we just read.
istep != restart_step):
from mirgecom.restart import write_restart_file
write_restart_file(
actx, restart_data={
"local_mesh": local_mesh,
"order": order,
"fields": fields,
"t": t,
"step": istep,
"nel_1d": nel_1d,
"num_parts": num_parts},
filename=snapshot_pattern.format(step=istep, rank=rank),
comm=comm
)
if istep % 10 == 0:
print(istep, t, discr.norm(fields[0]))
vis.write_parallel_vtk_file(
comm,
"fld-wave-mpi-%03d-%04d.vtu" % (rank, istep),
[
("u", fields[0]),
("v", fields[1:]),
], overwrite=True
)
fields = thaw(freeze(fields, actx), actx)
fields = rk4_step(fields, t, dt, compiled_rhs)
t += dt
istep += 1
if logmgr:
set_dt(logmgr, dt)
logmgr.tick_after()
final_soln = discr.norm(fields[0])
assert np.abs(final_soln - 0.04409852463947439) < 1e-14
