def my_checkpoint(step, t, dt, state): if check_step(step, nrestart) and step != restart_step: rst_filename = (restart_path + restart_file_pattern.format( casename=casename, step=step, rank=rank)) rst_data = { "local_mesh": local_mesh, "state": current_state, "t": t, "step": step, "global_nelements": global_nelements, "num_parts": nproc } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_filename, comm) # awful - computes potentially expensive viz quantities # regardless of whether it is time to viz reaction_rates = eos.get_production_rates(state) viz_fields = [("reaction_rates", reaction_rates)] return sim_checkpoint(discr, visualizer, eos, cv=state, vizname=casename, step=step, t=t, dt=dt, nstatus=nstatus, nviz=nviz, constant_cfl=constant_cfl, comm=comm, viz_fields=viz_fields)
def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) if rst_fname != rst_filename: rst_data = { "local_mesh": local_mesh, "state": state, "t": t, "step": step, "order": order, "global_nelements": global_nelements, "num_parts": nparts } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm)
def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) if rst_fname == rst_filename: if rank == 0: logger.info("Skipping overwrite of restart file.") else: rst_data = { "local_mesh": local_mesh, "state": state, "t": t, "step": step, "order": order, "global_nelements": global_nelements, "num_parts": nproc } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm)
def test_restart_cv(actx_factory, nspecies): """Test that restart can read a CV array container.""" actx = actx_factory() nel_1d = 4 dim = 3 from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim, b=(0.5, ) * dim, nelements_per_axis=(nel_1d, ) * dim) order = 3 discr = EagerDGDiscretization(actx, mesh, order=order) from meshmode.dof_array import thaw nodes = thaw(actx, discr.nodes()) mass = nodes[0] energy = nodes[1] mom = make_obj_array([nodes[2] * (i + 3) for i in range(dim)]) species_mass = None if nspecies > 0: mass_fractions = make_obj_array( [i * nodes[0] for i in range(nspecies)]) species_mass = mass * mass_fractions rst_filename = f"test_{nspecies}.pkl" from mirgecom.fluid import make_conserved test_state = make_conserved(dim, mass=mass, energy=energy, momentum=mom, species_mass=species_mass) rst_data = {"state": test_state} from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_filename) from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_filename) resid = test_state - restart_data["state"] assert discr.norm(resid.join(), np.inf) == 0
def my_checkpoint(step, t, dt, state, force=False): do_health = force or check_step(step, nhealth) and step > 0 do_viz = force or check_step(step, nviz) do_restart = force or check_step(step, nrestart) do_status = force or check_step(step, nstatus) if do_viz or do_health: dv = eos.dependent_vars(state) errors = False if do_health: health_message = "" if check_naninf_local(discr, "vol", dv.pressure): errors = True health_message += "Invalid pressure data found.\n" elif check_range_local(discr, "vol", dv.pressure, min_value=1, max_value=2.e6): errors = True health_message += "Pressure data failed health check.\n" errors = comm.allreduce(errors, MPI.LOR) if errors: if rank == 0: logger.info("Fluid solution failed health check.") if health_message: logger.info(f"{rank=}: {health_message}") #if check_step(step, nrestart) and step != restart_step and not errors: if do_restart or errors: filename = restart_path + snapshot_pattern.format( step=step, rank=rank, casename=casename) restart_dictionary = { "local_mesh": local_mesh, "order": order, "state": state, "t": t, "step": step, "global_nelements": global_nelements, "num_parts": nparts } write_restart_file(actx, restart_dictionary, filename, comm) if do_status or do_viz or errors: local_cfl = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) max_cfl = nodal_max(discr, "vol", local_cfl) log_cfl.set_quantity(max_cfl) #if ((check_step(step, nviz) and step != restart_step) or errors): if do_viz or errors: def loc_fn(t): return flame_start_loc + flame_speed * t #exact_soln = PlanarDiscontinuity(dim=dim, disc_location=loc_fn, #sigma=0.0000001, nspecies=nspecies, #temperature_left=temp_ignition, temperature_right=temp_unburned, #pressure_left=pres_burned, pressure_right=pres_unburned, #velocity_left=vel_burned, velocity_right=vel_unburned, #species_mass_left=y_burned, species_mass_right=y_unburned) reaction_rates = eos.get_production_rates(cv=state) # conserved quantities viz_fields = [ #("cv", state), ("CV_rho", state.mass), ("CV_rhoU", state.momentum[0]), ("CV_rhoV", state.momentum[1]), ("CV_rhoE", state.energy) ] # species mass fractions viz_fields.extend( ("Y_" + species_names[i], state.species_mass[i] / state.mass) for i in range(nspecies)) # dependent variables viz_fields.extend([ ("DV", eos.dependent_vars(state)), #("exact_soln", exact_soln), ("reaction_rates", reaction_rates), ("cfl", local_cfl) ]) write_visfile(discr, viz_fields, visualizer, vizname=viz_path + casename, step=step, t=t, overwrite=True, vis_timer=vis_timer) if errors: raise RuntimeError("Error detected by user checkpoint, exiting.")
def main(snapshot_pattern="wave-eager-{step:04d}-{rank:04d}.pkl", restart_step=None): """Drive the example.""" cl_ctx = cl.create_some_context() queue = cl.CommandQueue(cl_ctx) actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool( cl_tools.ImmediateAllocator(queue))) from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() num_parts = comm.Get_size() if restart_step is None: from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis mesh_dist = MPIMeshDistributor(comm) dim = 2 nel_1d = 16 if mesh_dist.is_mananger_rank(): from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim, b=(0.5, ) * dim, nelements_per_axis=(nel_1d, ) * dim) print("%d elements" % mesh.nelements) part_per_element = get_partition_by_pymetis(mesh, num_parts) local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts) del mesh else: local_mesh = mesh_dist.receive_mesh_part() fields = None else: from mirgecom.restart import read_restart_data restart_data = read_restart_data( actx, snapshot_pattern.format(step=restart_step, rank=rank)) local_mesh = restart_data["local_mesh"] nel_1d = restart_data["nel_1d"] assert comm.Get_size() == restart_data["num_parts"] order = 3 discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) if local_mesh.dim == 2: # no deep meaning here, just a fudge factor dt = 0.7 / (nel_1d * order**2) elif dim == 3: # no deep meaning here, just a fudge factor dt = 0.4 / (nel_1d * order**2) else: raise ValueError("don't have a stable time step guesstimate") t_final = 3 if restart_step is None: t = 0 istep = 0 fields = flat_obj_array(bump(actx, discr), [discr.zeros(actx) for i in range(discr.dim)]) else: t = restart_data["t"] istep = restart_step assert istep == restart_step restart_fields = restart_data["fields"] old_order = restart_data["order"] if old_order != order: old_discr = EagerDGDiscretization(actx, local_mesh, order=old_order, mpi_communicator=comm) from meshmode.discretization.connection import make_same_mesh_connection connection = make_same_mesh_connection( actx, discr.discr_from_dd("vol"), old_discr.discr_from_dd("vol")) fields = connection(restart_fields) else: fields = restart_fields vis = make_visualizer(discr) def rhs(t, w): return wave_operator(discr, c=1, w=w) while t < t_final: # restart must happen at beginning of step if istep % 100 == 0 and ( # Do not overwrite the restart file that we just read. istep != restart_step): from mirgecom.restart import write_restart_file write_restart_file(actx, restart_data={ "local_mesh": local_mesh, "order": order, "fields": fields, "t": t, "step": istep, "nel_1d": nel_1d, "num_parts": num_parts }, filename=snapshot_pattern.format(step=istep, rank=rank), comm=comm) if istep % 10 == 0: print(istep, t, discr.norm(fields[0])) vis.write_parallel_vtk_file( comm, "fld-wave-eager-mpi-%03d-%04d.vtu" % (rank, istep), [ ("u", fields[0]), ("v", fields[1:]), ]) fields = rk4_step(fields, t, dt, rhs) t += dt istep += 1
def main(snapshot_pattern="wave-mpi-{step:04d}-{rank:04d}.pkl", restart_step=None, use_profiling=False, use_logmgr=False, actx_class=PyOpenCLArrayContext): """Drive the example.""" cl_ctx = cl.create_some_context() queue = cl.CommandQueue(cl_ctx) from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() num_parts = comm.Get_size() logmgr = initialize_logmgr(use_logmgr, filename="wave-mpi.sqlite", mode="wu", mpi_comm=comm) if use_profiling: queue = cl.CommandQueue(cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) actx = actx_class(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), logmgr=logmgr) else: queue = cl.CommandQueue(cl_ctx) actx = actx_class(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) if restart_step is None: from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis mesh_dist = MPIMeshDistributor(comm) dim = 2 nel_1d = 16 if mesh_dist.is_mananger_rank(): from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh( a=(-0.5,)*dim, b=(0.5,)*dim, nelements_per_axis=(nel_1d,)*dim) print("%d elements" % mesh.nelements) part_per_element = get_partition_by_pymetis(mesh, num_parts) local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts) del mesh else: local_mesh = mesh_dist.receive_mesh_part() fields = None else: from mirgecom.restart import read_restart_data restart_data = read_restart_data( actx, snapshot_pattern.format(step=restart_step, rank=rank) ) local_mesh = restart_data["local_mesh"] nel_1d = restart_data["nel_1d"] assert comm.Get_size() == restart_data["num_parts"] order = 3 discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) current_cfl = 0.485 wave_speed = 1.0 from grudge.dt_utils import characteristic_lengthscales dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed from grudge.op import nodal_min dt = nodal_min(discr, "vol", dt) t_final = 1 if restart_step is None: t = 0 istep = 0 fields = flat_obj_array( bump(actx, discr), [discr.zeros(actx) for i in range(discr.dim)] ) else: t = restart_data["t"] istep = restart_step assert istep == restart_step restart_fields = restart_data["fields"] old_order = restart_data["order"] if old_order != order: old_discr = EagerDGDiscretization(actx, local_mesh, order=old_order, mpi_communicator=comm) from meshmode.discretization.connection import make_same_mesh_connection connection = make_same_mesh_connection(actx, discr.discr_from_dd("vol"), old_discr.discr_from_dd("vol")) fields = connection(restart_fields) else: fields = restart_fields if logmgr: logmgr_add_cl_device_info(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) logmgr.add_watches(["step.max", "t_step.max", "t_log.max"]) try: logmgr.add_watches(["memory_usage_python.max", "memory_usage_gpu.max"]) except KeyError: pass if use_profiling: logmgr.add_watches(["multiply_time.max"]) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) vis = make_visualizer(discr) def rhs(t, w): return wave_operator(discr, c=wave_speed, w=w) compiled_rhs = actx.compile(rhs) while t < t_final: if logmgr: logmgr.tick_before() # restart must happen at beginning of step if istep % 100 == 0 and ( # Do not overwrite the restart file that we just read. istep != restart_step): from mirgecom.restart import write_restart_file write_restart_file( actx, restart_data={ "local_mesh": local_mesh, "order": order, "fields": fields, "t": t, "step": istep, "nel_1d": nel_1d, "num_parts": num_parts}, filename=snapshot_pattern.format(step=istep, rank=rank), comm=comm ) if istep % 10 == 0: print(istep, t, discr.norm(fields[0])) vis.write_parallel_vtk_file( comm, "fld-wave-mpi-%03d-%04d.vtu" % (rank, istep), [ ("u", fields[0]), ("v", fields[1:]), ], overwrite=True ) fields = thaw(freeze(fields, actx), actx) fields = rk4_step(fields, t, dt, compiled_rhs) t += dt istep += 1 if logmgr: set_dt(logmgr, dt) logmgr.tick_after() final_soln = discr.norm(fields[0]) assert np.abs(final_soln - 0.04409852463947439) < 1e-14