def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename=None, rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive example.""" cl_ctx = ctx_factory() if casename is None: casename = "mirgecom" from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() nparts = comm.Get_size() logmgr = initialize_logmgr(use_logmgr, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: queue = cl.CommandQueue( cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) actx = actx_class( queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # timestepping control current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step t_final = 0.005 current_cfl = 1.0 current_dt = .001 current_t = 0 constant_cfl = False # some i/o frequencies nstatus = 1 nrestart = 5 nviz = 1 nhealth = 1 dim = 2 rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) if rst_filename: # read the grid from restart data rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["nparts"] == nparts else: # generate the grid from scratch box_ll = -5.0 box_ur = 5.0 nel_1d = 16 from meshmode.mesh.generation import generate_regular_rect_mesh generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) local_nelements = local_mesh.nelements order = 3 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) nodes = thaw(actx, discr.nodes()) vis_timer = None if logmgr: logmgr_add_device_name(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) logmgr.add_watches([ ("step.max", "step = {value}, "), ("t_sim.max", "sim time: {value:1.6e} s\n"), ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), ("max_pressure", "{value:1.9e})\n"), ("t_step.max", "------- step walltime: {value:6g} s, "), ("t_log.max", "log walltime: {value:6g} s") ]) # soln setup and init nspecies = 4 centers = make_obj_array([np.zeros(shape=(dim,)) for i in range(nspecies)]) spec_y0s = np.ones(shape=(nspecies,)) spec_amplitudes = np.ones(shape=(nspecies,)) eos = IdealSingleGas() velocity = np.ones(shape=(dim,)) initializer = MulticomponentLump(dim=dim, nspecies=nspecies, spec_centers=centers, velocity=velocity, spec_y0s=spec_y0s, spec_amplitudes=spec_amplitudes) boundaries = { BTAG_ALL: PrescribedInviscidBoundary(fluid_solution_func=initializer) } if rst_filename: current_t = restart_data["t"] current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_state = initializer(nodes) visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ eosname = eos.__class__.__name__ init_message = make_init_message(dim=dim, order=order, nelements=local_nelements, global_nelements=global_nelements, dt=current_dt, t_final=t_final, nstatus=nstatus, nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, initname=initname, eosname=eosname, casename=casename) if rank == 0: logger.info(init_message) def my_write_status(component_errors): if rank == 0: logger.info( "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) if exact is None: exact = initializer(x_vec=nodes, eos=eos, time=t) if resid is None: resid = state - exact viz_fields = [("cv", state), ("dv", dv), ("exact", exact), ("resid", resid)] from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True, vis_timer=vis_timer) def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) if rst_fname != rst_filename: rst_data = { "local_mesh": local_mesh, "state": state, "t": t, "step": step, "order": order, "global_nelements": global_nelements, "num_parts": nparts } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(pressure, component_errors): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure) \ or check_range_local(discr, "vol", pressure, .99999999, 1.00000001): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") exittol = .09 if max(component_errors) > exittol: health_error = True if rank == 0: logger.info("Solution diverged from exact soln.") return health_error def my_pre_step(step, t, dt, state): try: dv = None exact = None component_errors = None if logmgr: logmgr.tick_before() from mirgecom.simutil import check_step do_viz = check_step(step=step, interval=nviz) do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) do_status = check_step(step=step, interval=nstatus) if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync health_errors = allsync( my_health_check(dv.pressure, component_errors), comm, op=MPI.LOR ) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") if do_restart: my_write_restart(step=step, t=t, state=state) if do_viz: if dv is None: dv = eos.dependent_vars(state) if exact is None: exact = initializer(x_vec=nodes, eos=eos, time=t) resid = state - exact my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, resid=resid) if do_status: if component_errors is None: if exact is None: exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) my_write_status(component_errors) except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) my_write_restart(step=step, t=t, state=state) raise dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final, constant_cfl) return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() return state, dt def my_rhs(t, state): return euler_operator(discr, cv=state, time=t, boundaries=boundaries, eos=eos) current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, current_cfl, eos, t_final, constant_cfl) current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, dt=current_dt, post_step_callback=my_post_step, state=current_state, t=current_t, t_final=t_final) # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") final_dv = eos.dependent_vars(current_state) final_exact = initializer(x_vec=nodes, eos=eos, time=current_t) final_resid = current_state - final_exact my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, exact=final_exact, resid=final_resid) my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: logmgr.close() elif use_profiling: print(actx.tabulate_profiling_data()) finish_tol = 1e-16 time_err = current_t - t_final if np.abs(time_err) > finish_tol: raise ValueError(f"Simulation did not finish at expected time {time_err=}.")
def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_overintegration=False, use_leap=False, use_profiling=False, casename=None, rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive the example.""" cl_ctx = ctx_factory() if casename is None: casename = "mirgecom" from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() num_parts = comm.Get_size() from mirgecom.simutil import global_reduce as _global_reduce global_reduce = partial(_global_reduce, comm=comm) logmgr = initialize_logmgr(use_logmgr, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: queue = cl.CommandQueue( cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) actx = actx_class( queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # timestepping control current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step t_final = 0.1 current_cfl = 1.0 current_dt = .01 current_t = 0 constant_cfl = False # some i/o frequencies nstatus = 1 nrestart = 5 nviz = 10 nhealth = 1 dim = 3 rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) if rst_filename: # read the grid from restart data rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["num_parts"] == num_parts else: # generate the grid from scratch from meshmode.mesh.generation import generate_regular_rect_mesh box_ll = -1 box_ur = 1 nel_1d = 16 generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) local_nelements = local_mesh.nelements from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD from meshmode.discretization.poly_element import \ default_simplex_group_factory, QuadratureSimplexGroupFactory order = 1 discr = EagerDGDiscretization( actx, local_mesh, discr_tag_to_group_factory={ DISCR_TAG_BASE: default_simplex_group_factory( base_dim=local_mesh.dim, order=order), DISCR_TAG_QUAD: QuadratureSimplexGroupFactory(2*order + 1) }, mpi_communicator=comm ) nodes = thaw(discr.nodes(), actx) if use_overintegration: quadrature_tag = DISCR_TAG_QUAD else: quadrature_tag = None vis_timer = None if logmgr: logmgr_add_cl_device_info(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) logmgr.add_watches([ ("step.max", "step = {value}, "), ("t_sim.max", "sim time: {value:1.6e} s\n"), ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), ("max_pressure", "{value:1.9e})\n"), ("t_step.max", "------- step walltime: {value:6g} s, "), ("t_log.max", "log walltime: {value:6g} s") ]) eos = IdealSingleGas() gas_model = GasModel(eos=eos) vel = np.zeros(shape=(dim,)) orig = np.zeros(shape=(dim,)) initializer = Lump(dim=dim, center=orig, velocity=vel, rhoamp=0.0) wall = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: wall} uniform_state = initializer(nodes) acoustic_pulse = AcousticPulse(dim=dim, amplitude=1.0, width=.1, center=orig) if rst_filename: current_t = restart_data["t"] current_step = restart_data["step"] current_cv = restart_data["cv"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_cv = acoustic_pulse(x_vec=nodes, cv=uniform_state, eos=eos) current_state = make_fluid_state(current_cv, gas_model) visualizer = make_visualizer(discr) initname = "pulse" eosname = eos.__class__.__name__ init_message = make_init_message(dim=dim, order=order, nelements=local_nelements, global_nelements=global_nelements, dt=current_dt, t_final=t_final, nstatus=nstatus, nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, initname=initname, eosname=eosname, casename=casename) if rank == 0: logger.info(init_message) def my_write_viz(step, t, state, dv=None): if dv is None: dv = eos.dependent_vars(state) viz_fields = [("cv", state), ("dv", dv)] from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True, vis_timer=vis_timer) def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) if rst_fname != rst_filename: rst_data = { "local_mesh": local_mesh, "cv": state, "t": t, "step": step, "order": order, "global_nelements": global_nelements, "num_parts": num_parts } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(pressure): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure) \ or check_range_local(discr, "vol", pressure, .8, 1.5): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") return health_error def my_pre_step(step, t, dt, state): fluid_state = make_fluid_state(state, gas_model) dv = fluid_state.dv try: if logmgr: logmgr.tick_before() from mirgecom.simutil import check_step do_viz = check_step(step=step, interval=nviz) do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) if do_health: health_errors = global_reduce(my_health_check(dv.pressure), op="lor") if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") if do_restart: my_write_restart(step=step, t=t, state=state) if do_viz: my_write_viz(step=step, t=t, state=state, dv=dv) except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) my_write_restart(step=step, t=t, state=state) raise dt = get_sim_timestep(discr, fluid_state, t, dt, current_cfl, t_final, constant_cfl) return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() return state, dt def my_rhs(t, state): fluid_state = make_fluid_state(cv=state, gas_model=gas_model) return euler_operator(discr, state=fluid_state, time=t, boundaries=boundaries, gas_model=gas_model, quadrature_tag=quadrature_tag) current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, current_cfl, t_final, constant_cfl) current_step, current_t, current_cv = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, state=current_cv, t=current_t, t_final=t_final) # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") final_state = make_fluid_state(current_cv, gas_model) final_dv = final_state.dv my_write_viz(step=current_step, t=current_t, state=current_cv, dv=final_dv) my_write_restart(step=current_step, t=current_t, state=current_cv) if logmgr: logmgr.close() elif use_profiling: print(actx.tabulate_profiling_data()) finish_tol = 1e-16 assert np.abs(current_t - t_final) < finish_tol
def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename=None, rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive the example.""" cl_ctx = ctx_factory() if casename is None: casename = "mirgecom" from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() num_parts = comm.Get_size() from mirgecom.simutil import global_reduce as _global_reduce global_reduce = partial(_global_reduce, comm=comm) logmgr = initialize_logmgr(use_logmgr, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: queue = cl.CommandQueue( cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) actx = actx_class(queue, allocator=cl_tools.MemoryPool( cl_tools.ImmediateAllocator(queue))) # timestepping control current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step t_final = 0.01 current_cfl = 1.0 current_dt = .001 current_t = 0 constant_cfl = False # some i/o frequencies nrestart = 10 nstatus = 1 nviz = 10 nhealth = 10 dim = 2 if dim != 2: raise ValueError("This example must be run with dim = 2.") rst_path = "restart_data/" rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl") if rst_filename: # read the grid from restart data rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["num_parts"] == num_parts else: # generate the grid from scratch nel_1d = 16 box_ll = -5.0 box_ur = 5.0 from meshmode.mesh.generation import generate_regular_rect_mesh generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll, ) * dim, b=(box_ur, ) * dim, nelements_per_axis=(nel_1d, ) * dim) local_mesh, global_nelements = generate_and_distribute_mesh( comm, generate_mesh) local_nelements = local_mesh.nelements order = 3 discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) nodes = thaw(discr.nodes(), actx) vis_timer = None if logmgr: logmgr_add_device_name(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) logmgr.add_watches([ ("step.max", "step = {value}, "), ("t_sim.max", "sim time: {value:1.6e} s\n"), ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), ("max_pressure", "{value:1.9e})\n"), ("t_step.max", "------- step walltime: {value:6g} s, "), ("t_log.max", "log walltime: {value:6g} s") ]) try: logmgr.add_watches( ["memory_usage_python.max", "memory_usage_gpu.max"]) except KeyError: pass if use_profiling: logmgr.add_watches(["multiply_time.max"]) # soln setup and init eos = IdealSingleGas() vel = np.zeros(shape=(dim, )) orig = np.zeros(shape=(dim, )) vel[:dim] = 1.0 initializer = Vortex2D(center=orig, velocity=vel) gas_model = GasModel(eos=eos) def boundary_solution(discr, btag, gas_model, state_minus, **kwargs): actx = state_minus.array_context bnd_discr = discr.discr_from_dd(btag) nodes = thaw(bnd_discr.nodes(), actx) return make_fluid_state( initializer(x_vec=nodes, eos=gas_model.eos, **kwargs), gas_model) boundaries = { BTAG_ALL: PrescribedFluidBoundary(boundary_state_func=boundary_solution) } if rst_filename: current_t = restart_data["t"] current_step = restart_data["step"] current_cv = restart_data["cv"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_cv = initializer(nodes) current_state = make_fluid_state(current_cv, gas_model) visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ eosname = eos.__class__.__name__ init_message = make_init_message(dim=dim, order=order, nelements=local_nelements, global_nelements=global_nelements, dt=current_dt, t_final=t_final, nstatus=nstatus, nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, initname=initname, eosname=eosname, casename=casename) if rank == 0: logger.info(init_message) def my_write_status(state, component_errors, cfl=None): if cfl is None: if constant_cfl: cfl = current_cfl else: from grudge.op import nodal_max from mirgecom.inviscid import get_inviscid_cfl cfl = actx.to_numpy( nodal_max(discr, "vol", get_inviscid_cfl(discr, state, current_dt)))[()] if rank == 0: logger.info(f"------ {cfl=}\n" "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if exact is None: exact = initializer(x_vec=nodes, eos=eos, time=t) if resid is None: resid = state - exact viz_fields = [("cv", state), ("dv", dv), ("exact", exact), ("residual", resid)] from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True, vis_timer=vis_timer) def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) if rst_fname != rst_filename: rst_data = { "local_mesh": local_mesh, "cv": state, "t": t, "step": step, "order": order, "global_nelements": global_nelements, "num_parts": num_parts } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(pressure, component_errors): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure) \ or check_range_local(discr, "vol", pressure, .2, 1.02): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") exittol = .1 if max(component_errors) > exittol: health_error = True if rank == 0: logger.info("Solution diverged from exact soln.") return health_error def my_pre_step(step, t, dt, state): fluid_state = make_fluid_state(state, gas_model) cv = fluid_state.cv dv = fluid_state.dv try: exact = None component_errors = None if logmgr: logmgr.tick_before() do_viz = check_step(step=step, interval=nviz) do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) do_status = check_step(step=step, interval=nstatus) if do_health: exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, cv, exact) health_errors = global_reduce(my_health_check( dv.pressure, component_errors), op="lor") if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") if do_restart: my_write_restart(step=step, t=t, state=cv) if do_status: if component_errors is None: if exact is None: exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions( discr, cv, exact) my_write_status(fluid_state, component_errors) if do_viz: if exact is None: exact = initializer(x_vec=nodes, eos=eos, time=t) resid = state - exact my_write_viz(step=step, t=t, state=cv, dv=dv, exact=exact, resid=resid) except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=cv) my_write_restart(step=step, t=t, state=cv) raise dt = get_sim_timestep(discr, fluid_state, t, dt, current_cfl, t_final, constant_cfl) return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() return state, dt def my_rhs(t, state): fluid_state = make_fluid_state(state, gas_model) return euler_operator(discr, state=fluid_state, time=t, boundaries=boundaries, gas_model=gas_model) current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, current_cfl, t_final, constant_cfl) current_step, current_t, current_cv = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, state=current_state.cv, t=current_t, t_final=t_final) # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") current_state = make_fluid_state(current_cv, gas_model) final_dv = current_state.dv final_exact = initializer(x_vec=nodes, eos=eos, time=current_t) final_resid = current_state.cv - final_exact my_write_viz(step=current_step, t=current_t, state=current_state.cv, dv=final_dv, exact=final_exact, resid=final_resid) my_write_restart(step=current_step, t=current_t, state=current_state.cv) if logmgr: logmgr.close() elif use_profiling: print(actx.tabulate_profiling_data()) finish_tol = 1e-16 assert np.abs(current_t - t_final) < finish_tol
def main(ctx_factory=cl.create_some_context, casename="flame1d", user_input_file=None, snapshot_pattern="{casename}-{step:06d}-{rank:04d}.pkl", restart_step=None, restart_name=None, use_profiling=False, use_logmgr=False, use_lazy_eval=False): """Drive the 1D Flame example.""" from mpi4py import MPI comm = MPI.COMM_WORLD rank = 0 rank = comm.Get_rank() nparts = comm.Get_size() if restart_name is None: restart_name = casename """logging and profiling""" logmgr = initialize_logmgr(use_logmgr, filename=(f"{casename}.sqlite"), mode="wo", mpi_comm=comm) cl_ctx = ctx_factory() if use_profiling: if use_lazy_eval: raise RuntimeError("Cannot run lazy with profiling.") queue = cl.CommandQueue( cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) actx = PyOpenCLProfilingArrayContext( queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), logmgr=logmgr) else: queue = cl.CommandQueue(cl_ctx) if use_lazy_eval: actx = PytatoArrayContext(queue) else: actx = PyOpenCLArrayContext( queue, allocator=cl_tools.MemoryPool( cl_tools.ImmediateAllocator(queue))) # default input values that will be read from input (if they exist) nviz = 100 nrestart = 100 nhealth = 100 nstatus = 1 current_dt = 1e-9 t_final = 1.e-3 order = 1 integrator = "rk4" if user_input_file: if rank == 0: with open(user_input_file) as f: input_data = yaml.load(f, Loader=yaml.FullLoader) else: input_data = None input_data = comm.bcast(input_data, root=0) #print(input_data) try: nviz = int(input_data["nviz"]) except KeyError: pass try: nrestart = int(input_data["nrestart"]) except KeyError: pass try: nhealth = int(input_data["nhealth"]) except KeyError: pass try: nstatus = int(input_data["nstatus"]) except KeyError: pass try: current_dt = float(input_data["current_dt"]) except KeyError: pass try: t_final = float(input_data["t_final"]) except KeyError: pass try: order = int(input_data["order"]) except KeyError: pass try: integrator = input_data["integrator"] except KeyError: pass # param sanity check allowed_integrators = ["rk4", "euler", "lsrk54", "lsrk144"] if (integrator not in allowed_integrators): error_message = "Invalid time integrator: {}".format(integrator) raise RuntimeError(error_message) timestepper = rk4_step if integrator == "euler": timestepper = euler_step if integrator == "lsrk54": timestepper = lsrk54_step if integrator == "lsrk144": timestepper = lsrk144_step if (rank == 0): print(f'#### Simluation control data: ####') print(f'\tnviz = {nviz}') print(f'\tnrestart = {nrestart}') print(f'\tnhealth = {nhealth}') print(f'\tnstatus = {nstatus}') print(f'\tcurrent_dt = {current_dt}') print(f'\tt_final = {t_final}') print(f'\torder = {order}') print(f"\tTime integration {integrator}") print(f'#### Simluation control data: ####') restart_path = 'restart_data/' viz_path = 'viz_data/' #if(rank == 0): #if not os.path.exists(restart_path): #os.makedirs(restart_path) #if not os.path.exists(viz_path): #os.makedirs(viz_path) dim = 2 exittol = .09 current_cfl = 1.0 current_t = 0 constant_cfl = False checkpoint_t = current_t current_step = 0 vel_burned = np.zeros(shape=(dim, )) vel_unburned = np.zeros(shape=(dim, )) # {{{ Set up initial state using Cantera # Use Cantera for initialization # -- Pick up a CTI for the thermochemistry config # --- Note: Users may add their own CTI file by dropping it into # --- mirgecom/mechanisms alongside the other CTI files. fuel = "H2" allowed_fuels = ["H2", "C2H4"] if (fuel not in allowed_fuels): error_message = "Invalid fuel selection: {}".format(fuel) raise RuntimeError(error_message) if rank == 0: print(f"Fuel: {fuel}") from mirgecom.mechanisms import get_mechanism_cti if fuel == "C2H4": mech_cti = get_mechanism_cti("uiuc") elif fuel == "H2": mech_cti = get_mechanism_cti("sanDiego") cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti) nspecies = cantera_soln.n_species # Initial temperature, pressure, and mixutre mole fractions are needed to # set up the initial state in Cantera. temp_unburned = 300.0 temp_ignition = 1500.0 # Parameters for calculating the amounts of fuel, oxidizer, and inert species if fuel == "C2H4": stoich_ratio = 3.0 if fuel == "H2": stoich_ratio = 0.5 equiv_ratio = 1.0 ox_di_ratio = 0.21 # Grab the array indices for the specific species, ethylene, oxygen, and nitrogen i_fu = cantera_soln.species_index(fuel) i_ox = cantera_soln.species_index("O2") i_di = cantera_soln.species_index("N2") x = np.zeros(nspecies) # Set the species mole fractions according to our desired fuel/air mixture x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio + ox_di_ratio * equiv_ratio) x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio # Uncomment next line to make pylint fail when it can't find cantera.one_atm one_atm = cantera.one_atm # pylint: disable=no-member # one_atm = 101325.0 pres_unburned = one_atm # Let the user know about how Cantera is being initilized print(f"Input state (T,P,X) = ({temp_unburned}, {pres_unburned}, {x}") # Set Cantera internal gas temperature, pressure, and mole fractios cantera_soln.TPX = temp_unburned, pres_unburned, x # Pull temperature, total density, mass fractions, and pressure from Cantera # We need total density, and mass fractions to initialize the fluid/gas state. y_unburned = np.zeros(nspecies) can_t, rho_unburned, y_unburned = cantera_soln.TDY can_p = cantera_soln.P # *can_t*, *can_p* should not differ (significantly) from user's initial data, # but we want to ensure that we use exactly the same starting point as Cantera, # so we use Cantera's version of these data. # now find the conditions for the burned gas cantera_soln.equilibrate('TP') temp_burned, rho_burned, y_burned = cantera_soln.TDY pres_burned = cantera_soln.P pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np) kappa = 1.6e-5 # Pr = mu*rho/alpha = 0.75 mu = 1.e-5 species_diffusivity = 1.e-5 * np.ones(nspecies) transport_model = SimpleTransport(viscosity=mu, thermal_conductivity=kappa, species_diffusivity=species_diffusivity) eos = PyrometheusMixture(pyrometheus_mechanism, temperature_guess=temp_unburned, transport_model=transport_model) species_names = pyrometheus_mechanism.species_names print(f"Pyrometheus mechanism species names {species_names}") print( f"Unburned state (T,P,Y) = ({temp_unburned}, {pres_unburned}, {y_unburned}" ) print(f"Burned state (T,P,Y) = ({temp_burned}, {pres_burned}, {y_burned}") flame_start_loc = 0.10 flame_speed = 1000 # use the burned conditions with a lower temperature #bulk_init = PlanarDiscontinuity(dim=dim, disc_location=flame_start_loc, sigma=0.01, nspecies=nspecies, #temperature_left=temp_ignition, temperature_right=temp_unburned, #pressure_left=pres_burned, pressure_right=pres_unburned, #velocity_left=vel_burned, velocity_right=vel_unburned, #species_mass_left=y_burned, species_mass_right=y_unburned) bulk_init = PlanarDiscontinuity(dim=dim, disc_location=flame_start_loc, sigma=0.0005, nspecies=nspecies, temperature_right=temp_ignition, temperature_left=temp_unburned, pressure_right=pres_burned, pressure_left=pres_unburned, velocity_right=vel_burned, velocity_left=vel_unburned, species_mass_right=y_burned, species_mass_left=y_unburned) inflow_init = MixtureInitializer(dim=dim, nspecies=nspecies, pressure=pres_burned, temperature=temp_ignition, massfractions=y_burned, velocity=vel_burned) outflow_init = MixtureInitializer(dim=dim, nspecies=nspecies, pressure=pres_unburned, temperature=temp_unburned, massfractions=y_unburned, velocity=vel_unburned) def symmetry(nodes, eos, cv=None, **kwargs): dim = len(nodes) if cv is not None: #cv = split_conserved(dim, q) mass = cv.mass momentum = cv.momentum momentum[1] = -1.0 * momentum[1] ke = .5 * np.dot(cv.momentum, cv.momentum) / cv.mass energy = cv.energy species_mass = cv.species_mass return make_conserved(dim=dim, mass=mass, momentum=momentum, energy=energy, species_mass=species_mass) def dummy(nodes, eos, cv=None, **kwargs): dim = len(nodes) if cv is not None: #cv = split_conserved(dim, q) mass = cv.mass momentum = cv.momentum ke = .5 * np.dot(cv.momentum, cv.momentum) / cv.mass energy = cv.energy species_mass = cv.species_mass return make_conserved(dim=dim, mass=mass, momentum=momentum, energy=energy, species_mass=species_mass) inflow = PrescribedViscousBoundary(q_func=inflow_init) outflow = PrescribedViscousBoundary(q_func=outflow_init) wall_symmetry = PrescribedViscousBoundary(q_func=symmetry) wall_dummy = PrescribedViscousBoundary(q_func=dummy) wall = PrescribedViscousBoundary( ) # essentially a "dummy" use the interior solution for the exterior boundaries = { DTAG_BOUNDARY("Inflow"): inflow, DTAG_BOUNDARY("Outflow"): outflow, #DTAG_BOUNDARY("Wall"): wall_dummy} DTAG_BOUNDARY("Wall"): wall_symmetry } if restart_step is None: char_len = 0.0001 box_ll = (0.0, 0.0) box_ur = (0.2, 0.00125) num_elements = (int((box_ur[0] - box_ll[0]) / char_len), int((box_ur[1] - box_ll[1]) / char_len)) from meshmode.mesh.generation import generate_regular_rect_mesh generate_mesh = partial(generate_regular_rect_mesh, a=box_ll, b=box_ur, n=num_elements, boundary_tag_to_face={ "Inflow": ["+x"], "Outflow": ["-x"], "Wall": ["+y", "-y"] }) local_mesh, global_nelements = generate_and_distribute_mesh( comm, generate_mesh) local_nelements = local_mesh.nelements else: # Restart from mirgecom.restart import read_restart_data restart_file = restart_path + snapshot_pattern.format( casename=restart_name, step=restart_step, rank=rank) restart_data = read_restart_data(actx, restart_file) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert comm.Get_size() == restart_data["num_parts"] if rank == 0: logging.info("Making discretization") discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) nodes = thaw(actx, discr.nodes()) if restart_step is None: if rank == 0: logging.info("Initializing soln.") # for Discontinuity initial conditions current_state = bulk_init(x_vec=nodes, eos=eos, time=0.) # for uniform background initial condition #current_state = bulk_init(nodes, eos=eos) else: current_t = restart_data["t"] current_step = restart_step #current_state = make_fluid_restart_state(actx, discr.discr_from_dd("vol"), restart_data["state"]) current_state = restart_data["state"] vis_timer = None log_cfl = LogUserQuantity(name="cfl", value=current_cfl) if logmgr: logmgr_add_cl_device_info(logmgr, queue) logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) logmgr_set_time(logmgr, current_step, current_t) logmgr.add_quantity(log_cfl, interval=nstatus) #logmgr_add_package_versions(logmgr) logmgr.add_watches([ ("step.max", "step = {value}, "), ("t_sim.max", "sim time: {value:1.6e} s, "), ("cfl.max", "cfl = {value:1.4f}\n"), ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), ("max_pressure", "{value:1.9e})\n"), ("min_temperature", "------- T (min, max) (K) = ({value:7g}, "), ("max_temperature", "{value:7g})\n"), ("t_step.max", "------- step walltime: {value:6g} s, "), ("t_log.max", "log walltime: {value:6g} s") ]) try: logmgr.add_watches(["memory_usage.max"]) except KeyError: pass if use_profiling: logmgr.add_watches(["pyopencl_array_time.max"]) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) visualizer = make_visualizer(discr) initname = "flame1d" eosname = eos.__class__.__name__ init_message = make_init_message(dim=dim, order=order, nelements=local_nelements, global_nelements=global_nelements, dt=current_dt, t_final=t_final, nstatus=nstatus, nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, initname=initname, eosname=eosname, casename=casename) if rank == 0: logger.info(init_message) get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, dt=current_dt, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) def my_rhs(t, state): return ( ns_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) + eos.get_species_source_terms(cv=state)) def my_checkpoint(step, t, dt, state, force=False): do_health = force or check_step(step, nhealth) and step > 0 do_viz = force or check_step(step, nviz) do_restart = force or check_step(step, nrestart) do_status = force or check_step(step, nstatus) if do_viz or do_health: dv = eos.dependent_vars(state) errors = False if do_health: health_message = "" if check_naninf_local(discr, "vol", dv.pressure): errors = True health_message += "Invalid pressure data found.\n" elif check_range_local(discr, "vol", dv.pressure, min_value=1, max_value=2.e6): errors = True health_message += "Pressure data failed health check.\n" errors = comm.allreduce(errors, MPI.LOR) if errors: if rank == 0: logger.info("Fluid solution failed health check.") if health_message: logger.info(f"{rank=}: {health_message}") #if check_step(step, nrestart) and step != restart_step and not errors: if do_restart or errors: filename = restart_path + snapshot_pattern.format( step=step, rank=rank, casename=casename) restart_dictionary = { "local_mesh": local_mesh, "order": order, "state": state, "t": t, "step": step, "global_nelements": global_nelements, "num_parts": nparts } write_restart_file(actx, restart_dictionary, filename, comm) if do_status or do_viz or errors: local_cfl = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) max_cfl = nodal_max(discr, "vol", local_cfl) log_cfl.set_quantity(max_cfl) #if ((check_step(step, nviz) and step != restart_step) or errors): if do_viz or errors: def loc_fn(t): return flame_start_loc + flame_speed * t #exact_soln = PlanarDiscontinuity(dim=dim, disc_location=loc_fn, #sigma=0.0000001, nspecies=nspecies, #temperature_left=temp_ignition, temperature_right=temp_unburned, #pressure_left=pres_burned, pressure_right=pres_unburned, #velocity_left=vel_burned, velocity_right=vel_unburned, #species_mass_left=y_burned, species_mass_right=y_unburned) reaction_rates = eos.get_production_rates(cv=state) # conserved quantities viz_fields = [ #("cv", state), ("CV_rho", state.mass), ("CV_rhoU", state.momentum[0]), ("CV_rhoV", state.momentum[1]), ("CV_rhoE", state.energy) ] # species mass fractions viz_fields.extend( ("Y_" + species_names[i], state.species_mass[i] / state.mass) for i in range(nspecies)) # dependent variables viz_fields.extend([ ("DV", eos.dependent_vars(state)), #("exact_soln", exact_soln), ("reaction_rates", reaction_rates), ("cfl", local_cfl) ]) write_visfile(discr, viz_fields, visualizer, vizname=viz_path + casename, step=step, t=t, overwrite=True, vis_timer=vis_timer) if errors: raise RuntimeError("Error detected by user checkpoint, exiting.") if rank == 0: logging.info("Stepping.") (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, checkpoint=my_checkpoint, get_timestep=get_timestep, state=current_state, t_final=t_final, t=current_t, istep=current_step, logmgr=logmgr,eos=eos,dim=dim) if rank == 0: logger.info("Checkpointing final state ...") my_checkpoint(current_step, t=current_t, dt=(current_t - checkpoint_t), state=current_state, force=True) if logmgr: logmgr.close() elif use_profiling: print(actx.tabulate_profiling_data()) exit()
def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_overintegration=False, use_profiling=False, casename=None, rst_filename=None, actx_class=PyOpenCLArrayContext, log_dependent=True): """Drive example.""" cl_ctx = ctx_factory() if casename is None: casename = "mirgecom" from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() nproc = comm.Get_size() from mirgecom.simutil import global_reduce as _global_reduce global_reduce = partial(_global_reduce, comm=comm) logmgr = initialize_logmgr(use_logmgr, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: queue = cl.CommandQueue( cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) actx = actx_class(queue, allocator=cl_tools.MemoryPool( cl_tools.ImmediateAllocator(queue))) # Some discretization parameters dim = 2 nel_1d = 8 order = 1 # {{{ Time stepping control # This example runs only 3 steps by default (to keep CI ~short) # With the mixture defined below, equilibrium is achieved at ~40ms # To run to equilibrium, set t_final >= 40ms. # Time stepper selection if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step # Time loop control parameters current_step = 0 t_final = 1e-8 current_cfl = 1.0 current_dt = 1e-9 current_t = 0 constant_cfl = False # i.o frequencies nstatus = 1 nviz = 5 nhealth = 1 nrestart = 5 # }}} Time stepping control debug = False rst_path = "restart_data/" rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl") if rst_filename: # read the grid from restart data rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["num_parts"] == nproc rst_time = restart_data["t"] rst_step = restart_data["step"] rst_order = restart_data["order"] else: # generate the grid from scratch from meshmode.mesh.generation import generate_regular_rect_mesh box_ll = -0.005 box_ur = 0.005 generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll, ) * dim, b=(box_ur, ) * dim, nelements_per_axis=(nel_1d, ) * dim) local_mesh, global_nelements = generate_and_distribute_mesh( comm, generate_mesh) local_nelements = local_mesh.nelements from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD from meshmode.discretization.poly_element import \ default_simplex_group_factory, QuadratureSimplexGroupFactory discr = EagerDGDiscretization( actx, local_mesh, discr_tag_to_group_factory={ DISCR_TAG_BASE: default_simplex_group_factory(base_dim=local_mesh.dim, order=order), DISCR_TAG_QUAD: QuadratureSimplexGroupFactory(2 * order + 1) }, mpi_communicator=comm) nodes = thaw(discr.nodes(), actx) ones = discr.zeros(actx) + 1.0 if use_overintegration: quadrature_tag = DISCR_TAG_QUAD else: quadrature_tag = None ones = discr.zeros(actx) + 1.0 vis_timer = None if logmgr: logmgr_add_cl_device_info(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) logmgr.add_watches([("step.max", "step = {value}, "), ("t_sim.max", "sim time: {value:1.6e} s\n"), ("t_step.max", "------- step walltime: {value:6g} s, "), ("t_log.max", "log walltime: {value:6g} s")]) if log_dependent: logmgr_add_many_discretization_quantities( logmgr, discr, dim, extract_vars_for_logging, units_for_logging) logmgr.add_watches([ ("min_pressure", "\n------- P (min, max) (Pa) = ({value:1.9e}, "), ("max_pressure", "{value:1.9e})\n"), ("min_temperature", "------- T (min, max) (K) = ({value:7g}, "), ("max_temperature", "{value:7g})\n") ]) # {{{ Set up initial state using Cantera # Use Cantera for initialization # -- Pick up a CTI for the thermochemistry config # --- Note: Users may add their own CTI file by dropping it into # --- mirgecom/mechanisms alongside the other CTI files. from mirgecom.mechanisms import get_mechanism_cti mech_cti = get_mechanism_cti("uiuc") cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti) nspecies = cantera_soln.n_species # Initial temperature, pressure, and mixutre mole fractions are needed to # set up the initial state in Cantera. temperature_seed = 1500.0 # Initial temperature hot enough to burn # Parameters for calculating the amounts of fuel, oxidizer, and inert species equiv_ratio = 1.0 ox_di_ratio = 0.21 stoich_ratio = 3.0 # Grab the array indices for the specific species, ethylene, oxygen, and nitrogen i_fu = cantera_soln.species_index("C2H4") i_ox = cantera_soln.species_index("O2") i_di = cantera_soln.species_index("N2") x = np.zeros(nspecies) # Set the species mole fractions according to our desired fuel/air mixture x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio + ox_di_ratio * equiv_ratio) x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio # Uncomment next line to make pylint fail when it can't find cantera.one_atm one_atm = cantera.one_atm # pylint: disable=no-member # one_atm = 101325.0 # Let the user know about how Cantera is being initilized print(f"Input state (T,P,X) = ({temperature_seed}, {one_atm}, {x}") # Set Cantera internal gas temperature, pressure, and mole fractios cantera_soln.TPX = temperature_seed, one_atm, x # Pull temperature, total density, mass fractions, and pressure from Cantera # We need total density, and mass fractions to initialize the fluid/gas state. can_t, can_rho, can_y = cantera_soln.TDY can_p = cantera_soln.P # *can_t*, *can_p* should not differ (significantly) from user's initial data, # but we want to ensure that we use exactly the same starting point as Cantera, # so we use Cantera's version of these data. # }}} # {{{ Create Pyrometheus thermochemistry object & EOS # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and # generates a set of methods to calculate chemothermomechanical properties and # states for this particular mechanism. from mirgecom.thermochemistry import make_pyrometheus_mechanism_class pyro_mechanism = make_pyrometheus_mechanism_class(cantera_soln)(actx.np) eos = PyrometheusMixture(pyro_mechanism, temperature_guess=temperature_seed) gas_model = GasModel(eos=eos) from pytools.obj_array import make_obj_array def get_temperature_update(cv, temperature): y = cv.species_mass_fractions e = gas_model.eos.internal_energy(cv) / cv.mass return pyro_mechanism.get_temperature_update_energy(e, temperature, y) from mirgecom.gas_model import make_fluid_state def get_fluid_state(cv, tseed): return make_fluid_state(cv=cv, gas_model=gas_model, temperature_seed=tseed) compute_temperature_update = actx.compile(get_temperature_update) construct_fluid_state = actx.compile(get_fluid_state) # }}} # {{{ MIRGE-Com state initialization # Initialize the fluid/gas state with Cantera-consistent data: # (density, pressure, temperature, mass_fractions) print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}") velocity = np.zeros(shape=(dim, )) initializer = MixtureInitializer(dim=dim, nspecies=nspecies, pressure=can_p, temperature=can_t, massfractions=can_y, velocity=velocity) my_boundary = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: my_boundary} if rst_filename: current_step = rst_step current_t = rst_time if logmgr: from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) if order == rst_order: current_cv = restart_data["cv"] temperature_seed = restart_data["temperature_seed"] else: rst_cv = restart_data["cv"] old_discr = EagerDGDiscretization(actx, local_mesh, order=rst_order, mpi_communicator=comm) from meshmode.discretization.connection import make_same_mesh_connection connection = make_same_mesh_connection( actx, discr.discr_from_dd("vol"), old_discr.discr_from_dd("vol")) current_cv = connection(rst_cv) temperature_seed = connection(restart_data["temperature_seed"]) else: # Set the current state from time 0 current_cv = initializer(eos=gas_model.eos, x_vec=nodes) temperature_seed = temperature_seed * ones # The temperature_seed going into this function is: # - At time 0: the initial temperature input data (maybe from Cantera) # - On restart: the restarted temperature seed from restart file (saving # the *seed* allows restarts to be deterministic current_fluid_state = construct_fluid_state(current_cv, temperature_seed) current_dv = current_fluid_state.dv temperature_seed = current_dv.temperature # Inspection at physics debugging time if debug: print("Initial MIRGE-Com state:") print(f"Initial DV pressure: {current_fluid_state.pressure}") print(f"Initial DV temperature: {current_fluid_state.temperature}") # }}} visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ eosname = gas_model.eos.__class__.__name__ init_message = make_init_message(dim=dim, order=order, nelements=local_nelements, global_nelements=global_nelements, dt=current_dt, t_final=t_final, nstatus=nstatus, nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, initname=initname, eosname=eosname, casename=casename) # Cantera equilibrate calculates the expected end state @ chemical equilibrium # i.e. the expected state after all reactions cantera_soln.equilibrate("UV") eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY eq_pressure = cantera_soln.P # Report the expected final state to the user if rank == 0: logger.info(init_message) logger.info(f"Expected equilibrium state:" f" {eq_pressure=}, {eq_temperature=}," f" {eq_density=}, {eq_mass_fractions=}") def my_write_status(dt, cfl, dv=None): status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}" if ((dv is not None) and (not log_dependent)): temp = dv.temperature press = dv.pressure from grudge.op import nodal_min_loc, nodal_max_loc tmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", temp)), comm=comm, op=MPI.MIN) tmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", temp)), comm=comm, op=MPI.MAX) pmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", press)), comm=comm, op=MPI.MIN) pmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", press)), comm=comm, op=MPI.MAX) dv_status_msg = f"\nP({pmin}, {pmax}), T({tmin}, {tmax})" status_msg = status_msg + dv_status_msg if rank == 0: logger.info(status_msg) def my_write_viz(step, t, dt, state, ts_field, dv, production_rates, cfl): viz_fields = [("cv", state), ("dv", dv), ("production_rates", production_rates), ("dt" if constant_cfl else "cfl", ts_field)] write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True, vis_timer=vis_timer) def my_write_restart(step, t, state, temperature_seed): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) if rst_fname == rst_filename: if rank == 0: logger.info("Skipping overwrite of restart file.") else: rst_data = { "local_mesh": local_mesh, "cv": state.cv, "temperature_seed": temperature_seed, "t": t, "step": step, "order": order, "global_nelements": global_nelements, "num_parts": nproc } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(cv, dv): import grudge.op as op health_error = False pressure = dv.pressure temperature = dv.temperature from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") if check_range_local(discr, "vol", pressure, 1e5, 2.6e5): health_error = True logger.info(f"{rank=}: Pressure range violation.") if check_naninf_local(discr, "vol", temperature): health_error = True logger.info(f"{rank=}: Invalid temperature data found.") if check_range_local(discr, "vol", temperature, 1.498e3, 1.6e3): health_error = True logger.info(f"{rank=}: Temperature range violation.") # This check is the temperature convergence check # The current *temperature* is what Pyrometheus gets # after a fixed number of Newton iterations, *n_iter*. # Calling `compute_temperature` here with *temperature* # input as the guess returns the calculated gas temperature after # yet another *n_iter*. # The difference between those two temperatures is the # temperature residual, which can be used as an indicator of # convergence in Pyrometheus `get_temperature`. # Note: The local max jig below works around a very long compile # in lazy mode. temp_resid = compute_temperature_update(cv, temperature) / temperature temp_err = (actx.to_numpy(op.nodal_max_loc(discr, "vol", temp_resid))) if temp_err > 1e-8: health_error = True logger.info( f"{rank=}: Temperature is not converged {temp_resid=}.") return health_error from mirgecom.inviscid import get_inviscid_timestep def get_dt(state): return get_inviscid_timestep(discr, state=state) compute_dt = actx.compile(get_dt) from mirgecom.inviscid import get_inviscid_cfl def get_cfl(state, dt): return get_inviscid_cfl(discr, dt=dt, state=state) compute_cfl = actx.compile(get_cfl) def get_production_rates(cv, temperature): return eos.get_production_rates(cv, temperature) compute_production_rates = actx.compile(get_production_rates) def my_get_timestep(t, dt, state): # richer interface to calculate {dt,cfl} returns node-local estimates t_remaining = max(0, t_final - t) if constant_cfl: ts_field = current_cfl * compute_dt(state) from grudge.op import nodal_min_loc dt = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", ts_field)), comm=comm, op=MPI.MIN) cfl = current_cfl else: ts_field = compute_cfl(state, current_dt) from grudge.op import nodal_max_loc cfl = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", ts_field)), comm=comm, op=MPI.MAX) return ts_field, cfl, min(t_remaining, dt) def my_pre_step(step, t, dt, state): cv, tseed = state fluid_state = construct_fluid_state(cv, tseed) dv = fluid_state.dv try: if logmgr: logmgr.tick_before() from mirgecom.simutil import check_step do_viz = check_step(step=step, interval=nviz) do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) do_status = check_step(step=step, interval=nstatus) if do_health: health_errors = global_reduce(my_health_check(cv, dv), op="lor") if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=fluid_state) if do_status: my_write_status(dt=dt, cfl=cfl, dv=dv) if do_restart: my_write_restart(step=step, t=t, state=fluid_state, temperature_seed=tseed) if do_viz: production_rates = compute_production_rates( fluid_state.cv, fluid_state.temperature) my_write_viz(step=step, t=t, dt=dt, state=cv, dv=dv, production_rates=production_rates, ts_field=ts_field, cfl=cfl) except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") # my_write_viz(step=step, t=t, dt=dt, state=cv) # my_write_restart(step=step, t=t, state=fluid_state) raise return state, dt def my_post_step(step, t, dt, state): cv, tseed = state fluid_state = construct_fluid_state(cv, tseed) # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, cv, gas_model.eos) logmgr.tick_after() return make_obj_array([cv, fluid_state.temperature]), dt def my_rhs(t, state): cv, tseed = state from mirgecom.gas_model import make_fluid_state fluid_state = make_fluid_state(cv=cv, gas_model=gas_model, temperature_seed=tseed) return make_obj_array([ euler_operator(discr, state=fluid_state, time=t, boundaries=boundaries, gas_model=gas_model, quadrature_tag=quadrature_tag) + eos.get_species_source_terms(cv, fluid_state.temperature), 0 * tseed ]) current_dt = get_sim_timestep(discr, current_fluid_state, current_t, current_dt, current_cfl, t_final, constant_cfl) current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, state=make_obj_array([current_cv, temperature_seed]), t=current_t, t_final=t_final) # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") final_cv, tseed = current_state final_fluid_state = construct_fluid_state(final_cv, tseed) final_dv = final_fluid_state.dv final_dm = compute_production_rates(final_cv, final_dv.temperature) ts_field, cfl, dt = my_get_timestep(t=current_t, dt=current_dt, state=final_fluid_state) my_write_viz(step=current_step, t=current_t, dt=dt, state=final_cv, dv=final_dv, production_rates=final_dm, ts_field=ts_field, cfl=cfl) my_write_status(dt=dt, cfl=cfl, dv=final_dv) my_write_restart(step=current_step, t=current_t, state=final_fluid_state, temperature_seed=tseed) if logmgr: logmgr.close() elif use_profiling: print(actx.tabulate_profiling_data()) finish_tol = 1e-16 assert np.abs(current_t - t_final) < finish_tol
def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename=None, rst_filename=None, actx_class=PyOpenCLArrayContext, log_dependent=True): """Drive example.""" cl_ctx = ctx_factory() if casename is None: casename = "mirgecom" from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() nparts = comm.Get_size() from mirgecom.simutil import global_reduce as _global_reduce global_reduce = partial(_global_reduce, comm=comm) logmgr = initialize_logmgr(use_logmgr, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: queue = cl.CommandQueue( cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) actx = actx_class( queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # timestepping control if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step t_final = 1e-8 current_cfl = 1.0 current_dt = 1e-9 current_t = 0 current_step = 0 constant_cfl = False # some i/o frequencies nstatus = 1 nhealth = 1 nrestart = 5 nviz = 1 dim = 2 rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) if rst_filename: # read the grid from restart data rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["num_parts"] == nparts else: # generate the grid from scratch nel_1d = 16 box_ll = -5.0 box_ur = 5.0 from meshmode.mesh.generation import generate_regular_rect_mesh generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) local_nelements = local_mesh.nelements order = 3 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) nodes = thaw(discr.nodes(), actx) vis_timer = None if logmgr: logmgr_add_device_name(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) logmgr.add_watches([ ("step.max", "step = {value}, "), ("t_sim.max", "sim time: {value:1.6e} s\n"), ("t_step.max", "------- step walltime: {value:6g} s, "), ("t_log.max", "log walltime: {value:6g} s") ]) if log_dependent: logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) logmgr.add_watches([ ("min_pressure", "\n------- P (min, max) (Pa) = ({value:1.9e}, "), ("max_pressure", "{value:1.9e})\n"), ("min_temperature", "------- T (min, max) (K) = ({value:7g}, "), ("max_temperature", "{value:7g})\n")]) # Pyrometheus initialization from mirgecom.mechanisms import get_mechanism_cti mech_cti = get_mechanism_cti("uiuc") sol = cantera.Solution(phase_id="gas", source=mech_cti) from mirgecom.thermochemistry import make_pyrometheus_mechanism_class pyrometheus_mechanism = make_pyrometheus_mechanism_class(sol)(actx.np) nspecies = pyrometheus_mechanism.num_species eos = PyrometheusMixture(pyrometheus_mechanism) from mirgecom.gas_model import GasModel, make_fluid_state gas_model = GasModel(eos=eos) from pytools.obj_array import make_obj_array y0s = np.zeros(shape=(nspecies,)) for i in range(nspecies-1): y0s[i] = 1.0 / (10.0 ** (i + 1)) spec_sum = sum([y0s[i] for i in range(nspecies-1)]) y0s[nspecies-1] = 1.0 - spec_sum # Mixture defaults to STP (p, T) = (1atm, 300K) velocity = np.zeros(shape=(dim,)) + 1.0 initializer = MixtureInitializer(dim=dim, nspecies=nspecies, massfractions=y0s, velocity=velocity) def boundary_solution(discr, btag, gas_model, state_minus, **kwargs): actx = state_minus.array_context bnd_discr = discr.discr_from_dd(btag) nodes = thaw(bnd_discr.nodes(), actx) return make_fluid_state(initializer(x_vec=nodes, eos=gas_model.eos, **kwargs), gas_model, temperature_seed=state_minus.temperature) boundaries = { BTAG_ALL: PrescribedFluidBoundary(boundary_state_func=boundary_solution) } if rst_filename: current_t = restart_data["t"] current_step = restart_data["step"] current_cv = restart_data["cv"] tseed = restart_data["temperature_seed"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_cv = initializer(x_vec=nodes, eos=eos) tseed = 300.0 current_state = make_fluid_state(current_cv, gas_model, temperature_seed=tseed) visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ eosname = eos.__class__.__name__ init_message = make_init_message(dim=dim, order=order, nelements=local_nelements, global_nelements=global_nelements, dt=current_dt, t_final=t_final, nstatus=nstatus, nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, initname=initname, eosname=eosname, casename=casename) if rank == 0: logger.info(init_message) def my_write_status(component_errors, dv=None): from mirgecom.simutil import allsync status_msg = ( "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) if ((dv is not None) and (not log_dependent)): temp = dv.temperature press = dv.pressure from grudge.op import nodal_min_loc, nodal_max_loc tmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", temp)), comm=comm, op=MPI.MIN) tmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", temp)), comm=comm, op=MPI.MAX) pmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", press)), comm=comm, op=MPI.MIN) pmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", press)), comm=comm, op=MPI.MAX) dv_status_msg = f"\nP({pmin}, {pmax}), T({tmin}, {tmax})" status_msg = status_msg + dv_status_msg if rank == 0: logger.info(status_msg) if rank == 0: logger.info(status_msg) def my_write_viz(step, t, state, dv, exact=None, resid=None): if exact is None: exact = initializer(x_vec=nodes, eos=eos, time=t) if resid is None: resid = state - exact viz_fields = [("cv", state), ("dv", dv)] from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True, vis_timer=vis_timer) def my_write_restart(step, t, state, tseed): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) if rst_fname != rst_filename: rst_data = { "local_mesh": local_mesh, "cv": state, "temperature_seed": tseed, "t": t, "step": step, "order": order, "global_nelements": global_nelements, "num_parts": nparts } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(dv, component_errors): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ or check_range_local(discr, "vol", dv.pressure, 1e5, 1.1e5): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") exittol = .09 if max(component_errors) > exittol: health_error = True if rank == 0: logger.info("Solution diverged from exact soln.") return health_error def my_pre_step(step, t, dt, state): cv, tseed = state fluid_state = make_fluid_state(cv, gas_model, temperature_seed=tseed) dv = fluid_state.dv try: exact = None component_errors = None if logmgr: logmgr.tick_before() from mirgecom.simutil import check_step do_viz = check_step(step=step, interval=nviz) do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) do_status = check_step(step=step, interval=nstatus) if do_health: exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, cv, exact) health_errors = global_reduce( my_health_check(dv, component_errors), op="lor") if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") if do_restart: my_write_restart(step=step, t=t, state=cv, tseed=tseed) if do_viz: if exact is None: exact = initializer(x_vec=nodes, eos=eos, time=t) resid = state - exact my_write_viz(step=step, t=t, state=cv, dv=dv, exact=exact, resid=resid) if do_status: if component_errors is None: if exact is None: exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, cv, exact) my_write_status(component_errors, dv=dv) except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=cv, dv=dv) my_write_restart(step=step, t=t, state=cv, tseed=tseed) raise dt = get_sim_timestep(discr, fluid_state, t, dt, current_cfl, t_final, constant_cfl) return state, dt def my_post_step(step, t, dt, state): cv, tseed = state fluid_state = make_fluid_state(cv, gas_model, temperature_seed=tseed) tseed = fluid_state.temperature # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, cv, eos) logmgr.tick_after() return make_obj_array([fluid_state.cv, tseed]), dt def my_rhs(t, state): cv, tseed = state fluid_state = make_fluid_state(cv, gas_model, temperature_seed=tseed) return make_obj_array( [euler_operator(discr, state=fluid_state, time=t, boundaries=boundaries, gas_model=gas_model), 0*tseed]) current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, current_cfl, t_final, constant_cfl) current_step, current_t, advanced_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, state=make_obj_array([current_state.cv, current_state.temperature]), t=current_t, t_final=t_final, eos=eos, dim=dim) # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") current_cv, tseed = advanced_state current_state = make_fluid_state(current_cv, gas_model, temperature_seed=tseed) final_dv = current_state.dv final_exact = initializer(x_vec=nodes, eos=eos, time=current_t) final_resid = current_state.cv - final_exact my_write_viz(step=current_step, t=current_t, state=current_cv, dv=final_dv, exact=final_exact, resid=final_resid) my_write_restart(step=current_step, t=current_t, state=current_state.cv, tseed=tseed) if logmgr: logmgr.close() elif use_profiling: print(actx.tabulate_profiling_data()) finish_tol = 1e-16 assert np.abs(current_t - t_final) < finish_tol
def main(ctx_factory=cl.create_some_context, snapshot_pattern="y0euler-{step:06d}-{rank:04d}.pkl", restart_step=None, use_profiling=False, use_logmgr=False): """Drive the Y0 example.""" from mpi4py import MPI comm = MPI.COMM_WORLD rank = 0 rank = comm.Get_rank() nparts = comm.Get_size() """logging and profiling""" logmgr = initialize_logmgr(use_logmgr, filename="y0euler.sqlite", mode="wo", mpi_comm=comm) cl_ctx = ctx_factory() if use_profiling: queue = cl.CommandQueue( cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) actx = PyOpenCLProfilingArrayContext( queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), logmgr=logmgr) else: queue = cl.CommandQueue(cl_ctx) actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool( cl_tools.ImmediateAllocator(queue))) nviz = 500 nrestart = 500 #nviz = 10 #nrestart = 10 #current_dt = 2.5e-8 # stable with euler, first order current_dt = 2.5e-8 # stable with euler, first order #current_dt = 4e-7 # stable with lrsrk144 #t_final = 5.e-1 t_final = 2.5e-7 dim = 3 order = 4 exittol = .09 #t_final = 0.001 current_cfl = 1.0 vel_init = np.zeros(shape=(dim, )) vel_inflow = np.zeros(shape=(dim, )) vel_outflow = np.zeros(shape=(dim, )) orig = np.zeros(shape=(dim, )) orig[0] = 0.83 orig[2] = 0.001 #vel[0] = 340.0 #vel_inflow[0] = 100.0 # m/s current_t = 0 casename = "y0euler" constant_cfl = False nstatus = 10000000000 checkpoint_t = current_t current_step = 0 # working gas: CO2 # # gamma = 1.289 # MW=44.009 g/mol # cp = 37.135 J/mol-K, # rho= 1.977 kg/m^3 @298K gamma_CO2 = 1.289 R_CO2 = 8314.59 / 44.009 # background # 100 Pa # 298 K # rho = 1.77619667e-3 kg/m^3 # velocity = 0,0,0 rho_bkrnd = 1.77619667e-3 pres_bkrnd = 100 temp_bkrnd = 298 # nozzle inflow # # # stagnation tempertuare 298 K # stagnation pressure 1.5e Pa # # isentropic expansion based on the area ratios between the inlet (r=13e-3m) and the throat (r=6.3e-3) # # MJA, this is calculated offline, add some code to do it for us # # Mach number=0.139145 # pressure=148142 # temperature=297.169 # density=2.63872 # gamma=1.289 # calculate the inlet Mach number from the area ratio nozzleInletRadius = 13.e-3 nozzleThroatRadius = 6.3e-3 nozzleInletArea = math.pi * nozzleInletRadius * nozzleInletRadius nozzleThroatArea = math.pi * nozzleThroatRadius * nozzleThroatRadius inletAreaRatio = nozzleInletArea / nozzleThroatArea def getMachFromAreaRatio(area_ratio, gamma, mach_guess=0.01): error = 1.e-8 nextError = 1.e8 g = gamma M0 = mach_guess while nextError > error: R = ((2 / (g + 1) + ((g - 1) / (g + 1) * M0 * M0))**(((g + 1) / (2 * g - 2)))) / M0 - area_ratio dRdM = (2 * ((2 / (g + 1) + ((g - 1) / (g + 1) * M0 * M0))**(((g + 1) / (2 * g - 2)))) / (2 * g - 2) * (g - 1) / (2 / (g + 1) + ((g - 1) / (g + 1) * M0 * M0)) - ((2 / (g + 1) + ((g - 1) / (g + 1) * M0 * M0))**(((g + 1) / (2 * g - 2)))) * M0**(-2)) M1 = M0 - R / dRdM nextError = abs(R) M0 = M1 return M1 def getIsentropicPressure(mach, P0, gamma): pressure = (1. + (gamma - 1.) * 0.5 * math.pow(mach, 2)) pressure = P0 * math.pow(pressure, (-gamma / (gamma - 1.))) return pressure def getIsentropicTemperature(mach, T0, gamma): temperature = (1. + (gamma - 1.) * 0.5 * math.pow(mach, 2)) temperature = T0 * math.pow(temperature, -1.0) return temperature inlet_mach = getMachFromAreaRatio(area_ratio=inletAreaRatio, gamma=gamma_CO2, mach_guess=0.01) # ramp the stagnation pressure start_ramp_pres = 1000 ramp_interval = 5.e-3 t_ramp_start = 1e-5 pres_inflow = getIsentropicPressure(mach=inlet_mach, P0=start_ramp_pres, gamma=gamma_CO2) temp_inflow = getIsentropicTemperature(mach=inlet_mach, T0=298, gamma=gamma_CO2) rho_inflow = pres_inflow / temp_inflow / R_CO2 if rank == 0: print(f'inlet Mach number {inlet_mach}') print(f'inlet temperature {temp_inflow}') print(f'inlet pressure {pres_inflow}') end_ramp_pres = 150000 pres_inflow_final = getIsentropicPressure(mach=inlet_mach, P0=end_ramp_pres, gamma=gamma_CO2) if rank == 0: print(f'final inlet pressure {pres_inflow_final}') #pres_inflow=148142 #temp_inflow=297.169 #rho_inflow=2.63872 #mach_inflow=infloM = 0.139145 vel_inflow[0] = inlet_mach * math.sqrt( gamma_CO2 * pres_inflow / rho_inflow) # starting pressure for the inflow ramp #timestepper = rk4_step #timestepper = lsrk54_step #timestepper = lsrk144_step timestepper = euler_step eos = IdealSingleGas(gamma=gamma_CO2, gas_const=R_CO2) bulk_init = Discontinuity( dim=dim, x0=-.30, sigma=0.005, #bulk_init = Discontinuity(dim=dim, x0=-.31,sigma=0.04, rhol=rho_inflow, rhor=rho_bkrnd, pl=pres_inflow, pr=pres_bkrnd, ul=vel_inflow, ur=vel_outflow) #inflow_init = Lump(dim=dim, rho0=rho_inflow, p0=pres_inflow, #center=orig, velocity=vel_inflow, rhoamp=0.0) #outflow_init = Lump(dim=dim, rho0=rho_bkrnd, p0=pres_bkrnd, #center=orig, velocity=vel_outflow, rhoamp=0.0) # pressure ramp function def inflow_ramp_pressure(t, startP=start_ramp_pres, finalP=end_ramp_pres, ramp_interval=ramp_interval, t_ramp_start=t_ramp_start): if t > t_ramp_start: rampPressure = min( finalP, startP + (t - t_ramp_start) / ramp_interval * (finalP - startP)) else: rampPressure = startP return rampPressure class IsentropicInflow: def __init__(self, *, dim=1, direc=0, T0=298, P0=1e5, mach=0.01, p_fun=None): self._P0 = P0 self._T0 = T0 self._dim = dim self._direc = direc self._mach = mach if p_fun is not None: self._p_fun = p_fun def __call__(self, x_vec, *, t=0, eos): if self._p_fun is not None: P0 = self._p_fun(t) else: P0 = self._P0 T0 = self._T0 gamma = eos.gamma() gas_const = eos.gas_const() pressure = getIsentropicPressure(mach=self._mach, P0=P0, gamma=gamma) temperature = getIsentropicTemperature(mach=self._mach, T0=T0, gamma=gamma) rho = pressure / temperature / gas_const #print(f'ramp Mach number {self._mach}') #print(f'ramp stagnation pressure {P0}') #print(f'ramp stagnation temperature {T0}') #print(f'ramp pressure {pressure}') #print(f'ramp temperature {temperature}') velocity = np.zeros(shape=(self._dim, )) velocity[self._direc] = self._mach * math.sqrt( gamma * pressure / rho) mass = 0.0 * x_vec[0] + rho mom = velocity * mass energy = (pressure / (gamma - 1.0)) + np.dot(mom, mom) / (2.0 * mass) from mirgecom.euler import join_conserved return join_conserved(dim=self._dim, mass=mass, momentum=mom, energy=energy) inflow_init = IsentropicInflow(dim=dim, T0=298, P0=start_ramp_pres, mach=inlet_mach, p_fun=inflow_ramp_pressure) outflow_init = Uniform(dim=dim, rho=rho_bkrnd, p=pres_bkrnd, velocity=vel_outflow) inflow = PrescribedBoundary(inflow_init) outflow = PrescribedBoundary(outflow_init) wall = AdiabaticSlipBoundary() dummy = DummyBoundary() alpha_sc = 0.5 # s0 is ~p^-4 #s0_sc = -11.0 s0_sc = -5.0 # kappa is empirical ... kappa_sc = 0.5 if rank == 0: print( f"Shock capturing parameters: alpha {alpha_sc}, s0 {s0_sc}, kappa {kappa_sc}" ) # timestep estimate #wave_speed = max(mach2*c_bkrnd,c_shkd+velocity2[0]) #char_len = 0.001 #area=char_len*char_len/2 #perimeter = 2*char_len+math.sqrt(2*char_len*char_len) #h = 2*area/perimeter #dt_est = 1/(wave_speed*order*order/h) #print(f"Time step estimate {dt_est}\n") # #dt_est_visc = 1/(wave_speed*order*order/h+alpha_sc*order*order*order*order/h/h) #print(f"Viscous timestep estimate {dt_est_visc}\n") from grudge import sym boundaries = { sym.DTAG_BOUNDARY("Inflow"): inflow, sym.DTAG_BOUNDARY("Outflow"): outflow, sym.DTAG_BOUNDARY("Wall"): wall } if restart_step is None: local_mesh, global_nelements = create_parallel_grid( comm, get_pseudo_y0_mesh) local_nelements = local_mesh.nelements else: # Restart with open(snapshot_pattern.format(step=restart_step, rank=rank), "rb") as f: restart_data = pickle.load(f) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert comm.Get_size() == restart_data["num_parts"] if rank == 0: logging.info("Making discretization") discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) nodes = thaw(actx, discr.nodes()) # fixes boudaries in parallel local_boundaries = {} for btag in boundaries: bnd_discr = discr.discr_from_dd(btag) bnd_nodes = thaw(actx, bnd_discr.nodes()) if bnd_nodes[0][0].shape[0] > 0: local_boundaries[btag] = boundaries[btag] boundaries = local_boundaries # initialize the sponge field def gen_sponge(): thickness = 0.15 amplitude = 1. / current_dt / 1000. x0 = 0.05 return (amplitude * actx.np.where( nodes[0] > x0, zeros + ((nodes[0] - x0) / thickness) * ((nodes[0] - x0) / thickness), zeros + 0.0)) zeros = 0 * nodes[0] sponge_sigma = gen_sponge() ref_state = bulk_init(0, nodes, eos=eos) if restart_step is None: if rank == 0: logging.info("Initializing soln.") # for Discontinuity initial conditions current_state = bulk_init(0, nodes, eos=eos) # for uniform background initial condition #current_state = bulk_init(nodes, eos=eos) else: current_t = restart_data["t"] current_step = restart_step current_state = unflatten( actx, discr.discr_from_dd("vol"), obj_array_vectorize(actx.from_numpy, restart_data["state"])) vis_timer = None if logmgr: logmgr_add_device_name(logmgr, queue) logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) logmgr_set_time(logmgr, current_step, current_t) #logmgr_add_package_versions(logmgr) logmgr.add_watches([ "step.max", "t_sim.max", "t_step.max", "t_log.max", "min_pressure", "max_pressure", "min_temperature", "max_temperature" ]) try: logmgr.add_watches(["memory_usage.max"]) except KeyError: pass if use_profiling: logmgr.add_watches(["pyopencl_array_time.max"]) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) visualizer = make_visualizer( discr, discr.order + 3 if discr.dim == 2 else discr.order) # initname = initializer.__class__.__name__ initname = "pseudoY0" eosname = eos.__class__.__name__ init_message = make_init_message(dim=dim, order=order, nelements=local_nelements, global_nelements=global_nelements, dt=current_dt, t_final=t_final, nstatus=nstatus, nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, initname=initname, eosname=eosname, casename=casename) if rank == 0: logger.info(init_message) get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, dt=current_dt, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) def sponge(q, q_ref, sigma): return (sigma * (q_ref - q)) def my_rhs(t, state): # check for some troublesome output types inf_exists = not np.isfinite(discr.norm(state, np.inf)) if inf_exists: if rank == 0: logging.info( "Non-finite values detected in simulation, exiting...") # dump right now viz_fields = [("sponge_sigma", gen_sponge())] sim_checkpoint(discr=discr, visualizer=visualizer, eos=eos, q=state, vizname=casename, step=999999999, t=t, dt=current_dt, nviz=1, exittol=exittol, constant_cfl=constant_cfl, comm=comm, vis_timer=vis_timer, overwrite=True, s0=s0_sc, kappa=kappa_sc, viz_fields=viz_fields) exit() return (inviscid_operator( discr, q=state, t=t, boundaries=boundaries, eos=eos) + artificial_viscosity(discr, t=t, r=state, eos=eos, boundaries=boundaries, alpha=alpha_sc, s0=s0_sc, kappa=kappa_sc) + sponge(q=state, q_ref=ref_state, sigma=sponge_sigma)) def my_checkpoint(step, t, dt, state): write_restart = (check_step(step, nrestart) if step != restart_step else False) if write_restart is True: with open(snapshot_pattern.format(step=step, rank=rank), "wb") as f: pickle.dump( { "local_mesh": local_mesh, "state": obj_array_vectorize(actx.to_numpy, flatten(state)), "t": t, "step": step, "global_nelements": global_nelements, "num_parts": nparts, }, f) viz_fields = [("sponge_sigma", gen_sponge())] return sim_checkpoint(discr=discr, visualizer=visualizer, eos=eos, q=state, vizname=casename, step=step, t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm, vis_timer=vis_timer, overwrite=True, s0=s0_sc, kappa=kappa_sc, viz_fields=viz_fields) def my_checkpoint_rn(step, t, dt, state): viz_fields = [("sponge_sigma", gen_sponge())] return sim_checkpoint(discr=discr, visualizer=visualizer, eos=eos, q=state, vizname=casename, step=step, t=t, dt=dt, nviz=1, exittol=exittol, constant_cfl=constant_cfl, comm=comm, vis_timer=vis_timer, overwrite=True, s0=s0_sc, kappa=kappa_sc, viz_fields=viz_fields) if rank == 0: logging.info("Stepping.") (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, checkpoint=my_checkpoint, get_timestep=get_timestep, state=current_state, t_final=t_final, t=current_t, istep=current_step, logmgr=logmgr,eos=eos,dim=dim) # final dump and finish if rank == 0: logger.info("Checkpointing final state ...") my_checkpoint(current_step, t=current_t, dt=(current_t - checkpoint_t), state=current_state) if current_t - t_final < 0: #raise ValueError("Simulation exited abnormally") logger.info("Simulation exited abnormally") if logmgr: logmgr.close() elif use_profiling: print(actx.tabulate_profiling_data()) exit()
def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename=None, rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive example.""" cl_ctx = ctx_factory() if casename is None: casename = "mirgecom" from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() nproc = comm.Get_size() logmgr = initialize_logmgr(use_logmgr, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: queue = cl.CommandQueue( cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) actx = actx_class(queue, allocator=cl_tools.MemoryPool( cl_tools.ImmediateAllocator(queue))) # Some discretization parameters dim = 2 nel_1d = 8 order = 1 # {{{ Time stepping control # This example runs only 3 steps by default (to keep CI ~short) # With the mixture defined below, equilibrium is achieved at ~40ms # To run to equlibrium, set t_final >= 40ms. # Time stepper selection if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step # Time loop control parameters current_step = 0 t_final = 1e-8 current_cfl = 1.0 current_dt = 1e-9 current_t = 0 constant_cfl = False # i.o frequencies nstatus = 1 nviz = 5 nhealth = 1 nrestart = 5 # }}} Time stepping control debug = False rst_path = "restart_data/" rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl") if rst_filename: # read the grid from restart data rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["num_parts"] == nproc rst_time = restart_data["t"] rst_step = restart_data["step"] rst_order = restart_data["order"] else: # generate the grid from scratch from meshmode.mesh.generation import generate_regular_rect_mesh box_ll = -0.005 box_ur = 0.005 generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll, ) * dim, b=(box_ur, ) * dim, nelements_per_axis=(nel_1d, ) * dim) local_mesh, global_nelements = generate_and_distribute_mesh( comm, generate_mesh) local_nelements = local_mesh.nelements discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) nodes = thaw(actx, discr.nodes()) vis_timer = None if logmgr: logmgr_add_device_name(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) logmgr.add_watches([ ("step.max", "step = {value}, "), ("t_sim.max", "sim time: {value:1.6e} s\n"), ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), ("max_pressure", "{value:1.9e})\n"), ("min_temperature", "------- T (min, max) (K) = ({value:7g}, "), ("max_temperature", "{value:7g})\n"), ("t_step.max", "------- step walltime: {value:6g} s, "), ("t_log.max", "log walltime: {value:6g} s") ]) # {{{ Set up initial state using Cantera # Use Cantera for initialization # -- Pick up a CTI for the thermochemistry config # --- Note: Users may add their own CTI file by dropping it into # --- mirgecom/mechanisms alongside the other CTI files. from mirgecom.mechanisms import get_mechanism_cti mech_cti = get_mechanism_cti("uiuc") cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti) nspecies = cantera_soln.n_species # Initial temperature, pressure, and mixutre mole fractions are needed to # set up the initial state in Cantera. init_temperature = 1500.0 # Initial temperature hot enough to burn # Parameters for calculating the amounts of fuel, oxidizer, and inert species equiv_ratio = 1.0 ox_di_ratio = 0.21 stoich_ratio = 3.0 # Grab the array indices for the specific species, ethylene, oxygen, and nitrogen i_fu = cantera_soln.species_index("C2H4") i_ox = cantera_soln.species_index("O2") i_di = cantera_soln.species_index("N2") x = np.zeros(nspecies) # Set the species mole fractions according to our desired fuel/air mixture x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio + ox_di_ratio * equiv_ratio) x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio # Uncomment next line to make pylint fail when it can't find cantera.one_atm one_atm = cantera.one_atm # pylint: disable=no-member # one_atm = 101325.0 # Let the user know about how Cantera is being initilized print(f"Input state (T,P,X) = ({init_temperature}, {one_atm}, {x}") # Set Cantera internal gas temperature, pressure, and mole fractios cantera_soln.TPX = init_temperature, one_atm, x # Pull temperature, total density, mass fractions, and pressure from Cantera # We need total density, and mass fractions to initialize the fluid/gas state. can_t, can_rho, can_y = cantera_soln.TDY can_p = cantera_soln.P # *can_t*, *can_p* should not differ (significantly) from user's initial data, # but we want to ensure that we use exactly the same starting point as Cantera, # so we use Cantera's version of these data. # }}} # {{{ Create Pyrometheus thermochemistry object & EOS # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and # generates a set of methods to calculate chemothermomechanical properties and # states for this particular mechanism. pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np) eos = PyrometheusMixture(pyrometheus_mechanism, temperature_guess=init_temperature) # }}} # {{{ MIRGE-Com state initialization # Initialize the fluid/gas state with Cantera-consistent data: # (density, pressure, temperature, mass_fractions) print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}") velocity = np.zeros(shape=(dim, )) initializer = MixtureInitializer(dim=dim, nspecies=nspecies, pressure=can_p, temperature=can_t, massfractions=can_y, velocity=velocity) my_boundary = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: my_boundary} if rst_filename: current_step = rst_step current_t = rst_time if logmgr: from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) if order == rst_order: current_state = restart_data["state"] else: rst_state = restart_data["state"] old_discr = EagerDGDiscretization(actx, local_mesh, order=rst_order, mpi_communicator=comm) from meshmode.discretization.connection import make_same_mesh_connection connection = make_same_mesh_connection( actx, discr.discr_from_dd("vol"), old_discr.discr_from_dd("vol")) current_state = connection(rst_state) else: # Set the current state from time 0 current_state = initializer(eos=eos, x_vec=nodes) # Inspection at physics debugging time if debug: print("Initial MIRGE-Com state:") print(f"{current_state=}") print(f"Initial DV pressure: {eos.pressure(current_state)}") print(f"Initial DV temperature: {eos.temperature(current_state)}") # }}} visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ eosname = eos.__class__.__name__ init_message = make_init_message(dim=dim, order=order, nelements=local_nelements, global_nelements=global_nelements, dt=current_dt, t_final=t_final, nstatus=nstatus, nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, initname=initname, eosname=eosname, casename=casename) # Cantera equilibrate calculates the expected end state @ chemical equilibrium # i.e. the expected state after all reactions cantera_soln.equilibrate("UV") eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY eq_pressure = cantera_soln.P # Report the expected final state to the user if rank == 0: logger.info(init_message) logger.info(f"Expected equilibrium state:" f" {eq_pressure=}, {eq_temperature=}," f" {eq_density=}, {eq_mass_fractions=}") def my_write_status(dt, cfl): status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}" if rank == 0: logger.info(status_msg) def my_write_viz(step, t, dt, state, ts_field=None, dv=None, production_rates=None, cfl=None): if dv is None: dv = eos.dependent_vars(state) if production_rates is None: production_rates = eos.get_production_rates(state) if ts_field is None: ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state) viz_fields = [("cv", state), ("dv", dv), ("production_rates", production_rates), ("dt" if constant_cfl else "cfl", ts_field)] write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True, vis_timer=vis_timer) def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) if rst_fname == rst_filename: if rank == 0: logger.info("Skipping overwrite of restart file.") else: rst_data = { "local_mesh": local_mesh, "state": state, "t": t, "step": step, "order": order, "global_nelements": global_nelements, "num_parts": nproc } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(dv): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ or check_range_local(discr, "vol", dv.pressure, 1e5, 2.4e5): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") if check_range_local(discr, "vol", dv.temperature, 1.498e3, 1.52e3): health_error = True logger.info(f"{rank=}: Invalid temperature data found.") return health_error def my_get_timestep(t, dt, state): # richer interface to calculate {dt,cfl} returns node-local estimates t_remaining = max(0, t_final - t) if constant_cfl: from mirgecom.inviscid import get_inviscid_timestep ts_field = current_cfl * get_inviscid_timestep( discr, eos=eos, cv=state) from grudge.op import nodal_min dt = nodal_min(discr, "vol", ts_field) cfl = current_cfl else: from mirgecom.inviscid import get_inviscid_cfl ts_field = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) from grudge.op import nodal_max cfl = nodal_max(discr, "vol", ts_field) return ts_field, cfl, min(t_remaining, dt) def my_pre_step(step, t, dt, state): try: dv = None if logmgr: logmgr.tick_before() from mirgecom.simutil import check_step do_viz = check_step(step=step, interval=nviz) do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) do_status = check_step(step=step, interval=nstatus) if do_health: dv = eos.dependent_vars(state) from mirgecom.simutil import allsync health_errors = allsync(my_health_check(dv), comm, op=MPI.LOR) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state) if do_status: my_write_status(dt, cfl) if do_restart: my_write_restart(step=step, t=t, state=state) if do_viz: production_rates = eos.get_production_rates(state) if dv is None: dv = eos.dependent_vars(state) my_write_viz(step=step, t=t, dt=dt, state=state, dv=dv, production_rates=production_rates, ts_field=ts_field, cfl=cfl) except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, dt=dt, state=state) my_write_restart(step=step, t=t, state=state) raise return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() return state, dt def my_rhs(t, state): return (euler_operator( discr, cv=state, time=t, boundaries=boundaries, eos=eos) + eos.get_species_source_terms(state)) current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, current_cfl, eos, t_final, constant_cfl) current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, state=current_state, t=current_t, t_final=t_final) # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") final_dv = eos.dependent_vars(current_state) final_dm = eos.get_production_rates(current_state) ts_field, cfl, dt = my_get_timestep(t=current_t, dt=current_dt, state=current_state) my_write_viz(step=current_step, t=current_t, dt=dt, state=current_state, dv=final_dv, production_rates=final_dm, ts_field=ts_field, cfl=cfl) my_write_status(dt=dt, cfl=cfl) my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: logmgr.close() elif use_profiling: print(actx.tabulate_profiling_data()) finish_tol = 1e-16 assert np.abs(current_t - t_final) < finish_tol