def main():
logmgr = LogManager("mylog.dat", "w") # , comm=...
# set a run property
logmgr.set_constant("myconst", uniform(0, 1))
add_run_info(logmgr)
add_general_quantities(logmgr)
add_simulation_quantities(logmgr)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_quantity(Fifteen("fifteen"))
logmgr.add_watches(["step.max", "t_sim.max", "t_step.max"])
for istep in range(200):
logmgr.tick_before()
dt = uniform(0.01, 0.1)
set_dt(logmgr, dt)
sleep(dt)
# Illustrate custom timers
if istep % 10 == 0:
with vis_timer.start_sub_timer():
sleep(0.05)
# Illustrate warnings capture
if uniform(0, 1) < 0.05:
warn("Oof. Something went awry.")
logmgr.tick_after()
logmgr.close()
def main(ctx_factory=cl.create_some_context,
use_profiling=False,
use_logmgr=False,
use_leap=False):
"""Drive the example."""
from mpi4py import MPI
comm = MPI.COMM_WORLD
logmgr = initialize_logmgr(use_logmgr,
filename="vortex.sqlite",
mode="wu",
mpi_comm=comm)
cl_ctx = ctx_factory()
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLProfilingArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
dim = 2
nel_1d = 16
order = 3
exittol = .1
t_final = 0.1
current_cfl = 1.0
vel = np.zeros(shape=(dim, ))
orig = np.zeros(shape=(dim, ))
vel[:dim] = 1.0
current_dt = .001
current_t = 0
eos = IdealSingleGas()
initializer = Vortex2D(center=orig, velocity=vel)
casename = "vortex"
boundaries = {BTAG_ALL: PrescribedBoundary(initializer)}
constant_cfl = False
nstatus = 10
nviz = 10
rank = 0
checkpoint_t = current_t
current_step = 0
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
box_ll = -5.0
box_ur = 5.0
rank = comm.Get_rank()
if dim != 2:
raise ValueError("This example must be run with dim = 2.")
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
current_state = initializer(nodes)
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
logmgr.add_watches([
"step.max", "t_step.max", "t_log.max", "min_temperature",
"L2_norm_momentum1"
])
try:
logmgr.add_watches(
["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["multiply_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
return euler_operator(discr,
cv=state,
t=t,
boundaries=boundaries,
eos=eos)
def my_checkpoint(step, t, dt, state):
return sim_checkpoint(discr,
visualizer,
eos,
cv=state,
exact_soln=initializer,
vizname=casename,
step=step,
t=t,
dt=dt,
nstatus=nstatus,
nviz=nviz,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm,
vis_timer=vis_timer)
try:
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t=current_t, t_final=t_final, logmgr=logmgr,
eos=eos, dim=dim)
except ExactSolutionMismatch as ex:
current_step = ex.step
current_t = ex.t
current_state = ex.state
# if current_t != checkpoint_t:
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state)
if current_t - t_final < 0:
raise ValueError("Simulation exited abnormally")
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
def main(ctx_factory=cl.create_some_context, use_logmgr=True,
use_overintegration=False,
use_leap=False, use_profiling=False, casename=None,
rst_filename=None, actx_class=PyOpenCLArrayContext):
"""Drive the example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
num_parts = comm.Get_size()
from mirgecom.simutil import global_reduce as _global_reduce
global_reduce = partial(_global_reduce, comm=comm)
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
# timestepping control
current_step = 0
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
t_final = 0.1
current_cfl = 1.0
current_dt = .01
current_t = 0
constant_cfl = False
# some i/o frequencies
nstatus = 1
nrestart = 5
nviz = 10
nhealth = 1
dim = 3
rst_path = "restart_data/"
rst_pattern = (
rst_path + "{cname}-{step:04d}-{rank:04d}.pkl"
)
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["num_parts"] == num_parts
else: # generate the grid from scratch
from meshmode.mesh.generation import generate_regular_rect_mesh
box_ll = -1
box_ur = 1
nel_1d = 16
generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim,
b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim)
local_mesh, global_nelements = generate_and_distribute_mesh(comm,
generate_mesh)
local_nelements = local_mesh.nelements
from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD
from meshmode.discretization.poly_element import \
default_simplex_group_factory, QuadratureSimplexGroupFactory
order = 1
discr = EagerDGDiscretization(
actx, local_mesh,
discr_tag_to_group_factory={
DISCR_TAG_BASE: default_simplex_group_factory(
base_dim=local_mesh.dim, order=order),
DISCR_TAG_QUAD: QuadratureSimplexGroupFactory(2*order + 1)
},
mpi_communicator=comm
)
nodes = thaw(discr.nodes(), actx)
if use_overintegration:
quadrature_tag = DISCR_TAG_QUAD
else:
quadrature_tag = None
vis_timer = None
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging, units_for_logging)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
eos = IdealSingleGas()
gas_model = GasModel(eos=eos)
vel = np.zeros(shape=(dim,))
orig = np.zeros(shape=(dim,))
initializer = Lump(dim=dim, center=orig, velocity=vel, rhoamp=0.0)
wall = AdiabaticSlipBoundary()
boundaries = {BTAG_ALL: wall}
uniform_state = initializer(nodes)
acoustic_pulse = AcousticPulse(dim=dim, amplitude=1.0, width=.1,
center=orig)
if rst_filename:
current_t = restart_data["t"]
current_step = restart_data["step"]
current_cv = restart_data["cv"]
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
else:
# Set the current state from time 0
current_cv = acoustic_pulse(x_vec=nodes, cv=uniform_state, eos=eos)
current_state = make_fluid_state(current_cv, gas_model)
visualizer = make_visualizer(discr)
initname = "pulse"
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim, order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt, t_final=t_final, nstatus=nstatus,
nviz=nviz, cfl=current_cfl,
constant_cfl=constant_cfl, initname=initname,
eosname=eosname, casename=casename)
if rank == 0:
logger.info(init_message)
def my_write_viz(step, t, state, dv=None):
if dv is None:
dv = eos.dependent_vars(state)
viz_fields = [("cv", state),
("dv", dv)]
from mirgecom.simutil import write_visfile
write_visfile(discr, viz_fields, visualizer, vizname=casename,
step=step, t=t, overwrite=True, vis_timer=vis_timer)
def my_write_restart(step, t, state):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname != rst_filename:
rst_data = {
"local_mesh": local_mesh,
"cv": state,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": num_parts
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(pressure):
health_error = False
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", pressure) \
or check_range_local(discr, "vol", pressure, .8, 1.5):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
return health_error
def my_pre_step(step, t, dt, state):
fluid_state = make_fluid_state(state, gas_model)
dv = fluid_state.dv
try:
if logmgr:
logmgr.tick_before()
from mirgecom.simutil import check_step
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
if do_health:
health_errors = global_reduce(my_health_check(dv.pressure), op="lor")
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
if do_restart:
my_write_restart(step=step, t=t, state=state)
if do_viz:
my_write_viz(step=step, t=t, state=state, dv=dv)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
my_write_viz(step=step, t=t, state=state)
my_write_restart(step=step, t=t, state=state)
raise
dt = get_sim_timestep(discr, fluid_state, t, dt, current_cfl, t_final,
constant_cfl)
return state, dt
def my_post_step(step, t, dt, state):
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, state, eos)
logmgr.tick_after()
return state, dt
def my_rhs(t, state):
fluid_state = make_fluid_state(cv=state, gas_model=gas_model)
return euler_operator(discr, state=fluid_state, time=t,
boundaries=boundaries,
gas_model=gas_model,
quadrature_tag=quadrature_tag)
current_dt = get_sim_timestep(discr, current_state, current_t, current_dt,
current_cfl, t_final, constant_cfl)
current_step, current_t, current_cv = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step,
post_step_callback=my_post_step, dt=current_dt,
state=current_cv, t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
final_state = make_fluid_state(current_cv, gas_model)
final_dv = final_state.dv
my_write_viz(step=current_step, t=current_t, state=current_cv, dv=final_dv)
my_write_restart(step=current_step, t=current_t, state=current_cv)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
assert np.abs(current_t - t_final) < finish_tol
def main(ctx_factory=cl.create_some_context,
snapshot_pattern="flame1d-{step:06d}-{rank:04d}.pkl",
restart_step=None,
use_profiling=False,
use_logmgr=False):
"""Drive the Y0 example."""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = 0
rank = comm.Get_rank()
nparts = comm.Get_size()
"""logging and profiling"""
logmgr = initialize_logmgr(use_logmgr,
filename="flame1d.sqlite",
mode="wo",
mpi_comm=comm)
cl_ctx = ctx_factory()
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLProfilingArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
#nviz = 500
#nrestart = 500
nviz = 1
nrestart = 3
#current_dt = 5.0e-8 # stable with euler
current_dt = 5.0e-8 # stable with rk4
#current_dt = 4e-7 # stable with lrsrk144
t_final = 1.5e-7
dim = 2
order = 1
exittol = 1000000000000
#t_final = 0.001
current_cfl = 1.0
current_t = 0
constant_cfl = False
nstatus = 10000000000
rank = 0
checkpoint_t = current_t
current_step = 0
vel_burned = np.zeros(shape=(dim, ))
vel_unburned = np.zeros(shape=(dim, ))
# {{{ Set up initial state using Cantera
# Use Cantera for initialization
# -- Pick up a CTI for the thermochemistry config
# --- Note: Users may add their own CTI file by dropping it into
# --- mirgecom/mechanisms alongside the other CTI files.
from mirgecom.mechanisms import get_mechanism_cti
# uiuc C2H4
#mech_cti = get_mechanism_cti("uiuc")
# sanDiego H2
mech_cti = get_mechanism_cti("sanDiego")
cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti)
nspecies = cantera_soln.n_species
# Initial temperature, pressure, and mixutre mole fractions are needed to
# set up the initial state in Cantera.
temp_unburned = 300.0
temp_ignition = 1500.0
# Parameters for calculating the amounts of fuel, oxidizer, and inert species
equiv_ratio = 1.0
ox_di_ratio = 0.21
# H2
stoich_ratio = 0.5
#C2H4
#stoich_ratio = 3.0
# Grab the array indices for the specific species, ethylene, oxygen, and nitrogen
# C2H4
#i_fu = cantera_soln.species_index("C2H4")
# H2
i_fu = cantera_soln.species_index("H2")
i_ox = cantera_soln.species_index("O2")
i_di = cantera_soln.species_index("N2")
x = np.zeros(nspecies)
# Set the species mole fractions according to our desired fuel/air mixture
x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio +
ox_di_ratio * equiv_ratio)
x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio
x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio
# Uncomment next line to make pylint fail when it can't find cantera.one_atm
one_atm = cantera.one_atm # pylint: disable=no-member
# one_atm = 101325.0
pres_unburned = one_atm
# Let the user know about how Cantera is being initilized
print(f"Input state (T,P,X) = ({temp_unburned}, {pres_unburned}, {x}")
# Set Cantera internal gas temperature, pressure, and mole fractios
cantera_soln.TPX = temp_unburned, pres_unburned, x
# Pull temperature, total density, mass fractions, and pressure from Cantera
# We need total density, and mass fractions to initialize the fluid/gas state.
y_unburned = np.zeros(nspecies)
can_t, rho_unburned, y_unburned = cantera_soln.TDY
can_p = cantera_soln.P
# *can_t*, *can_p* should not differ (significantly) from user's initial data,
# but we want to ensure that we use exactly the same starting point as Cantera,
# so we use Cantera's version of these data.
# now find the conditions for the burned gas
cantera_soln.equilibrate('TP')
temp_burned, rho_burned, y_burned = cantera_soln.TDY
pres_burned = cantera_soln.P
casename = "flame1d"
pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np)
# C2H4
mu = 1.e-5
kappa = 1.6e-5 # Pr = mu*rho/alpha = 0.75
# H2
mu = 1.e-5
kappa = mu * 0.08988 / 0.75 # Pr = mu*rho/alpha = 0.75
species_diffusivity = 1.e-5 * np.ones(nspecies)
transport_model = SimpleTransport(viscosity=mu,
thermal_conductivity=kappa,
species_diffusivity=species_diffusivity)
eos = PyrometheusMixture(pyrometheus_mechanism,
temperature_guess=temp_unburned,
transport_model=transport_model)
species_names = pyrometheus_mechanism.species_names
print(f"Pyrometheus mechanism species names {species_names}")
print(
f"Unburned state (T,P,Y) = ({temp_unburned}, {pres_unburned}, {y_unburned}"
)
print(f"Burned state (T,P,Y) = ({temp_burned}, {pres_burned}, {y_burned}")
flame_start_loc = 0.05
flame_speed = 1000
# use the burned conditions with a lower temperature
bulk_init = PlanarDiscontinuity(dim=dim,
disc_location=flame_start_loc,
sigma=0.01,
nspecies=nspecies,
temperature_left=temp_ignition,
temperature_right=temp_unburned,
pressure_left=pres_burned,
pressure_right=pres_unburned,
velocity_left=vel_burned,
velocity_right=vel_unburned,
species_mass_left=y_burned,
species_mass_right=y_unburned)
inflow_init = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=pres_burned,
temperature=temp_ignition,
massfractions=y_burned,
velocity=vel_burned)
outflow_init = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=pres_unburned,
temperature=temp_unburned,
massfractions=y_unburned,
velocity=vel_unburned)
inflow = PrescribedViscousBoundary(q_func=inflow_init)
outflow = PrescribedViscousBoundary(q_func=outflow_init)
wall = PrescribedViscousBoundary(
) # essentially a "dummy" use the interior solution for the exterior
from grudge import sym
boundaries = {
sym.DTAG_BOUNDARY("Inflow"): inflow,
sym.DTAG_BOUNDARY("Outflow"): outflow,
sym.DTAG_BOUNDARY("Wall"): wall
}
if restart_step is None:
char_len = 0.001
box_ll = (0.0, 0.0)
box_ur = (0.25, 0.01)
num_elements = (int((box_ur[0] - box_ll[0]) / char_len),
int((box_ur[1] - box_ll[1]) / char_len))
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh,
a=box_ll,
b=box_ur,
n=num_elements,
mesh_type="X",
boundary_tag_to_face={
"Inflow": ["-x"],
"Outflow": ["+x"],
"Wall": ["+y", "-y"]
})
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
else: # Restart
with open(snapshot_pattern.format(step=restart_step, rank=rank),
"rb") as f:
restart_data = pickle.load(f)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert comm.Get_size() == restart_data["num_parts"]
if rank == 0:
logging.info("Making discretization")
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
if restart_step is None:
if rank == 0:
logging.info("Initializing soln.")
# for Discontinuity initial conditions
current_state = bulk_init(t=0., x_vec=nodes, eos=eos)
# for uniform background initial condition
#current_state = bulk_init(nodes, eos=eos)
else:
current_t = restart_data["t"]
current_step = restart_step
current_state = unflatten(
actx, discr.discr_from_dd("vol"),
obj_array_vectorize(actx.from_numpy, restart_data["state"]))
vis_timer = None
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
logmgr.add_watches([
"step.max", "t_sim.max", "t_step.max", "t_log.max", "min_pressure",
"max_pressure", "min_temperature", "max_temperature"
])
try:
logmgr.add_watches(
["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["pyopencl_array_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
visualizer = make_visualizer(discr, order)
# initname = initializer.__class__.__name__
initname = "flame1d"
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
#timestepper = rk4_step
#timestepper = lsrk54_step
#timestepper = lsrk144_step
timestepper = euler_step
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
# check for some troublesome output types
inf_exists = not np.isfinite(discr.norm(state, np.inf))
if inf_exists:
if rank == 0:
logging.info(
"Non-finite values detected in simulation, exiting...")
# dump right now
sim_checkpoint(discr=discr,
visualizer=visualizer,
eos=eos,
q=state,
vizname=casename,
step=999999999,
t=t,
dt=current_dt,
nviz=1,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm,
vis_timer=vis_timer,
overwrite=True,
s0=s0_sc,
kappa=kappa_sc)
exit()
cv = split_conserved(dim=dim, q=state)
return (
ns_operator(discr, q=state, t=t, boundaries=boundaries, eos=eos) +
eos.get_species_source_terms(cv))
def my_checkpoint(step, t, dt, state):
write_restart = (check_step(step, nrestart)
if step != restart_step else False)
if write_restart is True:
with open(snapshot_pattern.format(step=step, rank=rank),
"wb") as f:
pickle.dump(
{
"local_mesh": local_mesh,
"state": obj_array_vectorize(actx.to_numpy,
flatten(state)),
"t": t,
"step": step,
"global_nelements": global_nelements,
"num_parts": nparts,
}, f)
def loc_fn(t):
return flame_start_loc + flame_speed * t
exact_soln = PlanarDiscontinuity(dim=dim,
disc_location=loc_fn,
sigma=0.0000001,
nspecies=nspecies,
temperature_left=temp_ignition,
temperature_right=temp_unburned,
pressure_left=pres_burned,
pressure_right=pres_unburned,
velocity_left=vel_burned,
velocity_right=vel_unburned,
species_mass_left=y_burned,
species_mass_right=y_unburned)
cv = split_conserved(dim, state)
reaction_rates = eos.get_production_rates(cv)
viz_fields = [("reaction_rates", reaction_rates)]
return sim_checkpoint(discr=discr,
visualizer=visualizer,
eos=eos,
q=state,
vizname=casename,
step=step,
t=t,
dt=dt,
nstatus=nstatus,
nviz=nviz,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm,
vis_timer=vis_timer,
overwrite=True,
exact_soln=exact_soln,
viz_fields=viz_fields)
if rank == 0:
logging.info("Stepping.")
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t_final=t_final, t=current_t, istep=current_step,
logmgr=logmgr,eos=eos,dim=dim)
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state)
if current_t - t_final < 0:
raise ValueError("Simulation exited abnormally")
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
exit()
def simple_wave_entrypoint(dim=2, num_elems=256, order=4, num_steps=30,
log_filename="grudge.dat"):
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
from mpi4py import MPI
comm = MPI.COMM_WORLD
num_parts = comm.Get_size()
n = int(num_elems ** (1./dim))
from meshmode.distributed import MPIMeshDistributor
mesh_dist = MPIMeshDistributor(comm)
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5,)*dim,
b=(0.5,)*dim,
n=(n,)*dim)
from pymetis import part_graph
_, p = part_graph(num_parts,
xadj=mesh.nodal_adjacency.neighbors_starts.tolist(),
adjncy=mesh.nodal_adjacency.neighbors.tolist())
part_per_element = np.array(p)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts)
else:
local_mesh = mesh_dist.receive_mesh_part()
vol_discr = DGDiscretizationWithBoundaries(cl_ctx, local_mesh, order=order,
mpi_communicator=comm)
source_center = np.array([0.1, 0.22, 0.33])[:local_mesh.dim]
source_width = 0.05
source_omega = 3
sym_x = sym.nodes(local_mesh.dim)
sym_source_center_dist = sym_x - source_center
sym_t = sym.ScalarVariable("t")
from grudge.models.wave import StrongWaveOperator
from meshmode.mesh import BTAG_ALL, BTAG_NONE
op = StrongWaveOperator(-0.1, vol_discr.dim,
source_f=(
sym.sin(source_omega*sym_t)
* sym.exp(
-np.dot(sym_source_center_dist, sym_source_center_dist)
/ source_width**2)),
dirichlet_tag=BTAG_NONE,
neumann_tag=BTAG_NONE,
radiation_tag=BTAG_ALL,
flux_type="upwind")
from pytools.obj_array import join_fields
fields = join_fields(vol_discr.zeros(queue),
[vol_discr.zeros(queue) for i in range(vol_discr.dim)])
from logpyle import LogManager, \
add_general_quantities, \
add_run_info, \
IntervalTimer, EventCounter
# NOTE: LogManager hangs when using a file on a shared directory.
logmgr = LogManager(log_filename, "w", comm)
add_run_info(logmgr)
add_general_quantities(logmgr)
log_quantities =\
{"rank_data_swap_timer": IntervalTimer("rank_data_swap_timer",
"Time spent evaluating RankDataSwapAssign"),
"rank_data_swap_counter": EventCounter("rank_data_swap_counter",
"Number of RankDataSwapAssign instructions evaluated"),
"exec_timer": IntervalTimer("exec_timer",
"Total time spent executing instructions"),
"insn_eval_timer": IntervalTimer("insn_eval_timer",
"Time spend evaluating instructions"),
"future_eval_timer": IntervalTimer("future_eval_timer",
"Time spent evaluating futures"),
"busy_wait_timer": IntervalTimer("busy_wait_timer",
"Time wasted doing busy wait")}
for quantity in log_quantities.values():
logmgr.add_quantity(quantity)
bound_op = bind(vol_discr, op.sym_operator())
def rhs(t, w):
val, rhs.profile_data = bound_op(queue, profile_data=rhs.profile_data,
log_quantities=log_quantities,
t=t, w=w)
return val
rhs.profile_data = {}
dt = 0.04
dt_stepper = set_up_rk4("w", dt, fields, rhs)
logmgr.tick_before()
for event in dt_stepper.run(t_end=dt * num_steps):
if isinstance(event, dt_stepper.StateComputed):
logmgr.tick_after()
logmgr.tick_before()
logmgr.tick_after()
def print_profile_data(data):
print("""execute() for rank %d:
\tInstruction Evaluation: %f%%
\tFuture Evaluation: %f%%
\tBusy Wait: %f%%
\tTotal: %f seconds""" %
(comm.Get_rank(),
data['insn_eval_time'] / data['total_time'] * 100,
data['future_eval_time'] / data['total_time'] * 100,
data['busy_wait_time'] / data['total_time'] * 100,
data['total_time']))
print_profile_data(rhs.profile_data)
logmgr.close()
def mpi_communication_entrypoint():
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue)
from mpi4py import MPI
comm = MPI.COMM_WORLD
i_local_rank = comm.Get_rank()
num_parts = comm.Get_size()
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
dt = 0.04
order = 4
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim,
b=(0.5, ) * dim,
nelements_per_axis=(16, ) * dim)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element,
num_parts)
else:
local_mesh = mesh_dist.receive_mesh_part()
vol_discr = DiscretizationCollection(actx,
local_mesh,
order=order,
mpi_communicator=comm)
source_center = np.array([0.1, 0.22, 0.33])[:local_mesh.dim]
source_width = 0.05
source_omega = 3
sym_x = sym.nodes(local_mesh.dim)
sym_source_center_dist = sym_x - source_center
sym_t = sym.ScalarVariable("t")
from grudge.models.wave import WeakWaveOperator
from meshmode.mesh import BTAG_ALL, BTAG_NONE
op = WeakWaveOperator(
0.1,
vol_discr.dim,
source_f=(
sym.sin(source_omega * sym_t) *
sym.exp(-np.dot(sym_source_center_dist, sym_source_center_dist) /
source_width**2)),
dirichlet_tag=BTAG_NONE,
neumann_tag=BTAG_NONE,
radiation_tag=BTAG_ALL,
flux_type="upwind")
from pytools.obj_array import flat_obj_array
fields = flat_obj_array(
vol_discr.zeros(actx),
[vol_discr.zeros(actx) for i in range(vol_discr.dim)])
# FIXME
# dt = op.estimate_rk4_timestep(vol_discr, fields=fields)
# FIXME: Should meshmode consider BTAG_PARTITION to be a boundary?
# Fails because: "found faces without boundary conditions"
# op.check_bc_coverage(local_mesh)
from logpyle import LogManager, \
add_general_quantities, \
add_run_info, \
IntervalTimer, EventCounter
log_filename = None
# NOTE: LogManager hangs when using a file on a shared directory.
# log_filename = "grudge_log.dat"
logmgr = LogManager(log_filename, "w", comm)
add_run_info(logmgr)
add_general_quantities(logmgr)
log_quantities =\
{"rank_data_swap_timer": IntervalTimer("rank_data_swap_timer",
"Time spent evaluating RankDataSwapAssign"),
"rank_data_swap_counter": EventCounter("rank_data_swap_counter",
"Number of RankDataSwapAssign instructions evaluated"),
"exec_timer": IntervalTimer("exec_timer",
"Total time spent executing instructions"),
"insn_eval_timer": IntervalTimer("insn_eval_timer",
"Time spend evaluating instructions"),
"future_eval_timer": IntervalTimer("future_eval_timer",
"Time spent evaluating futures"),
"busy_wait_timer": IntervalTimer("busy_wait_timer",
"Time wasted doing busy wait")}
for quantity in log_quantities.values():
logmgr.add_quantity(quantity)
logger.debug("\n%s", sym.pretty(op.sym_operator()))
bound_op = bind(vol_discr, op.sym_operator())
def rhs(t, w):
val, rhs.profile_data = bound_op(profile_data=rhs.profile_data,
log_quantities=log_quantities,
t=t,
w=w)
return val
rhs.profile_data = {}
dt_stepper = set_up_rk4("w", dt, fields, rhs)
final_t = 4
nsteps = int(final_t / dt)
logger.info("[%04d] dt %.5e nsteps %4d", i_local_rank, dt, nsteps)
# from grudge.shortcuts import make_visualizer
# vis = make_visualizer(vol_discr, vis_order=order)
step = 0
norm = bind(vol_discr, sym.norm(2, sym.var("u")))
from time import time
t_last_step = time()
logmgr.tick_before()
for event in dt_stepper.run(t_end=final_t):
if isinstance(event, dt_stepper.StateComputed):
assert event.component_id == "w"
step += 1
logger.debug("[%04d] t = %.5e |u| = %.5e ellapsed %.5e", step,
event.t, norm(u=event.state_component[0]),
time() - t_last_step)
# if step % 10 == 0:
# vis.write_vtk_file("rank%d-fld-%04d.vtu" % (i_local_rank, step),
# [("u", event.state_component[0]),
# ("v", event.state_component[1:])])
t_last_step = time()
logmgr.tick_after()
logmgr.tick_before()
logmgr.tick_after()
def print_profile_data(data):
logger.info(
"""execute() for rank %d:\n
\tInstruction Evaluation: %g\n
\tFuture Evaluation: %g\n
\tBusy Wait: %g\n
\tTotal: %g seconds""", i_local_rank,
data["insn_eval_time"] / data["total_time"] * 100,
data["future_eval_time"] / data["total_time"] * 100,
data["busy_wait_time"] / data["total_time"] * 100,
data["total_time"])
print_profile_data(rhs.profile_data)
logmgr.close()
logger.info("Rank %d exiting", i_local_rank)
def main(snapshot_pattern="wave-mpi-{step:04d}-{rank:04d}.pkl", restart_step=None,
use_profiling=False, use_logmgr=False, actx_class=PyOpenCLArrayContext):
"""Drive the example."""
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
num_parts = comm.Get_size()
logmgr = initialize_logmgr(use_logmgr,
filename="wave-mpi.sqlite", mode="wu", mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(cl_ctx,
properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
if restart_step is None:
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
nel_1d = 16
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(
a=(-0.5,)*dim, b=(0.5,)*dim,
nelements_per_axis=(nel_1d,)*dim)
print("%d elements" % mesh.nelements)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts)
del mesh
else:
local_mesh = mesh_dist.receive_mesh_part()
fields = None
else:
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(
actx, snapshot_pattern.format(step=restart_step, rank=rank)
)
local_mesh = restart_data["local_mesh"]
nel_1d = restart_data["nel_1d"]
assert comm.Get_size() == restart_data["num_parts"]
order = 3
discr = EagerDGDiscretization(actx, local_mesh, order=order,
mpi_communicator=comm)
current_cfl = 0.485
wave_speed = 1.0
from grudge.dt_utils import characteristic_lengthscales
dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed
from grudge.op import nodal_min
dt = nodal_min(discr, "vol", dt)
t_final = 1
if restart_step is None:
t = 0
istep = 0
fields = flat_obj_array(
bump(actx, discr),
[discr.zeros(actx) for i in range(discr.dim)]
)
else:
t = restart_data["t"]
istep = restart_step
assert istep == restart_step
restart_fields = restart_data["fields"]
old_order = restart_data["order"]
if old_order != order:
old_discr = EagerDGDiscretization(actx, local_mesh, order=old_order,
mpi_communicator=comm)
from meshmode.discretization.connection import make_same_mesh_connection
connection = make_same_mesh_connection(actx, discr.discr_from_dd("vol"),
old_discr.discr_from_dd("vol"))
fields = connection(restart_fields)
else:
fields = restart_fields
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr.add_watches(["step.max", "t_step.max", "t_log.max"])
try:
logmgr.add_watches(["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["multiply_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
vis = make_visualizer(discr)
def rhs(t, w):
return wave_operator(discr, c=wave_speed, w=w)
compiled_rhs = actx.compile(rhs)
while t < t_final:
if logmgr:
logmgr.tick_before()
# restart must happen at beginning of step
if istep % 100 == 0 and (
# Do not overwrite the restart file that we just read.
istep != restart_step):
from mirgecom.restart import write_restart_file
write_restart_file(
actx, restart_data={
"local_mesh": local_mesh,
"order": order,
"fields": fields,
"t": t,
"step": istep,
"nel_1d": nel_1d,
"num_parts": num_parts},
filename=snapshot_pattern.format(step=istep, rank=rank),
comm=comm
)
if istep % 10 == 0:
print(istep, t, discr.norm(fields[0]))
vis.write_parallel_vtk_file(
comm,
"fld-wave-mpi-%03d-%04d.vtu" % (rank, istep),
[
("u", fields[0]),
("v", fields[1:]),
], overwrite=True
)
fields = thaw(freeze(fields, actx), actx)
fields = rk4_step(fields, t, dt, compiled_rhs)
t += dt
istep += 1
if logmgr:
set_dt(logmgr, dt)
logmgr.tick_after()
final_soln = discr.norm(fields[0])
assert np.abs(final_soln - 0.04409852463947439) < 1e-14
def main(ctx_factory=cl.create_some_context,
snapshot_pattern="y0euler-{step:06d}-{rank:04d}.pkl",
restart_step=None,
use_profiling=False,
use_logmgr=False):
"""Drive the Y0 example."""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = 0
rank = comm.Get_rank()
nparts = comm.Get_size()
"""logging and profiling"""
logmgr = initialize_logmgr(use_logmgr,
use_profiling,
filename="y0euler.sqlite",
mode="wu",
mpi_comm=comm)
cl_ctx = ctx_factory()
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLProfilingArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
#nviz = 500
#nrestart = 500
nviz = 50
nrestart = 10000000
current_dt = 1.0e-7
#t_final = 5.e-7
t_final = 3e-4
dim = 2
order = 1
exittol = 10000000 # do never exit when comparing to exact solution
#t_final = 0.001
current_cfl = 1.0
vel_init = np.zeros(shape=(dim, ))
vel_inflow = np.zeros(shape=(dim, ))
vel_outflow = np.zeros(shape=(dim, ))
orig = np.zeros(shape=(dim, ))
#vel[0] = 340.0
#vel_inflow[0] = 100.0 # m/s
current_t = 0
casename = "y0euler"
constant_cfl = False
# no internal euler status messages
nstatus = 1000000000
checkpoint_t = current_t
current_step = 0
# working gas: CO2 #
# gamma = 1.289
# MW=44.009 g/mol
# cp = 37.135 J/mol-K,
# rho= 1.977 kg/m^3 @298K
gamma_CO2 = 1.289
R_CO2 = 8314.59 / 44.009
# background
# 100 Pa
# 298 K
# rho = 1.77619667e-3 kg/m^3
# velocity = 0,0,0
rho_bkrnd = 1.77619667e-3
pres_bkrnd = 100
temp_bkrnd = 298
c_bkrnd = math.sqrt(gamma_CO2 * pres_bkrnd / rho_bkrnd)
# isentropic shock relations #
# lab frame, moving shock
# state 1 is behind (downstream) the shock, state 2 is in front (upstream) of the shock
mach = 2.0
pressure_ratio = (2. * gamma_CO2 * mach * mach -
(gamma_CO2 - 1.)) / (gamma_CO2 + 1.)
density_ratio = (gamma_CO2 + 1.) * mach * mach / (
(gamma_CO2 - 1.) * mach * mach + 2.)
mach2 = math.sqrt(((gamma_CO2 - 1.) * mach * mach + 2.) /
(2. * gamma_CO2 * mach * mach - (gamma_CO2 - 1.)))
rho1 = rho_bkrnd
pressure1 = pres_bkrnd
rho2 = rho1 * density_ratio
pressure2 = pressure1 * pressure_ratio
velocity1 = 0.
velocity2 = -mach * c_bkrnd * (1 / density_ratio - 1)
c_shkd = math.sqrt(gamma_CO2 * pressure2 / rho2)
vel_inflow[0] = velocity2
timestepper = rk4_step
eos = IdealSingleGas(gamma=gamma_CO2, gas_const=R_CO2)
bulk_init = Discontinuity(dim=dim,
x0=.05,
sigma=0.01,
rhol=rho2,
rhor=rho1,
pl=pressure2,
pr=pressure1,
ul=vel_inflow[0],
ur=0.)
inflow_init = Lump(dim=dim,
rho0=rho2,
p0=pressure2,
center=orig,
velocity=vel_inflow,
rhoamp=0.0)
outflow_init = Lump(dim=dim,
rho0=rho1,
p0=pressure1,
center=orig,
velocity=vel_outflow,
rhoamp=0.0)
inflow = PrescribedBoundary(inflow_init)
outflow = PrescribedBoundary(outflow_init)
wall = AdiabaticSlipBoundary()
dummy = DummyBoundary()
# shock capturing parameters
# sonic conditions
density_ratio = (gamma_CO2 + 1.) * 1.0 / ((gamma_CO2 - 1.) + 2.)
density_star = rho1 * density_ratio
shock_thickness = 20 * 0.001 # on the order of 3 elements, should match what is in mesh generator
# alpha is ~h/p (spacing/order)
#alpha_sc = shock_thickness*abs(velocity1-velocity2)*density_star
alpha_sc = 0.1
# sigma is ~p^-4
sigma_sc = -11.0
# kappa is empirical ...
kappa_sc = 0.5
print(
f"Shock capturing parameters: alpha {alpha_sc}, s0 {sigma_sc}, kappa {kappa_sc}"
)
# timestep estimate
wave_speed = max(mach2 * c_bkrnd, c_shkd + velocity2)
char_len = 0.001
area = char_len * char_len / 2
perimeter = 2 * char_len + math.sqrt(2 * char_len * char_len)
h = 2 * area / perimeter
dt_est = 1 / (wave_speed * order * order / h)
print(f"Time step estimate {dt_est}\n")
dt_est_visc = 1 / (wave_speed * order * order / h +
alpha_sc * order * order * order * order / h / h)
print(f"Viscous timestep estimate {dt_est_visc}\n")
from grudge import sym
# boundaries = {BTAG_ALL: DummyBoundary}
boundaries = {
sym.DTAG_BOUNDARY("Inflow"): inflow,
sym.DTAG_BOUNDARY("Outflow"): outflow,
sym.DTAG_BOUNDARY("Wall"): wall
}
#local_mesh, global_nelements = create_parallel_grid(comm,
#get_pseudo_y0_mesh)
#
#local_nelements = local_mesh.nelements
if restart_step is None:
local_mesh, global_nelements = create_parallel_grid(comm, get_mesh)
local_nelements = local_mesh.nelements
else: # Restart
with open(snapshot_pattern.format(step=restart_step, rank=rank),
"rb") as f:
restart_data = pickle.load(f)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert comm.Get_size() == restart_data["num_parts"]
if rank == 0:
logging.info("Making discretization")
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
if restart_step is None:
if rank == 0:
logging.info("Initializing soln.")
current_state = bulk_init(0, nodes, eos=eos)
else:
current_t = restart_data["t"]
current_step = restart_step
current_state = unflatten(
actx, discr.discr_from_dd("vol"),
obj_array_vectorize(actx.from_numpy, restart_data["state"]))
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_discretization_quantities(logmgr, discr, eos, dim)
#logmgr_add_package_versions(logmgr)
logmgr.add_watches([
"step.max", "t_sim.max", "t_step.max", "min_pressure",
"max_pressure", "min_temperature", "max_temperature"
])
try:
logmgr.add_watches(["memory_usage.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["pyopencl_array_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
#visualizer = make_visualizer(discr, discr.order + 3
#if discr.dim == 2 else discr.order)
visualizer = make_visualizer(discr, discr.order)
# initname = initializer.__class__.__name__
initname = "pseudoY0"
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
#return inviscid_operator(discr, eos=eos, boundaries=boundaries, q=state, t=t)
return (inviscid_operator(
discr, q=state, t=t, boundaries=boundaries, eos=eos) +
artificial_viscosity(discr,
t=t,
r=state,
eos=eos,
boundaries=boundaries,
alpha=alpha_sc,
sigma=sigma_sc,
kappa=kappa_sc))
def my_checkpoint(step, t, dt, state):
write_restart = (check_step(step, nrestart)
if step != restart_step else False)
if write_restart is True:
with open(snapshot_pattern.format(step=step, rank=rank),
"wb") as f:
pickle.dump(
{
"local_mesh": local_mesh,
"state": obj_array_vectorize(actx.to_numpy,
flatten(state)),
"t": t,
"step": step,
"global_nelements": global_nelements,
"num_parts": nparts,
}, f)
#x0=f(time)
exact_soln = Discontinuity(dim=dim,
x0=.05,
sigma=0.00001,
rhol=rho2,
rhor=rho1,
pl=pressure2,
pr=pressure1,
ul=vel_inflow[0],
ur=0.,
uc=mach * c_bkrnd)
return sim_checkpoint(discr=discr,
visualizer=visualizer,
eos=eos,
q=state,
vizname=casename,
step=step,
t=t,
dt=dt,
nstatus=nstatus,
nviz=nviz,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm,
vis_timer=vis_timer,
overwrite=True,
exact_soln=exact_soln,
sigma=sigma_sc,
kappa=kappa_sc)
if rank == 0:
logging.info("Stepping.")
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t_final=t_final, t=current_t, istep=current_step,
logmgr=logmgr,eos=eos,dim=dim)
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state)
if current_t - t_final < 0:
raise ValueError("Simulation exited abnormally")
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
def main(write_output=True):
from grudge.timestep.runge_kutta import LSRK4TimeStepper
from math import sqrt, pi, exp
from grudge.backends import guess_run_context
rcon = guess_run_context()
epsilon0 = 8.8541878176e-12 # C**2 / (N m**2)
mu0 = 4 * pi * 1e-7 # N/A**2.
epsilon = 1 * epsilon0
mu = 1 * mu0
c = 1 / sqrt(mu * epsilon)
pml_width = 0.5
#mesh = make_mesh(a=np.array((-1,-1,-1)), b=np.array((1,1,1)),
#mesh = make_mesh(a=np.array((-3,-3)), b=np.array((3,3)),
mesh = make_mesh(
a=np.array((-1, -1)),
b=np.array((1, 1)),
#mesh = make_mesh(a=np.array((-2,-2)), b=np.array((2,2)),
pml_width=pml_width,
max_volume=0.01)
if rcon.is_head_rank:
mesh_data = rcon.distribute_mesh(mesh)
else:
mesh_data = rcon.receive_mesh()
class Current:
def volume_interpolant(self, t, discr):
from grudge.tools import make_obj_array
result = discr.volume_zeros(kind="numpy", dtype=np.float64)
omega = 6 * c
if omega * t > 2 * pi:
return make_obj_array([result, result, result])
x = make_obj_array(discr.nodes.T)
r = np.sqrt(np.dot(x, x))
idx = r < 0.3
result[idx] = (1+np.cos(pi*r/0.3))[idx] \
*np.sin(omega*t)**3
result = discr.convert_volume(result,
kind=discr.compute_kind,
dtype=discr.default_scalar_type)
return make_obj_array([-result, result, result])
order = 3
discr = rcon.make_discretization(mesh_data,
order=order,
debug=["cuda_no_plan"])
from grudge.visualization import VtkVisualizer
if write_output:
vis = VtkVisualizer(discr, rcon, "em-%d" % order)
from grudge.mesh import BTAG_ALL, BTAG_NONE
from grudge.data import GivenFunction, TimeHarmonicGivenFunction, TimeIntervalGivenFunction
from grudge.models.em import MaxwellOperator
from grudge.models.pml import \
AbarbanelGottliebPMLMaxwellOperator, \
AbarbanelGottliebPMLTMMaxwellOperator, \
AbarbanelGottliebPMLTEMaxwellOperator
op = AbarbanelGottliebPMLTEMaxwellOperator(epsilon,
mu,
flux_type=1,
current=Current(),
pec_tag=BTAG_ALL,
absorb_tag=BTAG_NONE,
add_decay=True)
fields = op.assemble_ehpq(discr=discr)
stepper = LSRK4TimeStepper()
if rcon.is_head_rank:
print("order %d" % order)
print("#elements=", len(mesh.elements))
# diagnostics setup ---------------------------------------------------
from logpyle import LogManager, add_general_quantities, \
add_simulation_quantities, add_run_info
if write_output:
log_file_name = "maxwell-%d.dat" % order
else:
log_file_name = None
logmgr = LogManager(log_file_name, "w", rcon.communicator)
add_run_info(logmgr)
add_general_quantities(logmgr)
add_simulation_quantities(logmgr)
discr.add_instrumentation(logmgr)
stepper.add_instrumentation(logmgr)
from logpyle import IntervalTimer
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
from grudge.log import EMFieldGetter, add_em_quantities
field_getter = EMFieldGetter(discr, op, lambda: fields)
add_em_quantities(logmgr, op, field_getter)
logmgr.add_watches(
["step.max", "t_sim.max", ("W_field", "W_el+W_mag"), "t_step.max"])
from grudge.log import LpNorm
class FieldIdxGetter:
def __init__(self, whole_getter, idx):
self.whole_getter = whole_getter
self.idx = idx
def __call__(self):
return self.whole_getter()[self.idx]
# timestep loop -------------------------------------------------------
t = 0
pml_coeff = op.coefficients_from_width(discr, width=pml_width)
rhs = op.bind(discr, pml_coeff)
try:
from grudge.timestep import times_and_steps
step_it = times_and_steps(
final_time=4 / c,
logmgr=logmgr,
max_dt_getter=lambda t: op.estimate_timestep(
discr, stepper=stepper, t=t, fields=fields))
for step, t, dt in step_it:
if step % 10 == 0 and write_output:
e, h, p, q = op.split_ehpq(fields)
visf = vis.make_file("em-%d-%04d" % (order, step))
#pml_rhs_e, pml_rhs_h, pml_rhs_p, pml_rhs_q = \
#op.split_ehpq(rhs(t, fields))
j = Current().volume_interpolant(t, discr)
vis.add_data(
visf,
[
("e", discr.convert_volume(e, "numpy")),
("h", discr.convert_volume(h, "numpy")),
("p", discr.convert_volume(p, "numpy")),
("q", discr.convert_volume(q, "numpy")),
("j", discr.convert_volume(j, "numpy")),
#("pml_rhs_e", pml_rhs_e),
#("pml_rhs_h", pml_rhs_h),
#("pml_rhs_p", pml_rhs_p),
#("pml_rhs_q", pml_rhs_q),
#("max_rhs_e", max_rhs_e),
#("max_rhs_h", max_rhs_h),
#("max_rhs_p", max_rhs_p),
#("max_rhs_q", max_rhs_q),
],
time=t,
step=step)
visf.close()
fields = stepper(fields, t, dt, rhs)
_, _, energies_data = logmgr.get_expr_dataset("W_el+W_mag")
energies = [value for tick_nbr, value in energies_data]
assert energies[-1] < max(energies) * 1e-2
finally:
logmgr.close()
if write_output:
vis.close()
def main(ctx_factory=cl.create_some_context,
snapshot_pattern="y0euler-{step:06d}-{rank:04d}.pkl",
restart_step=None, use_profiling=False, use_logmgr=False):
"""Drive the Y0 example."""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = 0
rank = comm.Get_rank()
nparts = comm.Get_size()
"""logging and profiling"""
logmgr = initialize_logmgr(use_logmgr, filename="y0euler.sqlite",
mode="wo", mpi_comm=comm)
cl_ctx = ctx_factory()
if use_profiling:
queue = cl.CommandQueue(cl_ctx,
properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLProfilingArrayContext(queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
#nviz = 500
#nrestart = 500
nviz = 100
nrestart = 100
#current_dt = 2.5e-8 # stable with euler
current_dt = 4e-7 # stable with lrsrk144
t_final = 5.e-1
dim = 3
order = 1
exittol = .09
#t_final = 0.001
current_cfl = 1.0
vel_init = np.zeros(shape=(dim,))
vel_inflow = np.zeros(shape=(dim,))
vel_outflow = np.zeros(shape=(dim,))
orig = np.zeros(shape=(dim,))
orig[0] = 0.83
orig[2] = 0.001
#vel[0] = 340.0
#vel_inflow[0] = 100.0 # m/s
current_t = 0
casename = "y0euler"
constant_cfl = False
nstatus = 10000000000
rank = 0
checkpoint_t = current_t
current_step = 0
# working gas: CO2 #
# gamma = 1.289
# MW=44.009 g/mol
# cp = 37.135 J/mol-K,
# rho= 1.977 kg/m^3 @298K
gamma_CO2 = 1.289
R_CO2 = 8314.59/44.009
# background
# 100 Pa
# 298 K
# rho = 1.77619667e-3 kg/m^3
# velocity = 0,0,0
rho_bkrnd=1.77619667e-3
pres_bkrnd=100
temp_bkrnd=298
# nozzle inflow #
#
# stagnation tempertuare 298 K
# stagnation pressure 1.5e Pa
#
# isentropic expansion based on the area ratios between the inlet (r=13e-3m) and the throat (r=6.3e-3)
#
# MJA, this is calculated offline, add some code to do it for us
#
# Mach number=0.139145
# pressure=148142
# temperature=297.169
# density=2.63872
# gamma=1.289
# calculate the inlet Mach number from the area ratio
nozzleInletRadius = 13.e-3
nozzleThroatRadius = 6.3e-3
nozzleInletArea = math.pi*nozzleInletRadius*nozzleInletRadius
nozzleThroatArea = math.pi*nozzleThroatRadius*nozzleThroatRadius
inletAreaRatio = nozzleInletArea/nozzleThroatArea
def getMachFromAreaRatio(area_ratio, gamma, mach_guess=0.01):
error=1.e-8
nextError=1.e8
g=gamma
M0=mach_guess
while nextError > error:
R = ((2/(g+1)+((g-1)/(g+1)*M0*M0))**(((g+1)/(2*g-2))))/M0-area_ratio
dRdM = (2*((2/(g+1)+((g-1)/(g+1)*M0*M0))**(((g+1)/(2*g-2))))/
(2*g-2)*(g-1)/(2/(g+1)+((g-1)/(g+1)*M0*M0))-
((2/(g+1)+((g-1)/(g+1)*M0*M0))**(((g+1)/(2*g-2))))* M0**(-2))
M1=M0-R/dRdM
nextError=abs(R)
M0=M1
return M1
def getIsentropicPressure(mach, P0, gamma):
pressure=(1.+(gamma-1.)*0.5*math.pow(mach,2))
pressure=P0*math.pow(pressure,(-gamma/(gamma-1.)))
return pressure
def getIsentropicTemperature(mach, T0, gamma):
temperature=(1.+(gamma-1.)*0.5*math.pow(mach,2))
temperature=T0*math.pow(temperature,-1.0)
return temperature
inlet_mach = getMachFromAreaRatio(area_ratio = inletAreaRatio, gamma=gamma_CO2, mach_guess = 0.01);
# ramp the stagnation pressure
start_ramp_pres = 1000
ramp_interval = 5.e-3
t_ramp_start = 1e-5
pres_inflow = getIsentropicPressure(mach=inlet_mach, P0=start_ramp_pres, gamma=gamma_CO2)
temp_inflow = getIsentropicTemperature(mach=inlet_mach, T0=298, gamma=gamma_CO2)
rho_inflow = pres_inflow/temp_inflow/R_CO2
print(f'inlet Mach number {inlet_mach}')
print(f'inlet temperature {temp_inflow}')
print(f'inlet pressure {pres_inflow}')
end_ramp_pres = 150000
pres_inflow_final = getIsentropicPressure(mach=inlet_mach, P0=end_ramp_pres, gamma=gamma_CO2)
print(f'final inlet pressure {pres_inflow_final}')
#pres_inflow=148142
#temp_inflow=297.169
#rho_inflow=2.63872
#mach_inflow=infloM = 0.139145
vel_inflow[0] = inlet_mach*math.sqrt(gamma_CO2*pres_inflow/rho_inflow)
# starting pressure for the inflow ramp
#timestepper = rk4_step
#timestepper = lsrk54_step
timestepper = lsrk144_step
#timestepper = euler_step
eos = IdealSingleGas(gamma=gamma_CO2, gas_const=R_CO2)
bulk_init = Discontinuity(dim=dim, x0=-.30,sigma=0.005,
#bulk_init = Discontinuity(dim=dim, x0=-.31,sigma=0.04,
rhol=rho_inflow, rhor=rho_bkrnd,
pl=pres_inflow, pr=pres_bkrnd,
ul=vel_inflow, ur=vel_outflow)
#inflow_init = Lump(dim=dim, rho0=rho_inflow, p0=pres_inflow,
#center=orig, velocity=vel_inflow, rhoamp=0.0)
#outflow_init = Lump(dim=dim, rho0=rho_bkrnd, p0=pres_bkrnd,
#center=orig, velocity=vel_outflow, rhoamp=0.0)
# pressure ramp function
def inflow_ramp_pressure(t, startP=start_ramp_pres, finalP=end_ramp_pres,
ramp_interval=ramp_interval, t_ramp_start=t_ramp_start):
if t > t_ramp_start:
rampPressure = min(finalP, startP+(t-t_ramp_start)/ramp_interval*(finalP-startP))
else:
rampPressure = startP
return rampPressure
class IsentropicInflow:
def __init__(self, *, dim=1, direc=0, T0=298, P0=1e5, mach= 0.01, p_fun = None):
self._P0 = P0
self._T0 = T0
self._dim = dim
self._direc = direc
self._mach = mach
if p_fun is not None:
self._p_fun = p_fun
def __call__(self, x_vec, *, t=0, eos):
if self._p_fun is not None:
P0 = self._p_fun(t)
else:
P0 = self._P0
T0 = self._T0
gamma = eos.gamma()
gas_const = eos.gas_const()
pressure = getIsentropicPressure(mach=self._mach, P0=P0, gamma=gamma)
temperature = getIsentropicTemperature(mach=self._mach, T0=T0, gamma=gamma)
rho = pressure/temperature/gas_const
#print(f'ramp Mach number {self._mach}')
#print(f'ramp stagnation pressure {P0}')
#print(f'ramp stagnation temperature {T0}')
#print(f'ramp pressure {pressure}')
#print(f'ramp temperature {temperature}')
velocity = np.zeros(shape=(self._dim,))
velocity[self._direc] = self._mach*math.sqrt(gamma*pressure/rho)
mass = 0.0*x_vec[0] + rho
mom = velocity*mass
energy = (pressure/(gamma - 1.0)) + np.dot(mom, mom)/(2.0*mass)
from mirgecom.euler import join_conserved
return join_conserved(dim=self._dim, mass=mass, momentum=mom, energy=energy)
inflow_init = IsentropicInflow(dim=dim, T0=298, P0=start_ramp_pres,
mach = inlet_mach , p_fun=inflow_ramp_pressure)
outflow_init = Uniform(dim=dim, rho=rho_bkrnd, p=pres_bkrnd,
velocity=vel_outflow)
inflow = PrescribedBoundary(inflow_init)
outflow = PrescribedBoundary(outflow_init)
wall = AdiabaticSlipBoundary()
dummy = DummyBoundary()
alpha_sc = 0.5
# s0 is ~p^-4
#s0_sc = -11.0
s0_sc = -5.0
# kappa is empirical ...
kappa_sc = 0.5
print(f"Shock capturing parameters: alpha {alpha_sc}, s0 {s0_sc}, kappa {kappa_sc}")
# timestep estimate
#wave_speed = max(mach2*c_bkrnd,c_shkd+velocity2[0])
#char_len = 0.001
#area=char_len*char_len/2
#perimeter = 2*char_len+math.sqrt(2*char_len*char_len)
#h = 2*area/perimeter
#dt_est = 1/(wave_speed*order*order/h)
#print(f"Time step estimate {dt_est}\n")
#
#dt_est_visc = 1/(wave_speed*order*order/h+alpha_sc*order*order*order*order/h/h)
#print(f"Viscous timestep estimate {dt_est_visc}\n")
from grudge import sym
boundaries = {sym.DTAG_BOUNDARY("Inflow"): inflow,
sym.DTAG_BOUNDARY("Outflow"): outflow,
sym.DTAG_BOUNDARY("Wall"): wall}
if restart_step is None:
local_mesh, global_nelements = create_parallel_grid(comm, get_pseudo_y0_mesh)
local_nelements = local_mesh.nelements
else: # Restart
with open(snapshot_pattern.format(step=restart_step, rank=rank), "rb") as f:
restart_data = pickle.load(f)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert comm.Get_size() == restart_data["num_parts"]
if rank == 0:
logging.info("Making discretization")
discr = EagerDGDiscretization(
actx, local_mesh, order=order, mpi_communicator=comm
)
nodes = thaw(actx, discr.nodes())
if restart_step is None:
if rank == 0:
logging.info("Initializing soln.")
# for Discontinuity initial conditions
current_state = bulk_init(0, nodes, eos=eos)
# for uniform background initial condition
#current_state = bulk_init(nodes, eos=eos)
else:
current_t = restart_data["t"]
current_step = restart_step
current_state = unflatten(
actx, discr.discr_from_dd("vol"),
obj_array_vectorize(actx.from_numpy, restart_data["state"]))
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging, units_for_logging)
#logmgr_add_package_versions(logmgr)
logmgr.add_watches(["step.max", "t_sim.max", "t_step.max", "t_log.max",
"min_pressure", "max_pressure",
"min_temperature", "max_temperature"])
try:
logmgr.add_watches(["memory_usage.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["pyopencl_array_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
visualizer = make_visualizer(discr, discr.order + 3
if discr.dim == 2 else discr.order)
# initname = initializer.__class__.__name__
initname = "pseudoY0"
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim, order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt, t_final=t_final,
nstatus=nstatus, nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname, casename=casename)
if rank == 0:
logger.info(init_message)
get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t,
dt=current_dt, cfl=current_cfl, eos=eos,
t_final=t_final, constant_cfl=constant_cfl)
def my_rhs(t, state):
#return inviscid_operator(discr, eos=eos, boundaries=boundaries, q=state, t=t)
return ( inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos)
+ artificial_viscosity(discr,t=t, r=state, eos=eos, boundaries=boundaries,
alpha=alpha_sc, s0=s0_sc, kappa=kappa_sc))
def my_checkpoint(step, t, dt, state):
write_restart = (check_step(step, nrestart)
if step != restart_step else False)
if write_restart is True:
with open(snapshot_pattern.format(step=step, rank=rank), "wb") as f:
pickle.dump({
"local_mesh": local_mesh,
"state": obj_array_vectorize(actx.to_numpy, flatten(state)),
"t": t,
"step": step,
"global_nelements": global_nelements,
"num_parts": nparts,
}, f)
return sim_checkpoint(discr=discr, visualizer=visualizer, eos=eos,
q=state, vizname=casename,
step=step, t=t, dt=dt, nstatus=nstatus,
nviz=nviz, exittol=exittol,
constant_cfl=constant_cfl, comm=comm, vis_timer=vis_timer,
overwrite=True,s0=s0_sc,kappa=kappa_sc)
if rank == 0:
logging.info("Stepping.")
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t_final=t_final, t=current_t, istep=current_step,
logmgr=logmgr,eos=eos,dim=dim)
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step, t=current_t,
dt=(current_t - checkpoint_t),
state=current_state)
if current_t - t_final < 0:
raise ValueError("Simulation exited abnormally")
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
def main(write_output=True):
from math import sqrt, pi, exp
from os.path import join
from grudge.backends import guess_run_context
rcon = guess_run_context()
epsilon0 = 8.8541878176e-12 # C**2 / (N m**2)
mu0 = 4 * pi * 1e-7 # N/A**2.
epsilon = 1 * epsilon0
mu = 1 * mu0
output_dir = "maxwell-2d"
import os
if not os.access(output_dir, os.F_OK):
os.makedirs(output_dir)
from grudge.mesh.generator import make_disk_mesh
mesh = make_disk_mesh(r=0.5, max_area=1e-3)
if rcon.is_head_rank:
mesh_data = rcon.distribute_mesh(mesh)
else:
mesh_data = rcon.receive_mesh()
class CurrentSource:
shape = (3, )
def __call__(self, x, el):
return [0, 0, exp(-80 * la.norm(x))]
order = 3
final_time = 1e-8
discr = rcon.make_discretization(mesh_data,
order=order,
debug=["cuda_no_plan"])
from grudge.visualization import VtkVisualizer
if write_output:
vis = VtkVisualizer(discr, rcon, join(output_dir, "em-%d" % order))
if rcon.is_head_rank:
print("order %d" % order)
print("#elements=", len(mesh.elements))
from grudge.mesh import BTAG_ALL, BTAG_NONE
from grudge.models.em import TMMaxwellOperator
from grudge.data import make_tdep_given, TimeIntervalGivenFunction
op = TMMaxwellOperator(epsilon,
mu,
flux_type=1,
current=TimeIntervalGivenFunction(
make_tdep_given(CurrentSource()),
off_time=final_time / 10),
absorb_tag=BTAG_ALL,
pec_tag=BTAG_NONE)
fields = op.assemble_eh(discr=discr)
from grudge.timestep import LSRK4TimeStepper
stepper = LSRK4TimeStepper()
from time import time
last_tstep = time()
t = 0
# diagnostics setup ---------------------------------------------------
from logpyle import LogManager, add_general_quantities, \
add_simulation_quantities, add_run_info
if write_output:
log_file_name = join(output_dir, "maxwell-%d.dat" % order)
else:
log_file_name = None
logmgr = LogManager(log_file_name, "w", rcon.communicator)
add_run_info(logmgr)
add_general_quantities(logmgr)
add_simulation_quantities(logmgr)
discr.add_instrumentation(logmgr)
stepper.add_instrumentation(logmgr)
from logpyle import IntervalTimer
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
from grudge.log import EMFieldGetter, add_em_quantities
field_getter = EMFieldGetter(discr, op, lambda: fields)
add_em_quantities(logmgr, op, field_getter)
logmgr.add_watches(
["step.max", "t_sim.max", ("W_field", "W_el+W_mag"), "t_step.max"])
# timestep loop -------------------------------------------------------
rhs = op.bind(discr)
try:
from grudge.timestep import times_and_steps
step_it = times_and_steps(
final_time=final_time,
logmgr=logmgr,
max_dt_getter=lambda t: op.estimate_timestep(
discr, stepper=stepper, t=t, fields=fields))
for step, t, dt in step_it:
if step % 10 == 0 and write_output:
e, h = op.split_eh(fields)
visf = vis.make_file(
join(output_dir, "em-%d-%04d" % (order, step)))
vis.add_data(visf, [
("e", discr.convert_volume(e, "numpy")),
("h", discr.convert_volume(h, "numpy")),
],
time=t,
step=step)
visf.close()
fields = stepper(fields, t, dt, rhs)
assert discr.norm(fields) < 0.03
finally:
if write_output:
vis.close()
logmgr.close()
discr.close()
def main(ctx_factory=cl.create_some_context, use_logmgr=True,
use_leap=False, use_profiling=False, casename=None,
rst_filename=None, actx_class=PyOpenCLArrayContext,
log_dependent=True):
"""Drive example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nparts = comm.Get_size()
from mirgecom.simutil import global_reduce as _global_reduce
global_reduce = partial(_global_reduce, comm=comm)
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
# timestepping control
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
t_final = 1e-8
current_cfl = 1.0
current_dt = 1e-9
current_t = 0
current_step = 0
constant_cfl = False
# some i/o frequencies
nstatus = 1
nhealth = 1
nrestart = 5
nviz = 1
dim = 2
rst_path = "restart_data/"
rst_pattern = (
rst_path + "{cname}-{step:04d}-{rank:04d}.pkl"
)
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["num_parts"] == nparts
else: # generate the grid from scratch
nel_1d = 16
box_ll = -5.0
box_ur = 5.0
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim,
b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim)
local_mesh, global_nelements = generate_and_distribute_mesh(comm,
generate_mesh)
local_nelements = local_mesh.nelements
order = 3
discr = EagerDGDiscretization(
actx, local_mesh, order=order, mpi_communicator=comm
)
nodes = thaw(discr.nodes(), actx)
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
if log_dependent:
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
logmgr.add_watches([
("min_pressure", "\n------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("min_temperature", "------- T (min, max) (K) = ({value:7g}, "),
("max_temperature", "{value:7g})\n")])
# Pyrometheus initialization
from mirgecom.mechanisms import get_mechanism_cti
mech_cti = get_mechanism_cti("uiuc")
sol = cantera.Solution(phase_id="gas", source=mech_cti)
from mirgecom.thermochemistry import make_pyrometheus_mechanism_class
pyrometheus_mechanism = make_pyrometheus_mechanism_class(sol)(actx.np)
nspecies = pyrometheus_mechanism.num_species
eos = PyrometheusMixture(pyrometheus_mechanism)
from mirgecom.gas_model import GasModel, make_fluid_state
gas_model = GasModel(eos=eos)
from pytools.obj_array import make_obj_array
y0s = np.zeros(shape=(nspecies,))
for i in range(nspecies-1):
y0s[i] = 1.0 / (10.0 ** (i + 1))
spec_sum = sum([y0s[i] for i in range(nspecies-1)])
y0s[nspecies-1] = 1.0 - spec_sum
# Mixture defaults to STP (p, T) = (1atm, 300K)
velocity = np.zeros(shape=(dim,)) + 1.0
initializer = MixtureInitializer(dim=dim, nspecies=nspecies,
massfractions=y0s, velocity=velocity)
def boundary_solution(discr, btag, gas_model, state_minus, **kwargs):
actx = state_minus.array_context
bnd_discr = discr.discr_from_dd(btag)
nodes = thaw(bnd_discr.nodes(), actx)
return make_fluid_state(initializer(x_vec=nodes, eos=gas_model.eos,
**kwargs), gas_model,
temperature_seed=state_minus.temperature)
boundaries = {
BTAG_ALL: PrescribedFluidBoundary(boundary_state_func=boundary_solution)
}
if rst_filename:
current_t = restart_data["t"]
current_step = restart_data["step"]
current_cv = restart_data["cv"]
tseed = restart_data["temperature_seed"]
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
else:
# Set the current state from time 0
current_cv = initializer(x_vec=nodes, eos=eos)
tseed = 300.0
current_state = make_fluid_state(current_cv, gas_model, temperature_seed=tseed)
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim, order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt, t_final=t_final, nstatus=nstatus,
nviz=nviz, cfl=current_cfl,
constant_cfl=constant_cfl, initname=initname,
eosname=eosname, casename=casename)
if rank == 0:
logger.info(init_message)
def my_write_status(component_errors, dv=None):
from mirgecom.simutil import allsync
status_msg = (
"------- errors="
+ ", ".join("%.3g" % en for en in component_errors))
if ((dv is not None) and (not log_dependent)):
temp = dv.temperature
press = dv.pressure
from grudge.op import nodal_min_loc, nodal_max_loc
tmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", temp)),
comm=comm, op=MPI.MIN)
tmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", temp)),
comm=comm, op=MPI.MAX)
pmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", press)),
comm=comm, op=MPI.MIN)
pmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", press)),
comm=comm, op=MPI.MAX)
dv_status_msg = f"\nP({pmin}, {pmax}), T({tmin}, {tmax})"
status_msg = status_msg + dv_status_msg
if rank == 0:
logger.info(status_msg)
if rank == 0:
logger.info(status_msg)
def my_write_viz(step, t, state, dv, exact=None, resid=None):
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
if resid is None:
resid = state - exact
viz_fields = [("cv", state), ("dv", dv)]
from mirgecom.simutil import write_visfile
write_visfile(discr, viz_fields, visualizer, vizname=casename,
step=step, t=t, overwrite=True, vis_timer=vis_timer)
def my_write_restart(step, t, state, tseed):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname != rst_filename:
rst_data = {
"local_mesh": local_mesh,
"cv": state,
"temperature_seed": tseed,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": nparts
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(dv, component_errors):
health_error = False
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", dv.pressure) \
or check_range_local(discr, "vol", dv.pressure, 1e5, 1.1e5):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
exittol = .09
if max(component_errors) > exittol:
health_error = True
if rank == 0:
logger.info("Solution diverged from exact soln.")
return health_error
def my_pre_step(step, t, dt, state):
cv, tseed = state
fluid_state = make_fluid_state(cv, gas_model, temperature_seed=tseed)
dv = fluid_state.dv
try:
exact = None
component_errors = None
if logmgr:
logmgr.tick_before()
from mirgecom.simutil import check_step
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
do_status = check_step(step=step, interval=nstatus)
if do_health:
exact = initializer(x_vec=nodes, eos=eos, time=t)
from mirgecom.simutil import compare_fluid_solutions
component_errors = compare_fluid_solutions(discr, cv, exact)
health_errors = global_reduce(
my_health_check(dv, component_errors), op="lor")
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
if do_restart:
my_write_restart(step=step, t=t, state=cv, tseed=tseed)
if do_viz:
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
resid = state - exact
my_write_viz(step=step, t=t, state=cv, dv=dv, exact=exact,
resid=resid)
if do_status:
if component_errors is None:
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
from mirgecom.simutil import compare_fluid_solutions
component_errors = compare_fluid_solutions(discr, cv, exact)
my_write_status(component_errors, dv=dv)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
my_write_viz(step=step, t=t, state=cv, dv=dv)
my_write_restart(step=step, t=t, state=cv, tseed=tseed)
raise
dt = get_sim_timestep(discr, fluid_state, t, dt, current_cfl, t_final,
constant_cfl)
return state, dt
def my_post_step(step, t, dt, state):
cv, tseed = state
fluid_state = make_fluid_state(cv, gas_model, temperature_seed=tseed)
tseed = fluid_state.temperature
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, cv, eos)
logmgr.tick_after()
return make_obj_array([fluid_state.cv, tseed]), dt
def my_rhs(t, state):
cv, tseed = state
fluid_state = make_fluid_state(cv, gas_model, temperature_seed=tseed)
return make_obj_array(
[euler_operator(discr, state=fluid_state, time=t,
boundaries=boundaries, gas_model=gas_model),
0*tseed])
current_dt = get_sim_timestep(discr, current_state, current_t, current_dt,
current_cfl, t_final, constant_cfl)
current_step, current_t, advanced_state = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step,
post_step_callback=my_post_step, dt=current_dt,
state=make_obj_array([current_state.cv,
current_state.temperature]),
t=current_t, t_final=t_final, eos=eos, dim=dim)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
current_cv, tseed = advanced_state
current_state = make_fluid_state(current_cv, gas_model, temperature_seed=tseed)
final_dv = current_state.dv
final_exact = initializer(x_vec=nodes, eos=eos, time=current_t)
final_resid = current_state.cv - final_exact
my_write_viz(step=current_step, t=current_t, state=current_cv, dv=final_dv,
exact=final_exact, resid=final_resid)
my_write_restart(step=current_step, t=current_t, state=current_state.cv,
tseed=tseed)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
assert np.abs(current_t - t_final) < finish_tol
def main(use_profiling=False, use_logmgr=False, lazy_eval: bool = False):
"""Drive the example."""
cl_ctx = cl.create_some_context()
logmgr = initialize_logmgr(use_logmgr, filename="wave.sqlite", mode="wu")
if use_profiling:
if lazy_eval:
raise RuntimeError("Cannot run lazy with profiling.")
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLProfilingArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
else:
queue = cl.CommandQueue(cl_ctx)
if lazy_eval:
actx = PytatoPyOpenCLArrayContext(queue)
else:
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
dim = 2
nel_1d = 16
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim,
b=(0.5, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
order = 3
discr = EagerDGDiscretization(actx, mesh, order=order)
current_cfl = 0.485
wave_speed = 1.0
from grudge.dt_utils import characteristic_lengthscales
nodal_dt = characteristic_lengthscales(actx, discr) / wave_speed
from grudge.op import nodal_min
dt = actx.to_numpy(current_cfl * nodal_min(discr, "vol", nodal_dt))[()]
print("%d elements" % mesh.nelements)
fields = flat_obj_array(bump(actx, discr),
[discr.zeros(actx) for i in range(discr.dim)])
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr.add_watches(["step.max", "t_step.max", "t_log.max"])
try:
logmgr.add_watches(
["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["multiply_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
vis = make_visualizer(discr)
def rhs(t, w):
return wave_operator(discr, c=wave_speed, w=w)
compiled_rhs = actx.compile(rhs)
t = 0
t_final = 1
istep = 0
while t < t_final:
if logmgr:
logmgr.tick_before()
fields = thaw(freeze(fields, actx), actx)
fields = rk4_step(fields, t, dt, compiled_rhs)
if istep % 10 == 0:
if use_profiling:
print(actx.tabulate_profiling_data())
print(istep, t, actx.to_numpy(discr.norm(fields[0], np.inf)))
vis.write_vtk_file("fld-wave-%04d.vtu" % istep, [
("u", fields[0]),
("v", fields[1:]),
],
overwrite=True)
t += dt
istep += 1
if logmgr:
set_dt(logmgr, dt)
logmgr.tick_after()
def main(ctx_factory=cl.create_some_context,
use_logmgr=True,
use_leap=False,
use_profiling=False,
casename=None,
rst_filename=None,
actx_class=PyOpenCLArrayContext):
"""Drive the example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
num_parts = comm.Get_size()
from mirgecom.simutil import global_reduce as _global_reduce
global_reduce = partial(_global_reduce, comm=comm)
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite",
mode="wu",
mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
# timestepping control
current_step = 0
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
t_final = 0.01
current_cfl = 1.0
current_dt = .001
current_t = 0
constant_cfl = False
# some i/o frequencies
nrestart = 10
nstatus = 1
nviz = 10
nhealth = 10
dim = 2
if dim != 2:
raise ValueError("This example must be run with dim = 2.")
rst_path = "restart_data/"
rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl")
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["num_parts"] == num_parts
else: # generate the grid from scratch
nel_1d = 16
box_ll = -5.0
box_ur = 5.0
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
order = 3
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(discr.nodes(), actx)
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
try:
logmgr.add_watches(
["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["multiply_time.max"])
# soln setup and init
eos = IdealSingleGas()
vel = np.zeros(shape=(dim, ))
orig = np.zeros(shape=(dim, ))
vel[:dim] = 1.0
initializer = Vortex2D(center=orig, velocity=vel)
gas_model = GasModel(eos=eos)
def boundary_solution(discr, btag, gas_model, state_minus, **kwargs):
actx = state_minus.array_context
bnd_discr = discr.discr_from_dd(btag)
nodes = thaw(bnd_discr.nodes(), actx)
return make_fluid_state(
initializer(x_vec=nodes, eos=gas_model.eos, **kwargs), gas_model)
boundaries = {
BTAG_ALL:
PrescribedFluidBoundary(boundary_state_func=boundary_solution)
}
if rst_filename:
current_t = restart_data["t"]
current_step = restart_data["step"]
current_cv = restart_data["cv"]
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
else:
# Set the current state from time 0
current_cv = initializer(nodes)
current_state = make_fluid_state(current_cv, gas_model)
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
def my_write_status(state, component_errors, cfl=None):
if cfl is None:
if constant_cfl:
cfl = current_cfl
else:
from grudge.op import nodal_max
from mirgecom.inviscid import get_inviscid_cfl
cfl = actx.to_numpy(
nodal_max(discr, "vol",
get_inviscid_cfl(discr, state, current_dt)))[()]
if rank == 0:
logger.info(f"------ {cfl=}\n"
"------- errors=" +
", ".join("%.3g" % en for en in component_errors))
def my_write_viz(step, t, state, dv=None, exact=None, resid=None):
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
if resid is None:
resid = state - exact
viz_fields = [("cv", state), ("dv", dv), ("exact", exact),
("residual", resid)]
from mirgecom.simutil import write_visfile
write_visfile(discr,
viz_fields,
visualizer,
vizname=casename,
step=step,
t=t,
overwrite=True,
vis_timer=vis_timer)
def my_write_restart(step, t, state):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname != rst_filename:
rst_data = {
"local_mesh": local_mesh,
"cv": state,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": num_parts
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(pressure, component_errors):
health_error = False
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", pressure) \
or check_range_local(discr, "vol", pressure, .2, 1.02):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
exittol = .1
if max(component_errors) > exittol:
health_error = True
if rank == 0:
logger.info("Solution diverged from exact soln.")
return health_error
def my_pre_step(step, t, dt, state):
fluid_state = make_fluid_state(state, gas_model)
cv = fluid_state.cv
dv = fluid_state.dv
try:
exact = None
component_errors = None
if logmgr:
logmgr.tick_before()
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
do_status = check_step(step=step, interval=nstatus)
if do_health:
exact = initializer(x_vec=nodes, eos=eos, time=t)
from mirgecom.simutil import compare_fluid_solutions
component_errors = compare_fluid_solutions(discr, cv, exact)
health_errors = global_reduce(my_health_check(
dv.pressure, component_errors),
op="lor")
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
if do_restart:
my_write_restart(step=step, t=t, state=cv)
if do_status:
if component_errors is None:
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
from mirgecom.simutil import compare_fluid_solutions
component_errors = compare_fluid_solutions(
discr, cv, exact)
my_write_status(fluid_state, component_errors)
if do_viz:
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
resid = state - exact
my_write_viz(step=step,
t=t,
state=cv,
dv=dv,
exact=exact,
resid=resid)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
my_write_viz(step=step, t=t, state=cv)
my_write_restart(step=step, t=t, state=cv)
raise
dt = get_sim_timestep(discr, fluid_state, t, dt, current_cfl, t_final,
constant_cfl)
return state, dt
def my_post_step(step, t, dt, state):
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, state, eos)
logmgr.tick_after()
return state, dt
def my_rhs(t, state):
fluid_state = make_fluid_state(state, gas_model)
return euler_operator(discr,
state=fluid_state,
time=t,
boundaries=boundaries,
gas_model=gas_model)
current_dt = get_sim_timestep(discr, current_state, current_t, current_dt,
current_cfl, t_final, constant_cfl)
current_step, current_t, current_cv = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step,
post_step_callback=my_post_step, dt=current_dt,
state=current_state.cv, t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
current_state = make_fluid_state(current_cv, gas_model)
final_dv = current_state.dv
final_exact = initializer(x_vec=nodes, eos=eos, time=current_t)
final_resid = current_state.cv - final_exact
my_write_viz(step=current_step,
t=current_t,
state=current_state.cv,
dv=final_dv,
exact=final_exact,
resid=final_resid)
my_write_restart(step=current_step, t=current_t, state=current_state.cv)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
assert np.abs(current_t - t_final) < finish_tol
def main(ctx_factory=cl.create_some_context,
use_logmgr=True,
use_leap=False,
use_overintegration=False,
use_profiling=False,
casename=None,
rst_filename=None,
actx_class=PyOpenCLArrayContext,
log_dependent=True):
"""Drive example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nproc = comm.Get_size()
from mirgecom.simutil import global_reduce as _global_reduce
global_reduce = partial(_global_reduce, comm=comm)
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite",
mode="wu",
mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
# Some discretization parameters
dim = 2
nel_1d = 8
order = 1
# {{{ Time stepping control
# This example runs only 3 steps by default (to keep CI ~short)
# With the mixture defined below, equilibrium is achieved at ~40ms
# To run to equilibrium, set t_final >= 40ms.
# Time stepper selection
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
# Time loop control parameters
current_step = 0
t_final = 1e-8
current_cfl = 1.0
current_dt = 1e-9
current_t = 0
constant_cfl = False
# i.o frequencies
nstatus = 1
nviz = 5
nhealth = 1
nrestart = 5
# }}} Time stepping control
debug = False
rst_path = "restart_data/"
rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl")
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["num_parts"] == nproc
rst_time = restart_data["t"]
rst_step = restart_data["step"]
rst_order = restart_data["order"]
else: # generate the grid from scratch
from meshmode.mesh.generation import generate_regular_rect_mesh
box_ll = -0.005
box_ur = 0.005
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD
from meshmode.discretization.poly_element import \
default_simplex_group_factory, QuadratureSimplexGroupFactory
discr = EagerDGDiscretization(
actx,
local_mesh,
discr_tag_to_group_factory={
DISCR_TAG_BASE:
default_simplex_group_factory(base_dim=local_mesh.dim,
order=order),
DISCR_TAG_QUAD:
QuadratureSimplexGroupFactory(2 * order + 1)
},
mpi_communicator=comm)
nodes = thaw(discr.nodes(), actx)
ones = discr.zeros(actx) + 1.0
if use_overintegration:
quadrature_tag = DISCR_TAG_QUAD
else:
quadrature_tag = None
ones = discr.zeros(actx) + 1.0
vis_timer = None
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("t_step.max",
"------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")])
if log_dependent:
logmgr_add_many_discretization_quantities(
logmgr, discr, dim, extract_vars_for_logging,
units_for_logging)
logmgr.add_watches([
("min_pressure",
"\n------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("min_temperature",
"------- T (min, max) (K) = ({value:7g}, "),
("max_temperature", "{value:7g})\n")
])
# {{{ Set up initial state using Cantera
# Use Cantera for initialization
# -- Pick up a CTI for the thermochemistry config
# --- Note: Users may add their own CTI file by dropping it into
# --- mirgecom/mechanisms alongside the other CTI files.
from mirgecom.mechanisms import get_mechanism_cti
mech_cti = get_mechanism_cti("uiuc")
cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti)
nspecies = cantera_soln.n_species
# Initial temperature, pressure, and mixutre mole fractions are needed to
# set up the initial state in Cantera.
temperature_seed = 1500.0 # Initial temperature hot enough to burn
# Parameters for calculating the amounts of fuel, oxidizer, and inert species
equiv_ratio = 1.0
ox_di_ratio = 0.21
stoich_ratio = 3.0
# Grab the array indices for the specific species, ethylene, oxygen, and nitrogen
i_fu = cantera_soln.species_index("C2H4")
i_ox = cantera_soln.species_index("O2")
i_di = cantera_soln.species_index("N2")
x = np.zeros(nspecies)
# Set the species mole fractions according to our desired fuel/air mixture
x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio +
ox_di_ratio * equiv_ratio)
x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio
x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio
# Uncomment next line to make pylint fail when it can't find cantera.one_atm
one_atm = cantera.one_atm # pylint: disable=no-member
# one_atm = 101325.0
# Let the user know about how Cantera is being initilized
print(f"Input state (T,P,X) = ({temperature_seed}, {one_atm}, {x}")
# Set Cantera internal gas temperature, pressure, and mole fractios
cantera_soln.TPX = temperature_seed, one_atm, x
# Pull temperature, total density, mass fractions, and pressure from Cantera
# We need total density, and mass fractions to initialize the fluid/gas state.
can_t, can_rho, can_y = cantera_soln.TDY
can_p = cantera_soln.P
# *can_t*, *can_p* should not differ (significantly) from user's initial data,
# but we want to ensure that we use exactly the same starting point as Cantera,
# so we use Cantera's version of these data.
# }}}
# {{{ Create Pyrometheus thermochemistry object & EOS
# Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and
# generates a set of methods to calculate chemothermomechanical properties and
# states for this particular mechanism.
from mirgecom.thermochemistry import make_pyrometheus_mechanism_class
pyro_mechanism = make_pyrometheus_mechanism_class(cantera_soln)(actx.np)
eos = PyrometheusMixture(pyro_mechanism,
temperature_guess=temperature_seed)
gas_model = GasModel(eos=eos)
from pytools.obj_array import make_obj_array
def get_temperature_update(cv, temperature):
y = cv.species_mass_fractions
e = gas_model.eos.internal_energy(cv) / cv.mass
return pyro_mechanism.get_temperature_update_energy(e, temperature, y)
from mirgecom.gas_model import make_fluid_state
def get_fluid_state(cv, tseed):
return make_fluid_state(cv=cv,
gas_model=gas_model,
temperature_seed=tseed)
compute_temperature_update = actx.compile(get_temperature_update)
construct_fluid_state = actx.compile(get_fluid_state)
# }}}
# {{{ MIRGE-Com state initialization
# Initialize the fluid/gas state with Cantera-consistent data:
# (density, pressure, temperature, mass_fractions)
print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}")
velocity = np.zeros(shape=(dim, ))
initializer = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=can_p,
temperature=can_t,
massfractions=can_y,
velocity=velocity)
my_boundary = AdiabaticSlipBoundary()
boundaries = {BTAG_ALL: my_boundary}
if rst_filename:
current_step = rst_step
current_t = rst_time
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
if order == rst_order:
current_cv = restart_data["cv"]
temperature_seed = restart_data["temperature_seed"]
else:
rst_cv = restart_data["cv"]
old_discr = EagerDGDiscretization(actx,
local_mesh,
order=rst_order,
mpi_communicator=comm)
from meshmode.discretization.connection import make_same_mesh_connection
connection = make_same_mesh_connection(
actx, discr.discr_from_dd("vol"),
old_discr.discr_from_dd("vol"))
current_cv = connection(rst_cv)
temperature_seed = connection(restart_data["temperature_seed"])
else:
# Set the current state from time 0
current_cv = initializer(eos=gas_model.eos, x_vec=nodes)
temperature_seed = temperature_seed * ones
# The temperature_seed going into this function is:
# - At time 0: the initial temperature input data (maybe from Cantera)
# - On restart: the restarted temperature seed from restart file (saving
# the *seed* allows restarts to be deterministic
current_fluid_state = construct_fluid_state(current_cv, temperature_seed)
current_dv = current_fluid_state.dv
temperature_seed = current_dv.temperature
# Inspection at physics debugging time
if debug:
print("Initial MIRGE-Com state:")
print(f"Initial DV pressure: {current_fluid_state.pressure}")
print(f"Initial DV temperature: {current_fluid_state.temperature}")
# }}}
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = gas_model.eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
# Cantera equilibrate calculates the expected end state @ chemical equilibrium
# i.e. the expected state after all reactions
cantera_soln.equilibrate("UV")
eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY
eq_pressure = cantera_soln.P
# Report the expected final state to the user
if rank == 0:
logger.info(init_message)
logger.info(f"Expected equilibrium state:"
f" {eq_pressure=}, {eq_temperature=},"
f" {eq_density=}, {eq_mass_fractions=}")
def my_write_status(dt, cfl, dv=None):
status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}"
if ((dv is not None) and (not log_dependent)):
temp = dv.temperature
press = dv.pressure
from grudge.op import nodal_min_loc, nodal_max_loc
tmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", temp)),
comm=comm,
op=MPI.MIN)
tmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", temp)),
comm=comm,
op=MPI.MAX)
pmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", press)),
comm=comm,
op=MPI.MIN)
pmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", press)),
comm=comm,
op=MPI.MAX)
dv_status_msg = f"\nP({pmin}, {pmax}), T({tmin}, {tmax})"
status_msg = status_msg + dv_status_msg
if rank == 0:
logger.info(status_msg)
def my_write_viz(step, t, dt, state, ts_field, dv, production_rates, cfl):
viz_fields = [("cv", state), ("dv", dv),
("production_rates", production_rates),
("dt" if constant_cfl else "cfl", ts_field)]
write_visfile(discr,
viz_fields,
visualizer,
vizname=casename,
step=step,
t=t,
overwrite=True,
vis_timer=vis_timer)
def my_write_restart(step, t, state, temperature_seed):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname == rst_filename:
if rank == 0:
logger.info("Skipping overwrite of restart file.")
else:
rst_data = {
"local_mesh": local_mesh,
"cv": state.cv,
"temperature_seed": temperature_seed,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": nproc
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(cv, dv):
import grudge.op as op
health_error = False
pressure = dv.pressure
temperature = dv.temperature
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", pressure):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
if check_range_local(discr, "vol", pressure, 1e5, 2.6e5):
health_error = True
logger.info(f"{rank=}: Pressure range violation.")
if check_naninf_local(discr, "vol", temperature):
health_error = True
logger.info(f"{rank=}: Invalid temperature data found.")
if check_range_local(discr, "vol", temperature, 1.498e3, 1.6e3):
health_error = True
logger.info(f"{rank=}: Temperature range violation.")
# This check is the temperature convergence check
# The current *temperature* is what Pyrometheus gets
# after a fixed number of Newton iterations, *n_iter*.
# Calling `compute_temperature` here with *temperature*
# input as the guess returns the calculated gas temperature after
# yet another *n_iter*.
# The difference between those two temperatures is the
# temperature residual, which can be used as an indicator of
# convergence in Pyrometheus `get_temperature`.
# Note: The local max jig below works around a very long compile
# in lazy mode.
temp_resid = compute_temperature_update(cv, temperature) / temperature
temp_err = (actx.to_numpy(op.nodal_max_loc(discr, "vol", temp_resid)))
if temp_err > 1e-8:
health_error = True
logger.info(
f"{rank=}: Temperature is not converged {temp_resid=}.")
return health_error
from mirgecom.inviscid import get_inviscid_timestep
def get_dt(state):
return get_inviscid_timestep(discr, state=state)
compute_dt = actx.compile(get_dt)
from mirgecom.inviscid import get_inviscid_cfl
def get_cfl(state, dt):
return get_inviscid_cfl(discr, dt=dt, state=state)
compute_cfl = actx.compile(get_cfl)
def get_production_rates(cv, temperature):
return eos.get_production_rates(cv, temperature)
compute_production_rates = actx.compile(get_production_rates)
def my_get_timestep(t, dt, state):
# richer interface to calculate {dt,cfl} returns node-local estimates
t_remaining = max(0, t_final - t)
if constant_cfl:
ts_field = current_cfl * compute_dt(state)
from grudge.op import nodal_min_loc
dt = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", ts_field)),
comm=comm,
op=MPI.MIN)
cfl = current_cfl
else:
ts_field = compute_cfl(state, current_dt)
from grudge.op import nodal_max_loc
cfl = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", ts_field)),
comm=comm,
op=MPI.MAX)
return ts_field, cfl, min(t_remaining, dt)
def my_pre_step(step, t, dt, state):
cv, tseed = state
fluid_state = construct_fluid_state(cv, tseed)
dv = fluid_state.dv
try:
if logmgr:
logmgr.tick_before()
from mirgecom.simutil import check_step
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
do_status = check_step(step=step, interval=nstatus)
if do_health:
health_errors = global_reduce(my_health_check(cv, dv),
op="lor")
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=fluid_state)
if do_status:
my_write_status(dt=dt, cfl=cfl, dv=dv)
if do_restart:
my_write_restart(step=step,
t=t,
state=fluid_state,
temperature_seed=tseed)
if do_viz:
production_rates = compute_production_rates(
fluid_state.cv, fluid_state.temperature)
my_write_viz(step=step,
t=t,
dt=dt,
state=cv,
dv=dv,
production_rates=production_rates,
ts_field=ts_field,
cfl=cfl)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
# my_write_viz(step=step, t=t, dt=dt, state=cv)
# my_write_restart(step=step, t=t, state=fluid_state)
raise
return state, dt
def my_post_step(step, t, dt, state):
cv, tseed = state
fluid_state = construct_fluid_state(cv, tseed)
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, cv, gas_model.eos)
logmgr.tick_after()
return make_obj_array([cv, fluid_state.temperature]), dt
def my_rhs(t, state):
cv, tseed = state
from mirgecom.gas_model import make_fluid_state
fluid_state = make_fluid_state(cv=cv,
gas_model=gas_model,
temperature_seed=tseed)
return make_obj_array([
euler_operator(discr,
state=fluid_state,
time=t,
boundaries=boundaries,
gas_model=gas_model,
quadrature_tag=quadrature_tag) +
eos.get_species_source_terms(cv, fluid_state.temperature),
0 * tseed
])
current_dt = get_sim_timestep(discr, current_fluid_state, current_t,
current_dt, current_cfl, t_final,
constant_cfl)
current_step, current_t, current_state = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step,
post_step_callback=my_post_step, dt=current_dt,
state=make_obj_array([current_cv, temperature_seed]),
t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
final_cv, tseed = current_state
final_fluid_state = construct_fluid_state(final_cv, tseed)
final_dv = final_fluid_state.dv
final_dm = compute_production_rates(final_cv, final_dv.temperature)
ts_field, cfl, dt = my_get_timestep(t=current_t,
dt=current_dt,
state=final_fluid_state)
my_write_viz(step=current_step,
t=current_t,
dt=dt,
state=final_cv,
dv=final_dv,
production_rates=final_dm,
ts_field=ts_field,
cfl=cfl)
my_write_status(dt=dt, cfl=cfl, dv=final_dv)
my_write_restart(step=current_step,
t=current_t,
state=final_fluid_state,
temperature_seed=tseed)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
assert np.abs(current_t - t_final) < finish_tol
def main(ctx_factory=cl.create_some_context, use_logmgr=True,
use_leap=False, use_profiling=False, casename=None,
rst_filename=None, actx_class=PyOpenCLArrayContext):
"""Drive example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nparts = comm.Get_size()
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
# timestepping control
current_step = 0
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
t_final = 0.005
current_cfl = 1.0
current_dt = .001
current_t = 0
constant_cfl = False
# some i/o frequencies
nstatus = 1
nrestart = 5
nviz = 1
nhealth = 1
dim = 2
rst_path = "restart_data/"
rst_pattern = (
rst_path + "{cname}-{step:04d}-{rank:04d}.pkl"
)
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["nparts"] == nparts
else: # generate the grid from scratch
box_ll = -5.0
box_ur = 5.0
nel_1d = 16
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim,
b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim)
local_mesh, global_nelements = generate_and_distribute_mesh(comm,
generate_mesh)
local_nelements = local_mesh.nelements
order = 3
discr = EagerDGDiscretization(
actx, local_mesh, order=order, mpi_communicator=comm
)
nodes = thaw(actx, discr.nodes())
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging, units_for_logging)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
# soln setup and init
nspecies = 4
centers = make_obj_array([np.zeros(shape=(dim,)) for i in range(nspecies)])
spec_y0s = np.ones(shape=(nspecies,))
spec_amplitudes = np.ones(shape=(nspecies,))
eos = IdealSingleGas()
velocity = np.ones(shape=(dim,))
initializer = MulticomponentLump(dim=dim, nspecies=nspecies,
spec_centers=centers, velocity=velocity,
spec_y0s=spec_y0s,
spec_amplitudes=spec_amplitudes)
boundaries = {
BTAG_ALL: PrescribedInviscidBoundary(fluid_solution_func=initializer)
}
if rst_filename:
current_t = restart_data["t"]
current_step = restart_data["step"]
current_state = restart_data["state"]
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
else:
# Set the current state from time 0
current_state = initializer(nodes)
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim, order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt, t_final=t_final, nstatus=nstatus,
nviz=nviz, cfl=current_cfl,
constant_cfl=constant_cfl, initname=initname,
eosname=eosname, casename=casename)
if rank == 0:
logger.info(init_message)
def my_write_status(component_errors):
if rank == 0:
logger.info(
"------- errors="
+ ", ".join("%.3g" % en for en in component_errors))
def my_write_viz(step, t, state, dv=None, exact=None, resid=None):
if dv is None:
dv = eos.dependent_vars(state)
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
if resid is None:
resid = state - exact
viz_fields = [("cv", state),
("dv", dv),
("exact", exact),
("resid", resid)]
from mirgecom.simutil import write_visfile
write_visfile(discr, viz_fields, visualizer, vizname=casename,
step=step, t=t, overwrite=True, vis_timer=vis_timer)
def my_write_restart(step, t, state):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname != rst_filename:
rst_data = {
"local_mesh": local_mesh,
"state": state,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": nparts
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(pressure, component_errors):
health_error = False
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", pressure) \
or check_range_local(discr, "vol", pressure, .99999999, 1.00000001):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
exittol = .09
if max(component_errors) > exittol:
health_error = True
if rank == 0:
logger.info("Solution diverged from exact soln.")
return health_error
def my_pre_step(step, t, dt, state):
try:
dv = None
exact = None
component_errors = None
if logmgr:
logmgr.tick_before()
from mirgecom.simutil import check_step
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
do_status = check_step(step=step, interval=nstatus)
if do_health:
dv = eos.dependent_vars(state)
exact = initializer(x_vec=nodes, eos=eos, time=t)
from mirgecom.simutil import compare_fluid_solutions
component_errors = compare_fluid_solutions(discr, state, exact)
from mirgecom.simutil import allsync
health_errors = allsync(
my_health_check(dv.pressure, component_errors),
comm, op=MPI.LOR
)
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
if do_restart:
my_write_restart(step=step, t=t, state=state)
if do_viz:
if dv is None:
dv = eos.dependent_vars(state)
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
resid = state - exact
my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact,
resid=resid)
if do_status:
if component_errors is None:
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
from mirgecom.simutil import compare_fluid_solutions
component_errors = compare_fluid_solutions(discr, state, exact)
my_write_status(component_errors)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
my_write_viz(step=step, t=t, state=state)
my_write_restart(step=step, t=t, state=state)
raise
dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final,
constant_cfl)
return state, dt
def my_post_step(step, t, dt, state):
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, state, eos)
logmgr.tick_after()
return state, dt
def my_rhs(t, state):
return euler_operator(discr, cv=state, time=t,
boundaries=boundaries, eos=eos)
current_dt = get_sim_timestep(discr, current_state, current_t, current_dt,
current_cfl, eos, t_final, constant_cfl)
current_step, current_t, current_state = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step, dt=current_dt,
post_step_callback=my_post_step,
state=current_state, t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
final_dv = eos.dependent_vars(current_state)
final_exact = initializer(x_vec=nodes, eos=eos, time=current_t)
final_resid = current_state - final_exact
my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv,
exact=final_exact, resid=final_resid)
my_write_restart(step=current_step, t=current_t, state=current_state)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
time_err = current_t - t_final
if np.abs(time_err) > finish_tol:
raise ValueError(f"Simulation did not finish at expected time {time_err=}.")
def main(ctx_factory=cl.create_some_context,
casename="flame1d",
user_input_file=None,
snapshot_pattern="{casename}-{step:06d}-{rank:04d}.pkl",
restart_step=None,
restart_name=None,
use_profiling=False,
use_logmgr=False,
use_lazy_eval=False):
"""Drive the 1D Flame example."""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = 0
rank = comm.Get_rank()
nparts = comm.Get_size()
if restart_name is None:
restart_name = casename
"""logging and profiling"""
logmgr = initialize_logmgr(use_logmgr,
filename=(f"{casename}.sqlite"),
mode="wo",
mpi_comm=comm)
cl_ctx = ctx_factory()
if use_profiling:
if use_lazy_eval:
raise RuntimeError("Cannot run lazy with profiling.")
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLProfilingArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
if use_lazy_eval:
actx = PytatoArrayContext(queue)
else:
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
# default input values that will be read from input (if they exist)
nviz = 100
nrestart = 100
nhealth = 100
nstatus = 1
current_dt = 1e-9
t_final = 1.e-3
order = 1
integrator = "rk4"
if user_input_file:
if rank == 0:
with open(user_input_file) as f:
input_data = yaml.load(f, Loader=yaml.FullLoader)
else:
input_data = None
input_data = comm.bcast(input_data, root=0)
#print(input_data)
try:
nviz = int(input_data["nviz"])
except KeyError:
pass
try:
nrestart = int(input_data["nrestart"])
except KeyError:
pass
try:
nhealth = int(input_data["nhealth"])
except KeyError:
pass
try:
nstatus = int(input_data["nstatus"])
except KeyError:
pass
try:
current_dt = float(input_data["current_dt"])
except KeyError:
pass
try:
t_final = float(input_data["t_final"])
except KeyError:
pass
try:
order = int(input_data["order"])
except KeyError:
pass
try:
integrator = input_data["integrator"]
except KeyError:
pass
# param sanity check
allowed_integrators = ["rk4", "euler", "lsrk54", "lsrk144"]
if (integrator not in allowed_integrators):
error_message = "Invalid time integrator: {}".format(integrator)
raise RuntimeError(error_message)
timestepper = rk4_step
if integrator == "euler":
timestepper = euler_step
if integrator == "lsrk54":
timestepper = lsrk54_step
if integrator == "lsrk144":
timestepper = lsrk144_step
if (rank == 0):
print(f'#### Simluation control data: ####')
print(f'\tnviz = {nviz}')
print(f'\tnrestart = {nrestart}')
print(f'\tnhealth = {nhealth}')
print(f'\tnstatus = {nstatus}')
print(f'\tcurrent_dt = {current_dt}')
print(f'\tt_final = {t_final}')
print(f'\torder = {order}')
print(f"\tTime integration {integrator}")
print(f'#### Simluation control data: ####')
restart_path = 'restart_data/'
viz_path = 'viz_data/'
#if(rank == 0):
#if not os.path.exists(restart_path):
#os.makedirs(restart_path)
#if not os.path.exists(viz_path):
#os.makedirs(viz_path)
dim = 2
exittol = .09
current_cfl = 1.0
current_t = 0
constant_cfl = False
checkpoint_t = current_t
current_step = 0
vel_burned = np.zeros(shape=(dim, ))
vel_unburned = np.zeros(shape=(dim, ))
# {{{ Set up initial state using Cantera
# Use Cantera for initialization
# -- Pick up a CTI for the thermochemistry config
# --- Note: Users may add their own CTI file by dropping it into
# --- mirgecom/mechanisms alongside the other CTI files.
fuel = "H2"
allowed_fuels = ["H2", "C2H4"]
if (fuel not in allowed_fuels):
error_message = "Invalid fuel selection: {}".format(fuel)
raise RuntimeError(error_message)
if rank == 0:
print(f"Fuel: {fuel}")
from mirgecom.mechanisms import get_mechanism_cti
if fuel == "C2H4":
mech_cti = get_mechanism_cti("uiuc")
elif fuel == "H2":
mech_cti = get_mechanism_cti("sanDiego")
cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti)
nspecies = cantera_soln.n_species
# Initial temperature, pressure, and mixutre mole fractions are needed to
# set up the initial state in Cantera.
temp_unburned = 300.0
temp_ignition = 1500.0
# Parameters for calculating the amounts of fuel, oxidizer, and inert species
if fuel == "C2H4":
stoich_ratio = 3.0
if fuel == "H2":
stoich_ratio = 0.5
equiv_ratio = 1.0
ox_di_ratio = 0.21
# Grab the array indices for the specific species, ethylene, oxygen, and nitrogen
i_fu = cantera_soln.species_index(fuel)
i_ox = cantera_soln.species_index("O2")
i_di = cantera_soln.species_index("N2")
x = np.zeros(nspecies)
# Set the species mole fractions according to our desired fuel/air mixture
x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio +
ox_di_ratio * equiv_ratio)
x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio
x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio
# Uncomment next line to make pylint fail when it can't find cantera.one_atm
one_atm = cantera.one_atm # pylint: disable=no-member
# one_atm = 101325.0
pres_unburned = one_atm
# Let the user know about how Cantera is being initilized
print(f"Input state (T,P,X) = ({temp_unburned}, {pres_unburned}, {x}")
# Set Cantera internal gas temperature, pressure, and mole fractios
cantera_soln.TPX = temp_unburned, pres_unburned, x
# Pull temperature, total density, mass fractions, and pressure from Cantera
# We need total density, and mass fractions to initialize the fluid/gas state.
y_unburned = np.zeros(nspecies)
can_t, rho_unburned, y_unburned = cantera_soln.TDY
can_p = cantera_soln.P
# *can_t*, *can_p* should not differ (significantly) from user's initial data,
# but we want to ensure that we use exactly the same starting point as Cantera,
# so we use Cantera's version of these data.
# now find the conditions for the burned gas
cantera_soln.equilibrate('TP')
temp_burned, rho_burned, y_burned = cantera_soln.TDY
pres_burned = cantera_soln.P
pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np)
kappa = 1.6e-5 # Pr = mu*rho/alpha = 0.75
mu = 1.e-5
species_diffusivity = 1.e-5 * np.ones(nspecies)
transport_model = SimpleTransport(viscosity=mu,
thermal_conductivity=kappa,
species_diffusivity=species_diffusivity)
eos = PyrometheusMixture(pyrometheus_mechanism,
temperature_guess=temp_unburned,
transport_model=transport_model)
species_names = pyrometheus_mechanism.species_names
print(f"Pyrometheus mechanism species names {species_names}")
print(
f"Unburned state (T,P,Y) = ({temp_unburned}, {pres_unburned}, {y_unburned}"
)
print(f"Burned state (T,P,Y) = ({temp_burned}, {pres_burned}, {y_burned}")
flame_start_loc = 0.10
flame_speed = 1000
# use the burned conditions with a lower temperature
#bulk_init = PlanarDiscontinuity(dim=dim, disc_location=flame_start_loc, sigma=0.01, nspecies=nspecies,
#temperature_left=temp_ignition, temperature_right=temp_unburned,
#pressure_left=pres_burned, pressure_right=pres_unburned,
#velocity_left=vel_burned, velocity_right=vel_unburned,
#species_mass_left=y_burned, species_mass_right=y_unburned)
bulk_init = PlanarDiscontinuity(dim=dim,
disc_location=flame_start_loc,
sigma=0.0005,
nspecies=nspecies,
temperature_right=temp_ignition,
temperature_left=temp_unburned,
pressure_right=pres_burned,
pressure_left=pres_unburned,
velocity_right=vel_burned,
velocity_left=vel_unburned,
species_mass_right=y_burned,
species_mass_left=y_unburned)
inflow_init = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=pres_burned,
temperature=temp_ignition,
massfractions=y_burned,
velocity=vel_burned)
outflow_init = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=pres_unburned,
temperature=temp_unburned,
massfractions=y_unburned,
velocity=vel_unburned)
def symmetry(nodes, eos, cv=None, **kwargs):
dim = len(nodes)
if cv is not None:
#cv = split_conserved(dim, q)
mass = cv.mass
momentum = cv.momentum
momentum[1] = -1.0 * momentum[1]
ke = .5 * np.dot(cv.momentum, cv.momentum) / cv.mass
energy = cv.energy
species_mass = cv.species_mass
return make_conserved(dim=dim,
mass=mass,
momentum=momentum,
energy=energy,
species_mass=species_mass)
def dummy(nodes, eos, cv=None, **kwargs):
dim = len(nodes)
if cv is not None:
#cv = split_conserved(dim, q)
mass = cv.mass
momentum = cv.momentum
ke = .5 * np.dot(cv.momentum, cv.momentum) / cv.mass
energy = cv.energy
species_mass = cv.species_mass
return make_conserved(dim=dim,
mass=mass,
momentum=momentum,
energy=energy,
species_mass=species_mass)
inflow = PrescribedViscousBoundary(q_func=inflow_init)
outflow = PrescribedViscousBoundary(q_func=outflow_init)
wall_symmetry = PrescribedViscousBoundary(q_func=symmetry)
wall_dummy = PrescribedViscousBoundary(q_func=dummy)
wall = PrescribedViscousBoundary(
) # essentially a "dummy" use the interior solution for the exterior
boundaries = {
DTAG_BOUNDARY("Inflow"): inflow,
DTAG_BOUNDARY("Outflow"): outflow,
#DTAG_BOUNDARY("Wall"): wall_dummy}
DTAG_BOUNDARY("Wall"): wall_symmetry
}
if restart_step is None:
char_len = 0.0001
box_ll = (0.0, 0.0)
box_ur = (0.2, 0.00125)
num_elements = (int((box_ur[0] - box_ll[0]) / char_len),
int((box_ur[1] - box_ll[1]) / char_len))
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh,
a=box_ll,
b=box_ur,
n=num_elements,
boundary_tag_to_face={
"Inflow": ["+x"],
"Outflow": ["-x"],
"Wall": ["+y", "-y"]
})
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
else: # Restart
from mirgecom.restart import read_restart_data
restart_file = restart_path + snapshot_pattern.format(
casename=restart_name, step=restart_step, rank=rank)
restart_data = read_restart_data(actx, restart_file)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert comm.Get_size() == restart_data["num_parts"]
if rank == 0:
logging.info("Making discretization")
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
if restart_step is None:
if rank == 0:
logging.info("Initializing soln.")
# for Discontinuity initial conditions
current_state = bulk_init(x_vec=nodes, eos=eos, time=0.)
# for uniform background initial condition
#current_state = bulk_init(nodes, eos=eos)
else:
current_t = restart_data["t"]
current_step = restart_step
#current_state = make_fluid_restart_state(actx, discr.discr_from_dd("vol"), restart_data["state"])
current_state = restart_data["state"]
vis_timer = None
log_cfl = LogUserQuantity(name="cfl", value=current_cfl)
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
logmgr_set_time(logmgr, current_step, current_t)
logmgr.add_quantity(log_cfl, interval=nstatus)
#logmgr_add_package_versions(logmgr)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s, "),
("cfl.max", "cfl = {value:1.4f}\n"),
("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("min_temperature", "------- T (min, max) (K) = ({value:7g}, "),
("max_temperature", "{value:7g})\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
try:
logmgr.add_watches(["memory_usage.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["pyopencl_array_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
visualizer = make_visualizer(discr)
initname = "flame1d"
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
return (
ns_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) +
eos.get_species_source_terms(cv=state))
def my_checkpoint(step, t, dt, state, force=False):
do_health = force or check_step(step, nhealth) and step > 0
do_viz = force or check_step(step, nviz)
do_restart = force or check_step(step, nrestart)
do_status = force or check_step(step, nstatus)
if do_viz or do_health:
dv = eos.dependent_vars(state)
errors = False
if do_health:
health_message = ""
if check_naninf_local(discr, "vol", dv.pressure):
errors = True
health_message += "Invalid pressure data found.\n"
elif check_range_local(discr,
"vol",
dv.pressure,
min_value=1,
max_value=2.e6):
errors = True
health_message += "Pressure data failed health check.\n"
errors = comm.allreduce(errors, MPI.LOR)
if errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
if health_message:
logger.info(f"{rank=}: {health_message}")
#if check_step(step, nrestart) and step != restart_step and not errors:
if do_restart or errors:
filename = restart_path + snapshot_pattern.format(
step=step, rank=rank, casename=casename)
restart_dictionary = {
"local_mesh": local_mesh,
"order": order,
"state": state,
"t": t,
"step": step,
"global_nelements": global_nelements,
"num_parts": nparts
}
write_restart_file(actx, restart_dictionary, filename, comm)
if do_status or do_viz or errors:
local_cfl = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state)
max_cfl = nodal_max(discr, "vol", local_cfl)
log_cfl.set_quantity(max_cfl)
#if ((check_step(step, nviz) and step != restart_step) or errors):
if do_viz or errors:
def loc_fn(t):
return flame_start_loc + flame_speed * t
#exact_soln = PlanarDiscontinuity(dim=dim, disc_location=loc_fn,
#sigma=0.0000001, nspecies=nspecies,
#temperature_left=temp_ignition, temperature_right=temp_unburned,
#pressure_left=pres_burned, pressure_right=pres_unburned,
#velocity_left=vel_burned, velocity_right=vel_unburned,
#species_mass_left=y_burned, species_mass_right=y_unburned)
reaction_rates = eos.get_production_rates(cv=state)
# conserved quantities
viz_fields = [
#("cv", state),
("CV_rho", state.mass),
("CV_rhoU", state.momentum[0]),
("CV_rhoV", state.momentum[1]),
("CV_rhoE", state.energy)
]
# species mass fractions
viz_fields.extend(
("Y_" + species_names[i], state.species_mass[i] / state.mass)
for i in range(nspecies))
# dependent variables
viz_fields.extend([
("DV", eos.dependent_vars(state)),
#("exact_soln", exact_soln),
("reaction_rates", reaction_rates),
("cfl", local_cfl)
])
write_visfile(discr,
viz_fields,
visualizer,
vizname=viz_path + casename,
step=step,
t=t,
overwrite=True,
vis_timer=vis_timer)
if errors:
raise RuntimeError("Error detected by user checkpoint, exiting.")
if rank == 0:
logging.info("Stepping.")
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t_final=t_final, t=current_t, istep=current_step,
logmgr=logmgr,eos=eos,dim=dim)
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state,
force=True)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
exit()
def main(ctx_factory=cl.create_some_context, use_logmgr=True,
use_leap=False, use_profiling=False, casename=None,
rst_filename=None, actx_class=PyOpenCLArrayContext):
"""Run the example."""
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
from mpi4py import MPI
comm = MPI.COMM_WORLD
num_parts = comm.Get_size()
logmgr = initialize_logmgr(use_logmgr,
filename="heat-source.sqlite", mode="wu", mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
nel_1d = 16
t = 0
t_final = 0.0002
istep = 0
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(
a=(-0.5,)*dim,
b=(0.5,)*dim,
nelements_per_axis=(nel_1d,)*dim,
boundary_tag_to_face={
"dirichlet": ["+x", "-x"],
"neumann": ["+y", "-y"]
}
)
print("%d elements" % mesh.nelements)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts)
del mesh
else:
local_mesh = mesh_dist.receive_mesh_part()
order = 3
discr = EagerDGDiscretization(actx, local_mesh, order=order,
mpi_communicator=comm)
if dim == 2:
# no deep meaning here, just a fudge factor
dt = 0.0025/(nel_1d*order**2)
else:
raise ValueError("don't have a stable time step guesstimate")
source_width = 0.2
from arraycontext import thaw
nodes = thaw(discr.nodes(), actx)
boundaries = {
DTAG_BOUNDARY("dirichlet"): DirichletDiffusionBoundary(0.),
DTAG_BOUNDARY("neumann"): NeumannDiffusionBoundary(0.)
}
u = discr.zeros(actx)
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr.add_watches(["step.max", "t_step.max", "t_log.max"])
try:
logmgr.add_watches(["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["multiply_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
vis = make_visualizer(discr)
def rhs(t, u):
return (
diffusion_operator(
discr, quad_tag=DISCR_TAG_BASE,
alpha=1, boundaries=boundaries, u=u)
+ actx.np.exp(-np.dot(nodes, nodes)/source_width**2))
compiled_rhs = actx.compile(rhs)
rank = comm.Get_rank()
while t < t_final:
if logmgr:
logmgr.tick_before()
if istep % 10 == 0:
print(istep, t, actx.to_numpy(actx.np.linalg.norm(u[0])))
vis.write_vtk_file("fld-heat-source-mpi-%03d-%04d.vtu" % (rank, istep),
[
("u", u)
], overwrite=True)
u = rk4_step(u, t, dt, compiled_rhs)
t += dt
istep += 1
if logmgr:
set_dt(logmgr, dt)
logmgr.tick_after()
final_answer = discr.norm(u, np.inf)
resid = abs(final_answer - 0.00020620711665201585)
if resid > 1e-15:
raise ValueError(f"Run did not produce the expected result {resid=}")
def main(ctx_factory=cl.create_some_context,
use_logmgr=True,
use_leap=False,
use_profiling=False,
casename=None,
rst_filename=None,
actx_class=PyOpenCLArrayContext):
"""Drive example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nproc = comm.Get_size()
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite",
mode="wu",
mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
# Some discretization parameters
dim = 2
nel_1d = 8
order = 1
# {{{ Time stepping control
# This example runs only 3 steps by default (to keep CI ~short)
# With the mixture defined below, equilibrium is achieved at ~40ms
# To run to equlibrium, set t_final >= 40ms.
# Time stepper selection
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
# Time loop control parameters
current_step = 0
t_final = 1e-8
current_cfl = 1.0
current_dt = 1e-9
current_t = 0
constant_cfl = False
# i.o frequencies
nstatus = 1
nviz = 5
nhealth = 1
nrestart = 5
# }}} Time stepping control
debug = False
rst_path = "restart_data/"
rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl")
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["num_parts"] == nproc
rst_time = restart_data["t"]
rst_step = restart_data["step"]
rst_order = restart_data["order"]
else: # generate the grid from scratch
from meshmode.mesh.generation import generate_regular_rect_mesh
box_ll = -0.005
box_ur = 0.005
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("min_temperature", "------- T (min, max) (K) = ({value:7g}, "),
("max_temperature", "{value:7g})\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
# {{{ Set up initial state using Cantera
# Use Cantera for initialization
# -- Pick up a CTI for the thermochemistry config
# --- Note: Users may add their own CTI file by dropping it into
# --- mirgecom/mechanisms alongside the other CTI files.
from mirgecom.mechanisms import get_mechanism_cti
mech_cti = get_mechanism_cti("uiuc")
cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti)
nspecies = cantera_soln.n_species
# Initial temperature, pressure, and mixutre mole fractions are needed to
# set up the initial state in Cantera.
init_temperature = 1500.0 # Initial temperature hot enough to burn
# Parameters for calculating the amounts of fuel, oxidizer, and inert species
equiv_ratio = 1.0
ox_di_ratio = 0.21
stoich_ratio = 3.0
# Grab the array indices for the specific species, ethylene, oxygen, and nitrogen
i_fu = cantera_soln.species_index("C2H4")
i_ox = cantera_soln.species_index("O2")
i_di = cantera_soln.species_index("N2")
x = np.zeros(nspecies)
# Set the species mole fractions according to our desired fuel/air mixture
x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio +
ox_di_ratio * equiv_ratio)
x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio
x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio
# Uncomment next line to make pylint fail when it can't find cantera.one_atm
one_atm = cantera.one_atm # pylint: disable=no-member
# one_atm = 101325.0
# Let the user know about how Cantera is being initilized
print(f"Input state (T,P,X) = ({init_temperature}, {one_atm}, {x}")
# Set Cantera internal gas temperature, pressure, and mole fractios
cantera_soln.TPX = init_temperature, one_atm, x
# Pull temperature, total density, mass fractions, and pressure from Cantera
# We need total density, and mass fractions to initialize the fluid/gas state.
can_t, can_rho, can_y = cantera_soln.TDY
can_p = cantera_soln.P
# *can_t*, *can_p* should not differ (significantly) from user's initial data,
# but we want to ensure that we use exactly the same starting point as Cantera,
# so we use Cantera's version of these data.
# }}}
# {{{ Create Pyrometheus thermochemistry object & EOS
# Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and
# generates a set of methods to calculate chemothermomechanical properties and
# states for this particular mechanism.
pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np)
eos = PyrometheusMixture(pyrometheus_mechanism,
temperature_guess=init_temperature)
# }}}
# {{{ MIRGE-Com state initialization
# Initialize the fluid/gas state with Cantera-consistent data:
# (density, pressure, temperature, mass_fractions)
print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}")
velocity = np.zeros(shape=(dim, ))
initializer = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=can_p,
temperature=can_t,
massfractions=can_y,
velocity=velocity)
my_boundary = AdiabaticSlipBoundary()
boundaries = {BTAG_ALL: my_boundary}
if rst_filename:
current_step = rst_step
current_t = rst_time
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
if order == rst_order:
current_state = restart_data["state"]
else:
rst_state = restart_data["state"]
old_discr = EagerDGDiscretization(actx,
local_mesh,
order=rst_order,
mpi_communicator=comm)
from meshmode.discretization.connection import make_same_mesh_connection
connection = make_same_mesh_connection(
actx, discr.discr_from_dd("vol"),
old_discr.discr_from_dd("vol"))
current_state = connection(rst_state)
else:
# Set the current state from time 0
current_state = initializer(eos=eos, x_vec=nodes)
# Inspection at physics debugging time
if debug:
print("Initial MIRGE-Com state:")
print(f"{current_state=}")
print(f"Initial DV pressure: {eos.pressure(current_state)}")
print(f"Initial DV temperature: {eos.temperature(current_state)}")
# }}}
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
# Cantera equilibrate calculates the expected end state @ chemical equilibrium
# i.e. the expected state after all reactions
cantera_soln.equilibrate("UV")
eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY
eq_pressure = cantera_soln.P
# Report the expected final state to the user
if rank == 0:
logger.info(init_message)
logger.info(f"Expected equilibrium state:"
f" {eq_pressure=}, {eq_temperature=},"
f" {eq_density=}, {eq_mass_fractions=}")
def my_write_status(dt, cfl):
status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}"
if rank == 0:
logger.info(status_msg)
def my_write_viz(step,
t,
dt,
state,
ts_field=None,
dv=None,
production_rates=None,
cfl=None):
if dv is None:
dv = eos.dependent_vars(state)
if production_rates is None:
production_rates = eos.get_production_rates(state)
if ts_field is None:
ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state)
viz_fields = [("cv", state), ("dv", dv),
("production_rates", production_rates),
("dt" if constant_cfl else "cfl", ts_field)]
write_visfile(discr,
viz_fields,
visualizer,
vizname=casename,
step=step,
t=t,
overwrite=True,
vis_timer=vis_timer)
def my_write_restart(step, t, state):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname == rst_filename:
if rank == 0:
logger.info("Skipping overwrite of restart file.")
else:
rst_data = {
"local_mesh": local_mesh,
"state": state,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": nproc
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(dv):
health_error = False
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", dv.pressure) \
or check_range_local(discr, "vol", dv.pressure, 1e5, 2.4e5):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
if check_range_local(discr, "vol", dv.temperature, 1.498e3, 1.52e3):
health_error = True
logger.info(f"{rank=}: Invalid temperature data found.")
return health_error
def my_get_timestep(t, dt, state):
# richer interface to calculate {dt,cfl} returns node-local estimates
t_remaining = max(0, t_final - t)
if constant_cfl:
from mirgecom.inviscid import get_inviscid_timestep
ts_field = current_cfl * get_inviscid_timestep(
discr, eos=eos, cv=state)
from grudge.op import nodal_min
dt = nodal_min(discr, "vol", ts_field)
cfl = current_cfl
else:
from mirgecom.inviscid import get_inviscid_cfl
ts_field = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state)
from grudge.op import nodal_max
cfl = nodal_max(discr, "vol", ts_field)
return ts_field, cfl, min(t_remaining, dt)
def my_pre_step(step, t, dt, state):
try:
dv = None
if logmgr:
logmgr.tick_before()
from mirgecom.simutil import check_step
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
do_status = check_step(step=step, interval=nstatus)
if do_health:
dv = eos.dependent_vars(state)
from mirgecom.simutil import allsync
health_errors = allsync(my_health_check(dv), comm, op=MPI.LOR)
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state)
if do_status:
my_write_status(dt, cfl)
if do_restart:
my_write_restart(step=step, t=t, state=state)
if do_viz:
production_rates = eos.get_production_rates(state)
if dv is None:
dv = eos.dependent_vars(state)
my_write_viz(step=step,
t=t,
dt=dt,
state=state,
dv=dv,
production_rates=production_rates,
ts_field=ts_field,
cfl=cfl)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
my_write_viz(step=step, t=t, dt=dt, state=state)
my_write_restart(step=step, t=t, state=state)
raise
return state, dt
def my_post_step(step, t, dt, state):
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, state, eos)
logmgr.tick_after()
return state, dt
def my_rhs(t, state):
return (euler_operator(
discr, cv=state, time=t, boundaries=boundaries, eos=eos) +
eos.get_species_source_terms(state))
current_dt = get_sim_timestep(discr, current_state, current_t, current_dt,
current_cfl, eos, t_final, constant_cfl)
current_step, current_t, current_state = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step,
post_step_callback=my_post_step, dt=current_dt,
state=current_state, t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
final_dv = eos.dependent_vars(current_state)
final_dm = eos.get_production_rates(current_state)
ts_field, cfl, dt = my_get_timestep(t=current_t,
dt=current_dt,
state=current_state)
my_write_viz(step=current_step,
t=current_t,
dt=dt,
state=current_state,
dv=final_dv,
production_rates=final_dm,
ts_field=ts_field,
cfl=cfl)
my_write_status(dt=dt, cfl=cfl)
my_write_restart(step=current_step, t=current_t, state=current_state)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
assert np.abs(current_t - t_final) < finish_tol