def op_test_data(ctx_factory):
"""Test fixtures for lazy tests."""
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
eager_actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
lazy_actx = PytatoPyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
def get_discr(order):
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(
a=(-0.5,)*2,
b=(0.5,)*2,
nelements_per_axis=(4,)*2,
boundary_tag_to_face={
"x": ["-x", "+x"],
"y": ["-y", "+y"]
})
from grudge.eager import EagerDGDiscretization
return EagerDGDiscretization(eager_actx, mesh, order=order)
return eager_actx, lazy_actx, get_discr
def sim_health_index(n_runs):
# Set up OpenCL context and command queue
ctx = cl.create_some_context()
queue = cl.CommandQueue(ctx)
mem_pool = cltools.MemoryPool(cltools.ImmediateAllocator(queue))
t0 = time.time()
rho = 0.5
mu = 3.0
sigma = 1.0
z_0 = mu
# Generate an array of Normal Random Numbers on GPU of length n_sims*n_steps
n_steps = int(4160) #4160
rand_gen = clrand.PhiloxGenerator(ctx)
ran = rand_gen.normal(queue, (n_runs * n_steps),
np.float32,
mu=0,
sigma=1.0)
# Establish boundaries for each simulated walk (i.e. start and end)
# Necessary so that we perform scan only within rand walks and not between
seg_boundaries = [1] + [0] * (n_steps - 1)
seg_boundaries = np.array(seg_boundaries, dtype=np.uint8)
seg_boundary_flags = np.tile(seg_boundaries, int(n_runs))
seg_boundary_flags = cl_array.to_device(queue, seg_boundary_flags)
# GPU: Define Segmented Scan Kernel, scanning simulations: f(n-1) + f(n)
prefix_sum = GenericScanKernel(
ctx,
np.float32,
arguments="__global float *ary, __global char *segflags, "
"__global float *out, float rho, float mu",
input_expr="segflags[i] ? (ary[i]+mu):(ary[i]+(1-rho)*mu)",
scan_expr="across_seg_boundary ? (b):(rho*a+b)",
neutral="0",
is_segment_start_expr="segflags[i]",
output_statement="out[i] = item",
options=[])
dev_result = cl_array.arange(queue,
len(ran),
dtype=np.float32,
allocator=mem_pool)
# Enqueue and Run Scan Kernel
prefix_sum(ran, seg_boundary_flags, dev_result, rho, mu)
# Get results back on CPU to plot and do final calcs, just as in Lab 1
health_index_all = (dev_result.get().reshape(n_runs, n_steps).transpose())
final_time = time.time()
time_elapsed = final_time - t0
print("Simulated %d Health Index in: %f seconds" % (n_runs, time_elapsed))
#print(health_index_all)
#print(ran.reshape(n_runs, n_steps).transpose())
#plt.plot(health_index_all)
return
def main(write_output=True):
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
force_device_scalars=True,
)
from meshmode.mesh import BTAG_ALL
from meshmode.mesh.generation import generate_warped_rect_mesh
mesh = generate_warped_rect_mesh(dim=2, order=4, nelements_side=6)
dcoll = DiscretizationCollection(actx, mesh, order=4)
nodes = thaw(dcoll.nodes(), actx)
bdry_nodes = thaw(dcoll.nodes(dd=BTAG_ALL), actx)
bdry_normals = thaw(dcoll.normal(dd=BTAG_ALL), actx)
if write_output:
vis = shortcuts.make_visualizer(dcoll)
vis.write_vtk_file("geo.vtu", [("nodes", nodes)])
bvis = shortcuts.make_boundary_visualizer(dcoll)
bvis.write_vtk_file("bgeo.vtu", [("bdry normals", bdry_normals),
("bdry nodes", bdry_nodes)])
def problem_stats(order=3):
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))
)
with open_output_file("grudge-problem-stats.txt") as outf:
_, dg_discr_2d = get_wave_op_with_discr(
actx, dims=2, order=order)
print("Number of 2D elements:", dg_discr_2d.mesh.nelements, file=outf)
vol_discr_2d = dg_discr_2d.discr_from_dd("vol")
dofs_2d = {group.nunit_dofs for group in vol_discr_2d.groups}
from pytools import one
print("Number of DOFs per 2D element:", one(dofs_2d), file=outf)
_, dg_discr_3d = get_wave_op_with_discr(
actx, dims=3, order=order)
print("Number of 3D elements:", dg_discr_3d.mesh.nelements, file=outf)
vol_discr_3d = dg_discr_3d.discr_from_dd("vol")
dofs_3d = {group.nunit_dofs for group in vol_discr_3d.groups}
from pytools import one
print("Number of DOFs per 3D element:", one(dofs_3d), file=outf)
logger.info("Wrote '%s'", outf.name)
def statement_counts_table():
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))
)
fused_stepper = get_example_stepper(actx, use_fusion=True)
stepper = get_example_stepper(actx, use_fusion=False)
with open_output_file("statement-counts.tex") as outf:
if not PAPER_OUTPUT:
print("==== Statement Counts ====", file=outf)
print(table(
"lr",
("Operator", "Grudge Node Count"),
(
("Time integration: baseline",
r"\num{%d}"
% len(stepper.bound_op.eval_code.instructions)),
("Right-hand side: baseline",
r"\num{%d}"
% len(stepper.grudge_bound_op.eval_code.instructions)),
("Inlined operator",
r"\num{%d}"
% len(fused_stepper.bound_op.eval_code.instructions))
)),
file=outf)
logger.info("Wrote '%s'", outf.name)
def test_assignment_memory_model(ctx_factory):
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))
)
_, discr = get_wave_op_with_discr(actx, dims=2, order=3)
# Assignment instruction
bound_op = bind(
discr,
sym.Variable("input0", dof_desc.DD_VOLUME)
+ sym.Variable("input1", dof_desc.DD_VOLUME),
exec_mapper_factory=ExecutionMapperWithMemOpCounting)
input0 = discr.zeros(actx)
input1 = discr.zeros(actx)
result, profile_data = bound_op(
profile_data={},
input0=input0,
input1=input1)
assert profile_data["bytes_read"] == \
dof_array_nbytes(input0) + dof_array_nbytes(input1)
assert profile_data["bytes_written"] == dof_array_nbytes(result)
def test_reduction(ctx_factory,
grid_shape,
proc_shape,
dtype,
op,
_grid_shape,
pass_grid_dims,
timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
h = 1
grid_shape = _grid_shape or grid_shape
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
from pymbolic import var
from pystella import Field
tmp_insns = [(var("x"), Field("f") / 2 + .31)]
reducers = {}
reducers["avg"] = [(var("x"), op)]
if pass_grid_dims:
reducer = ps.Reduction(mpi,
reducers,
rank_shape=rank_shape,
tmp_instructions=tmp_insns,
grid_size=np.product(grid_shape))
else:
reducer = ps.Reduction(mpi, reducers, tmp_instructions=tmp_insns)
f = clr.rand(queue, rank_shape, dtype=dtype)
import pyopencl.tools as clt
pool = clt.MemoryPool(clt.ImmediateAllocator(queue))
result = reducer(queue, f=f, allocator=pool)
avg = result["avg"]
avg_test = reducer.reduce_array(f / 2 + .31, op)
if op == "avg":
avg_test /= np.product(grid_shape)
rtol = 5e-14 if dtype == np.float64 else 1e-5
assert np.allclose(avg, avg_test, rtol=rtol, atol=0), \
f"{op} reduction innaccurate for {grid_shape=}, {proc_shape=}"
if timing:
from common import timer
t = timer(lambda: reducer(queue, f=f, allocator=pool), ntime=1000)
if mpi.rank == 0:
print(
f"reduction took {t:.3f} ms for {grid_shape=}, {proc_shape=}")
bandwidth = f.nbytes / 1024**3 / t * 1000
print(f"Bandwidth = {bandwidth:.1f} GB/s")
def test_field_statistics(ctx_factory,
grid_shape,
proc_shape,
dtype,
_grid_shape,
pass_grid_dims,
timing=False):
ctx = ctx_factory()
queue = cl.CommandQueue(ctx)
h = 1
grid_shape = _grid_shape or grid_shape
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
# make select parameters local for convenience
h = 2
f = clr.rand(queue, (2, 1) + tuple(ni + 2 * h for ni in rank_shape),
dtype=dtype)
if pass_grid_dims:
statistics = ps.FieldStatistics(mpi,
h,
rank_shape=rank_shape,
grid_size=np.product(grid_shape))
else:
statistics = ps.FieldStatistics(mpi, h)
import pyopencl.tools as clt
pool = clt.MemoryPool(clt.ImmediateAllocator(queue))
stats = statistics(f, allocator=pool)
avg = stats["mean"]
var = stats["variance"]
f_h = f.get()
rank_sum = np.sum(f_h[..., h:-h, h:-h, h:-h], axis=(-3, -2, -1))
avg_test = mpi.allreduce(rank_sum) / np.product(grid_shape)
rank_sum = np.sum(f_h[..., h:-h, h:-h, h:-h]**2, axis=(-3, -2, -1))
var_test = mpi.allreduce(rank_sum) / np.product(grid_shape) - avg_test**2
rtol = 5e-14 if dtype == np.float64 else 1e-5
assert np.allclose(avg, avg_test, rtol=rtol, atol=0), \
f"average innaccurate for {grid_shape=}, {proc_shape=}"
assert np.allclose(var, var_test, rtol=rtol, atol=0), \
f"variance innaccurate for {grid_shape=}, {proc_shape=}"
if timing:
from common import timer
t = timer(lambda: statistics(f, allocator=pool))
if mpi.rank == 0:
print(
f"field stats took {t:.3f} ms "
f"for outer shape {f.shape[:-3]}, {grid_shape=}, {proc_shape=}"
)
def test_mem_pool_with_arrays(ctx_factory):
context = ctx_factory()
queue = cl.CommandQueue(context)
mem_pool = cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))
a_dev = cl_array.arange(queue, 2000, dtype=np.float32, allocator=mem_pool)
b_dev = cl_array.to_device(queue, np.arange(2000), allocator=mem_pool) + 4000
assert a_dev.allocator is mem_pool
assert b_dev.allocator is mem_pool
def test_stepper_timing(ctx_factory, use_fusion):
cl_ctx = ctx_factory()
queue = cl.CommandQueue(
cl_ctx,
properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))
)
dims = 3
sym_operator, discr = get_wave_op_with_discr(
actx, dims=dims, order=3)
t_start = 0
dt = 0.04
t_end = 0.1
ic = flat_obj_array(discr.zeros(actx),
[discr.zeros(actx) for i in range(discr.dim)])
if not use_fusion:
bound_op = bind(
discr, sym_operator,
exec_mapper_factory=ExecutionMapperWithTiming)
stepper = RK4TimeStepper(
discr, "w", bound_op, 1 + discr.dim,
get_wave_component,
exec_mapper_factory=ExecutionMapperWithTiming)
else:
stepper = FusedRK4TimeStepper(
discr, "w", sym_operator, 1 + discr.dim,
get_wave_component,
exec_mapper_factory=ExecutionMapperWithTiming)
step = 0
import time
t = time.time()
nsteps = int(np.ceil((t_end + 1e-9) / dt))
for (_, _, profile_data) in stepper.run( # noqa: B007
ic, t_start, dt, t_end, return_profile_data=True):
step += 1
tn = time.time()
logger.info("step %d/%d: %f", step, nsteps, tn - t)
t = tn
logger.info("fusion? %s", use_fusion)
for key, value in profile_data.items():
if isinstance(value, TimingFutureList):
print(key, value.elapsed())
def __init__(self, ctx, queue):
self.ctx = ctx
self.queue = queue
self.allocator = clt.ImmediateAllocator(self.queue)
self.memory_pool = clt.MemoryPool(self.allocator)
self.program_cache = {}
self.array_cache = {}
self.arrays = None
self.square_array = ElementwiseKernel(self.ctx,
"float *in, float *out",
"out[i] = in[i]*in[i]", "square")
def test_stepper_mem_ops(ctx_factory, use_fusion):
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))
)
dims = 2
sym_operator, discr = get_wave_op_with_discr(
actx, dims=dims, order=3)
t_start = 0
dt = 0.04
t_end = 0.2
ic = flat_obj_array(discr.zeros(actx),
[discr.zeros(actx) for i in range(discr.dim)])
if not use_fusion:
bound_op = bind(
discr, sym_operator,
exec_mapper_factory=ExecutionMapperWithMemOpCounting)
stepper = RK4TimeStepper(
discr, "w", bound_op, 1 + discr.dim,
get_wave_component,
exec_mapper_factory=ExecutionMapperWithMemOpCounting)
else:
stepper = FusedRK4TimeStepper(
discr, "w", sym_operator, 1 + discr.dim,
get_wave_component,
exec_mapper_factory=ExecutionMapperWithMemOpCounting)
step = 0
nsteps = int(np.ceil((t_end + 1e-9) / dt))
for (_, _, profile_data) in stepper.run( # noqa: B007
ic, t_start, dt, t_end, return_profile_data=True):
step += 1
logger.info("step %d/%d", step, nsteps)
logger.info("using fusion? %s", use_fusion)
logger.info("bytes read: %d", profile_data["bytes_read"])
logger.info("bytes written: %d", profile_data["bytes_written"])
logger.info("bytes total: %d",
profile_data["bytes_read"] + profile_data["bytes_written"])
def test_stepper_equivalence(ctx_factory, order=4):
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))
)
dims = 2
sym_operator, discr = get_wave_op_with_discr(
actx, dims=dims, order=order)
#sym_operator_direct, discr = get_wave_op_with_discr_direct(
# actx, dims=dims, order=order)
if dims == 2:
dt = 0.04
elif dims == 3:
dt = 0.02
ic = flat_obj_array(discr.zeros(actx),
[discr.zeros(actx) for i in range(discr.dim)])
bound_op = bind(discr, sym_operator)
stepper = RK4TimeStepper(
discr, "w", bound_op, 1 + discr.dim, get_wave_component)
fused_stepper = FusedRK4TimeStepper(
discr, "w", sym_operator, 1 + discr.dim,
get_wave_component)
t_start = 0
t_end = 0.5
nsteps = int(np.ceil((t_end + 1e-9) / dt))
print("dt=%g nsteps=%d" % (dt, nsteps))
step = 0
norm = bind(discr, sym.norm(2, sym.var("u_ref") - sym.var("u")))
fused_steps = fused_stepper.run(ic, t_start, dt, t_end)
for t_ref, (u_ref, _v_ref) in stepper.run(ic, t_start, dt, t_end):
step += 1
logger.debug("step %d/%d", step, nsteps)
t, (u, v) = next(fused_steps)
assert t == t_ref, step
assert norm(u=u, u_ref=u_ref) <= 1e-13, step
def scalar_assignment_percent_of_total_mem_ops_table():
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))
)
result2d = mem_ops_results(actx, 2)
result3d = mem_ops_results(actx, 3)
with open_output_file("scalar-assignments-mem-op-percentage.tex") as outf:
if not PAPER_OUTPUT:
print("==== Scalar Assigment % of Total Mem Ops ====", file=outf)
print(
table(
"lr",
("Operator",
r"\parbox{1in}{\centering \% Memory Ops. "
r"Due to Scalar Assignments}"),
(
("2D: Baseline",
"%.1f" % (
100 * result2d["nonfused_bytes_total_by_scalar_assignments"]
/ result2d["nonfused_bytes_total"])),
("2D: Inlined",
"%.1f" % (
100 * result2d["fused_bytes_total_by_scalar_assignments"]
/ result2d["fused_bytes_total"])),
("3D: Baseline",
"%.1f" % (
100 * result3d["nonfused_bytes_total_by_scalar_assignments"]
/ result3d["nonfused_bytes_total"])),
("3D: Inlined",
"%.1f" % (
100 * result3d["fused_bytes_total_by_scalar_assignments"]
/ result3d["fused_bytes_total"])),
)),
file=outf)
logger.info("Wrote '%s'", outf.name)
def main(ctx_factory=cl.create_some_context, use_logmgr=True,
use_leap=False, use_profiling=False, casename=None,
rst_filename=None, actx_class=PyOpenCLArrayContext):
"""Drive example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nparts = comm.Get_size()
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
# timestepping control
current_step = 0
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
t_final = 0.005
current_cfl = 1.0
current_dt = .001
current_t = 0
constant_cfl = False
# some i/o frequencies
nstatus = 1
nrestart = 5
nviz = 1
nhealth = 1
dim = 2
rst_path = "restart_data/"
rst_pattern = (
rst_path + "{cname}-{step:04d}-{rank:04d}.pkl"
)
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["nparts"] == nparts
else: # generate the grid from scratch
box_ll = -5.0
box_ur = 5.0
nel_1d = 16
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim,
b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim)
local_mesh, global_nelements = generate_and_distribute_mesh(comm,
generate_mesh)
local_nelements = local_mesh.nelements
order = 3
discr = EagerDGDiscretization(
actx, local_mesh, order=order, mpi_communicator=comm
)
nodes = thaw(actx, discr.nodes())
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging, units_for_logging)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
# soln setup and init
nspecies = 4
centers = make_obj_array([np.zeros(shape=(dim,)) for i in range(nspecies)])
spec_y0s = np.ones(shape=(nspecies,))
spec_amplitudes = np.ones(shape=(nspecies,))
eos = IdealSingleGas()
velocity = np.ones(shape=(dim,))
initializer = MulticomponentLump(dim=dim, nspecies=nspecies,
spec_centers=centers, velocity=velocity,
spec_y0s=spec_y0s,
spec_amplitudes=spec_amplitudes)
boundaries = {
BTAG_ALL: PrescribedInviscidBoundary(fluid_solution_func=initializer)
}
if rst_filename:
current_t = restart_data["t"]
current_step = restart_data["step"]
current_state = restart_data["state"]
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
else:
# Set the current state from time 0
current_state = initializer(nodes)
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim, order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt, t_final=t_final, nstatus=nstatus,
nviz=nviz, cfl=current_cfl,
constant_cfl=constant_cfl, initname=initname,
eosname=eosname, casename=casename)
if rank == 0:
logger.info(init_message)
def my_write_status(component_errors):
if rank == 0:
logger.info(
"------- errors="
+ ", ".join("%.3g" % en for en in component_errors))
def my_write_viz(step, t, state, dv=None, exact=None, resid=None):
if dv is None:
dv = eos.dependent_vars(state)
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
if resid is None:
resid = state - exact
viz_fields = [("cv", state),
("dv", dv),
("exact", exact),
("resid", resid)]
from mirgecom.simutil import write_visfile
write_visfile(discr, viz_fields, visualizer, vizname=casename,
step=step, t=t, overwrite=True, vis_timer=vis_timer)
def my_write_restart(step, t, state):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname != rst_filename:
rst_data = {
"local_mesh": local_mesh,
"state": state,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": nparts
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(pressure, component_errors):
health_error = False
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", pressure) \
or check_range_local(discr, "vol", pressure, .99999999, 1.00000001):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
exittol = .09
if max(component_errors) > exittol:
health_error = True
if rank == 0:
logger.info("Solution diverged from exact soln.")
return health_error
def my_pre_step(step, t, dt, state):
try:
dv = None
exact = None
component_errors = None
if logmgr:
logmgr.tick_before()
from mirgecom.simutil import check_step
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
do_status = check_step(step=step, interval=nstatus)
if do_health:
dv = eos.dependent_vars(state)
exact = initializer(x_vec=nodes, eos=eos, time=t)
from mirgecom.simutil import compare_fluid_solutions
component_errors = compare_fluid_solutions(discr, state, exact)
from mirgecom.simutil import allsync
health_errors = allsync(
my_health_check(dv.pressure, component_errors),
comm, op=MPI.LOR
)
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
if do_restart:
my_write_restart(step=step, t=t, state=state)
if do_viz:
if dv is None:
dv = eos.dependent_vars(state)
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
resid = state - exact
my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact,
resid=resid)
if do_status:
if component_errors is None:
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
from mirgecom.simutil import compare_fluid_solutions
component_errors = compare_fluid_solutions(discr, state, exact)
my_write_status(component_errors)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
my_write_viz(step=step, t=t, state=state)
my_write_restart(step=step, t=t, state=state)
raise
dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final,
constant_cfl)
return state, dt
def my_post_step(step, t, dt, state):
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, state, eos)
logmgr.tick_after()
return state, dt
def my_rhs(t, state):
return euler_operator(discr, cv=state, time=t,
boundaries=boundaries, eos=eos)
current_dt = get_sim_timestep(discr, current_state, current_t, current_dt,
current_cfl, eos, t_final, constant_cfl)
current_step, current_t, current_state = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step, dt=current_dt,
post_step_callback=my_post_step,
state=current_state, t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
final_dv = eos.dependent_vars(current_state)
final_exact = initializer(x_vec=nodes, eos=eos, time=current_t)
final_resid = current_state - final_exact
my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv,
exact=final_exact, resid=final_resid)
my_write_restart(step=current_step, t=current_t, state=current_state)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
time_err = current_t - t_final
if np.abs(time_err) > finish_tol:
raise ValueError(f"Simulation did not finish at expected time {time_err=}.")
def main(ctx_factory=cl.create_some_context):
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
dim = 3
nel_1d = 16
order = 3
exittol = .09
t_final = 0.01
current_cfl = 1.0
vel = np.zeros(shape=(dim, ))
orig = np.zeros(shape=(dim, ))
vel[:dim] = 1.0
current_dt = .001
current_t = 0
eos = IdealSingleGas()
initializer = Lump(numdim=dim, center=orig, velocity=vel)
casename = "lump"
boundaries = {BTAG_ALL: PrescribedBoundary(initializer)}
constant_cfl = False
nstatus = 1
nviz = 1
rank = 0
checkpoint_t = current_t
current_step = 0
timestepper = rk4_step
box_ll = -5.0
box_ur = 5.0
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_grid = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
n=(nel_1d, ) * dim)
local_mesh, global_nelements = create_parallel_grid(comm, generate_grid)
local_nelements = local_mesh.nelements
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
current_state = initializer(0, nodes)
visualizer = make_visualizer(
discr, discr.order + 3 if discr.dim == 2 else discr.order)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
return inviscid_operator(discr,
q=state,
t=t,
boundaries=boundaries,
eos=eos)
def my_checkpoint(step, t, dt, state):
return sim_checkpoint(discr,
visualizer,
eos,
q=state,
exact_soln=initializer,
vizname=casename,
step=step,
t=t,
dt=dt,
nstatus=nstatus,
nviz=nviz,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm)
try:
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t=current_t, t_final=t_final)
except ExactSolutionMismatch as ex:
current_step = ex.step
current_t = ex.t
current_state = ex.state
# if current_t != checkpoint_t:
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state)
if current_t - t_final < 0:
raise ValueError("Simulation exited abnormally")
def main(ctx_factory=cl.create_some_context, use_leap=False):
"""Drive example."""
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
dim = 3
nel_1d = 16
order = 3
exittol = 10.0
t_final = 0.002
current_cfl = 1.0
velocity = np.zeros(shape=(dim, ))
velocity[:dim] = 1.0
current_dt = .001
current_t = 0
constant_cfl = False
nstatus = 1
nviz = 1
rank = 0
checkpoint_t = current_t
current_step = 0
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
box_ll = -5.0
box_ur = 5.0
error_state = 0
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
casename = "uiuc_mixture"
# Pyrometheus initialization
from mirgecom.mechanisms import get_mechanism_cti
mech_cti = get_mechanism_cti("uiuc")
sol = cantera.Solution(phase_id="gas", source=mech_cti)
pyrometheus_mechanism = pyro.get_thermochem_class(sol)(actx.np)
nspecies = pyrometheus_mechanism.num_species
eos = PyrometheusMixture(pyrometheus_mechanism)
y0s = np.zeros(shape=(nspecies, ))
for i in range(nspecies - 1):
y0s[i] = 1.0 / (10.0**(i + 1))
spec_sum = sum([y0s[i] for i in range(nspecies - 1)])
y0s[nspecies - 1] = 1.0 - spec_sum
# Mixture defaults to STP (p, T) = (1atm, 300K)
initializer = MixtureInitializer(dim=dim,
nspecies=nspecies,
massfractions=y0s,
velocity=velocity)
boundaries = {BTAG_ALL: PrescribedBoundary(initializer)}
nodes = thaw(actx, discr.nodes())
current_state = initializer(x_vec=nodes, eos=eos)
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
return euler_operator(discr,
cv=state,
t=t,
boundaries=boundaries,
eos=eos)
def my_checkpoint(step, t, dt, state):
global checkpoint_t
checkpoint_t = t
return sim_checkpoint(discr,
visualizer,
eos,
cv=state,
exact_soln=initializer,
vizname=casename,
step=step,
t=t,
dt=dt,
nstatus=nstatus,
nviz=nviz,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm)
try:
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t=current_t, t_final=t_final)
except ExactSolutionMismatch as ex:
error_state = 1
current_step = ex.step
current_t = ex.t
current_state = ex.state
if current_t != checkpoint_t: # This check because !overwrite
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state)
if current_t - t_final < 0:
error_state = 1
if error_state:
raise ValueError("Simulation did not complete successfully.")
def main():
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
from mpi4py import MPI
comm = MPI.COMM_WORLD
num_parts = comm.Get_size()
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
nel_1d = 16
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim,
b=(0.5, ) * dim,
n=(nel_1d, ) * dim,
boundary_tag_to_face={
"dirichlet": ["+x", "-x"],
"neumann": ["+y", "-y"]
})
print("%d elements" % mesh.nelements)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element,
num_parts)
del mesh
else:
local_mesh = mesh_dist.receive_mesh_part()
order = 3
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
if dim == 2:
# no deep meaning here, just a fudge factor
dt = 0.0025 / (nel_1d * order**2)
else:
raise ValueError("don't have a stable time step guesstimate")
source_width = 0.2
nodes = thaw(actx, discr.nodes())
u = discr.zeros(actx)
vis = make_visualizer(discr, order + 3 if dim == 2 else order)
boundaries = {
grudge_sym.DTAG_BOUNDARY("dirichlet"): DirichletDiffusionBoundary(0.),
grudge_sym.DTAG_BOUNDARY("neumann"): NeumannDiffusionBoundary(0.)
}
def rhs(t, u):
return (
diffusion_operator(discr, alpha=1, boundaries=boundaries, u=u) +
actx.np.exp(-np.dot(nodes, nodes) / source_width**2))
rank = comm.Get_rank()
t = 0
t_final = 0.01
istep = 0
while True:
if istep % 10 == 0:
print(istep, t, discr.norm(u))
vis.write_vtk_file(
"fld-heat-source-mpi-%03d-%04d.vtu" % (rank, istep),
[("u", u)])
if t >= t_final:
break
u = rk4_step(u, t, dt, rhs)
t += dt
istep += 1
def main(ctx_factory,
dim=2,
order=4,
use_quad=False,
visualize=False,
flux_type="upwind"):
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
force_device_scalars=True,
)
# {{{ parameters
# domain [0, d]^dim
d = 1.0
# number of points in each dimension
npoints = 25
# final time
final_time = 1
if use_quad:
qtag = dof_desc.DISCR_TAG_QUAD
else:
qtag = None
# }}}
# {{{ discretization
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(0, ) * dim,
b=(d, ) * dim,
npoints_per_axis=(npoints, ) * dim,
order=order)
from meshmode.discretization.poly_element import \
QuadratureSimplexGroupFactory
if use_quad:
discr_tag_to_group_factory = {
qtag: QuadratureSimplexGroupFactory(order=4 * order)
}
else:
discr_tag_to_group_factory = {}
from grudge import DiscretizationCollection
dcoll = DiscretizationCollection(
actx,
mesh,
order=order,
discr_tag_to_group_factory=discr_tag_to_group_factory)
# }}}
# {{{ advection operator
# gaussian parameters
def f_halfcircle(x):
source_center = np.array([d / 2, d / 2, d / 2])[:dim]
dist = x - source_center
return ((0.5 + 0.5 * actx.np.tanh(500 *
(-np.dot(dist, dist) + 0.4**2))) *
(0.5 + 0.5 * actx.np.tanh(500 * (dist[0]))))
def zero_inflow_bc(dtag, t=0):
dd = dof_desc.DOFDesc(dtag, qtag)
return dcoll.discr_from_dd(dd).zeros(actx)
from grudge.models.advection import VariableCoefficientAdvectionOperator
x = thaw(dcoll.nodes(), actx)
# velocity field
if dim == 1:
c = x
else:
# solid body rotation
c = flat_obj_array(np.pi * (d / 2 - x[1]), np.pi * (x[0] - d / 2),
0)[:dim]
adv_operator = VariableCoefficientAdvectionOperator(
dcoll,
c,
inflow_u=lambda t: zero_inflow_bc(BTAG_ALL, t),
quad_tag=qtag,
flux_type=flux_type)
u = f_halfcircle(x)
def rhs(t, u):
return adv_operator.operator(t, u)
dt = actx.to_numpy(
adv_operator.estimate_rk4_timestep(actx, dcoll, fields=u))
logger.info("Timestep size: %g", dt)
# }}}
# {{{ time stepping
from grudge.shortcuts import set_up_rk4
dt_stepper = set_up_rk4("u", dt, u, rhs)
plot = Plotter(actx, dcoll, order, visualize=visualize, ylim=[-0.1, 1.1])
step = 0
for event in dt_stepper.run(t_end=final_time):
if not isinstance(event, dt_stepper.StateComputed):
continue
if step % 10 == 0:
norm_u = actx.to_numpy(op.norm(dcoll, event.state_component, 2))
plot(event, "fld-var-velocity-%04d" % step)
step += 1
logger.info("[%04d] t = %.5f |u| = %.5e", step, event.t, norm_u)
# NOTE: These are here to ensure the solution is bounded for the
# time interval specified
assert norm_u < 1
def main(ctx_factory=cl.create_some_context,
snapshot_pattern="y0euler-{step:06d}-{rank:04d}.pkl",
restart_step=None,
use_profiling=False,
use_logmgr=False):
"""Drive the Y0 example."""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = 0
rank = comm.Get_rank()
nparts = comm.Get_size()
"""logging and profiling"""
logmgr = initialize_logmgr(use_logmgr,
use_profiling,
filename="y0euler.sqlite",
mode="wu",
mpi_comm=comm)
cl_ctx = ctx_factory()
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLProfilingArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
#nviz = 500
#nrestart = 500
nviz = 50
nrestart = 10000000
current_dt = 1.0e-7
#t_final = 5.e-7
t_final = 3e-4
dim = 2
order = 1
exittol = 10000000 # do never exit when comparing to exact solution
#t_final = 0.001
current_cfl = 1.0
vel_init = np.zeros(shape=(dim, ))
vel_inflow = np.zeros(shape=(dim, ))
vel_outflow = np.zeros(shape=(dim, ))
orig = np.zeros(shape=(dim, ))
#vel[0] = 340.0
#vel_inflow[0] = 100.0 # m/s
current_t = 0
casename = "y0euler"
constant_cfl = False
# no internal euler status messages
nstatus = 1000000000
checkpoint_t = current_t
current_step = 0
# working gas: CO2 #
# gamma = 1.289
# MW=44.009 g/mol
# cp = 37.135 J/mol-K,
# rho= 1.977 kg/m^3 @298K
gamma_CO2 = 1.289
R_CO2 = 8314.59 / 44.009
# background
# 100 Pa
# 298 K
# rho = 1.77619667e-3 kg/m^3
# velocity = 0,0,0
rho_bkrnd = 1.77619667e-3
pres_bkrnd = 100
temp_bkrnd = 298
c_bkrnd = math.sqrt(gamma_CO2 * pres_bkrnd / rho_bkrnd)
# isentropic shock relations #
# lab frame, moving shock
# state 1 is behind (downstream) the shock, state 2 is in front (upstream) of the shock
mach = 2.0
pressure_ratio = (2. * gamma_CO2 * mach * mach -
(gamma_CO2 - 1.)) / (gamma_CO2 + 1.)
density_ratio = (gamma_CO2 + 1.) * mach * mach / (
(gamma_CO2 - 1.) * mach * mach + 2.)
mach2 = math.sqrt(((gamma_CO2 - 1.) * mach * mach + 2.) /
(2. * gamma_CO2 * mach * mach - (gamma_CO2 - 1.)))
rho1 = rho_bkrnd
pressure1 = pres_bkrnd
rho2 = rho1 * density_ratio
pressure2 = pressure1 * pressure_ratio
velocity1 = 0.
velocity2 = -mach * c_bkrnd * (1 / density_ratio - 1)
c_shkd = math.sqrt(gamma_CO2 * pressure2 / rho2)
vel_inflow[0] = velocity2
timestepper = rk4_step
eos = IdealSingleGas(gamma=gamma_CO2, gas_const=R_CO2)
bulk_init = Discontinuity(dim=dim,
x0=.05,
sigma=0.01,
rhol=rho2,
rhor=rho1,
pl=pressure2,
pr=pressure1,
ul=vel_inflow[0],
ur=0.)
inflow_init = Lump(dim=dim,
rho0=rho2,
p0=pressure2,
center=orig,
velocity=vel_inflow,
rhoamp=0.0)
outflow_init = Lump(dim=dim,
rho0=rho1,
p0=pressure1,
center=orig,
velocity=vel_outflow,
rhoamp=0.0)
inflow = PrescribedBoundary(inflow_init)
outflow = PrescribedBoundary(outflow_init)
wall = AdiabaticSlipBoundary()
dummy = DummyBoundary()
# shock capturing parameters
# sonic conditions
density_ratio = (gamma_CO2 + 1.) * 1.0 / ((gamma_CO2 - 1.) + 2.)
density_star = rho1 * density_ratio
shock_thickness = 20 * 0.001 # on the order of 3 elements, should match what is in mesh generator
# alpha is ~h/p (spacing/order)
#alpha_sc = shock_thickness*abs(velocity1-velocity2)*density_star
alpha_sc = 0.1
# sigma is ~p^-4
sigma_sc = -11.0
# kappa is empirical ...
kappa_sc = 0.5
print(
f"Shock capturing parameters: alpha {alpha_sc}, s0 {sigma_sc}, kappa {kappa_sc}"
)
# timestep estimate
wave_speed = max(mach2 * c_bkrnd, c_shkd + velocity2)
char_len = 0.001
area = char_len * char_len / 2
perimeter = 2 * char_len + math.sqrt(2 * char_len * char_len)
h = 2 * area / perimeter
dt_est = 1 / (wave_speed * order * order / h)
print(f"Time step estimate {dt_est}\n")
dt_est_visc = 1 / (wave_speed * order * order / h +
alpha_sc * order * order * order * order / h / h)
print(f"Viscous timestep estimate {dt_est_visc}\n")
from grudge import sym
# boundaries = {BTAG_ALL: DummyBoundary}
boundaries = {
sym.DTAG_BOUNDARY("Inflow"): inflow,
sym.DTAG_BOUNDARY("Outflow"): outflow,
sym.DTAG_BOUNDARY("Wall"): wall
}
#local_mesh, global_nelements = create_parallel_grid(comm,
#get_pseudo_y0_mesh)
#
#local_nelements = local_mesh.nelements
if restart_step is None:
local_mesh, global_nelements = create_parallel_grid(comm, get_mesh)
local_nelements = local_mesh.nelements
else: # Restart
with open(snapshot_pattern.format(step=restart_step, rank=rank),
"rb") as f:
restart_data = pickle.load(f)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert comm.Get_size() == restart_data["num_parts"]
if rank == 0:
logging.info("Making discretization")
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
if restart_step is None:
if rank == 0:
logging.info("Initializing soln.")
current_state = bulk_init(0, nodes, eos=eos)
else:
current_t = restart_data["t"]
current_step = restart_step
current_state = unflatten(
actx, discr.discr_from_dd("vol"),
obj_array_vectorize(actx.from_numpy, restart_data["state"]))
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_discretization_quantities(logmgr, discr, eos, dim)
#logmgr_add_package_versions(logmgr)
logmgr.add_watches([
"step.max", "t_sim.max", "t_step.max", "min_pressure",
"max_pressure", "min_temperature", "max_temperature"
])
try:
logmgr.add_watches(["memory_usage.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["pyopencl_array_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
#visualizer = make_visualizer(discr, discr.order + 3
#if discr.dim == 2 else discr.order)
visualizer = make_visualizer(discr, discr.order)
# initname = initializer.__class__.__name__
initname = "pseudoY0"
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
#return inviscid_operator(discr, eos=eos, boundaries=boundaries, q=state, t=t)
return (inviscid_operator(
discr, q=state, t=t, boundaries=boundaries, eos=eos) +
artificial_viscosity(discr,
t=t,
r=state,
eos=eos,
boundaries=boundaries,
alpha=alpha_sc,
sigma=sigma_sc,
kappa=kappa_sc))
def my_checkpoint(step, t, dt, state):
write_restart = (check_step(step, nrestart)
if step != restart_step else False)
if write_restart is True:
with open(snapshot_pattern.format(step=step, rank=rank),
"wb") as f:
pickle.dump(
{
"local_mesh": local_mesh,
"state": obj_array_vectorize(actx.to_numpy,
flatten(state)),
"t": t,
"step": step,
"global_nelements": global_nelements,
"num_parts": nparts,
}, f)
#x0=f(time)
exact_soln = Discontinuity(dim=dim,
x0=.05,
sigma=0.00001,
rhol=rho2,
rhor=rho1,
pl=pressure2,
pr=pressure1,
ul=vel_inflow[0],
ur=0.,
uc=mach * c_bkrnd)
return sim_checkpoint(discr=discr,
visualizer=visualizer,
eos=eos,
q=state,
vizname=casename,
step=step,
t=t,
dt=dt,
nstatus=nstatus,
nviz=nviz,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm,
vis_timer=vis_timer,
overwrite=True,
exact_soln=exact_soln,
sigma=sigma_sc,
kappa=kappa_sc)
if rank == 0:
logging.info("Stepping.")
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t_final=t_final, t=current_t, istep=current_step,
logmgr=logmgr,eos=eos,dim=dim)
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state)
if current_t - t_final < 0:
raise ValueError("Simulation exited abnormally")
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
def main(ctx_factory=cl.create_some_context,
snapshot_pattern="flame1d-{step:06d}-{rank:04d}.pkl",
restart_step=None,
use_profiling=False,
use_logmgr=False):
"""Drive the Y0 example."""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = 0
rank = comm.Get_rank()
nparts = comm.Get_size()
"""logging and profiling"""
logmgr = initialize_logmgr(use_logmgr,
filename="flame1d.sqlite",
mode="wo",
mpi_comm=comm)
cl_ctx = ctx_factory()
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLProfilingArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
#nviz = 500
#nrestart = 500
nviz = 1
nrestart = 3
#current_dt = 5.0e-8 # stable with euler
current_dt = 5.0e-8 # stable with rk4
#current_dt = 4e-7 # stable with lrsrk144
t_final = 1.5e-7
dim = 2
order = 1
exittol = 1000000000000
#t_final = 0.001
current_cfl = 1.0
current_t = 0
constant_cfl = False
nstatus = 10000000000
rank = 0
checkpoint_t = current_t
current_step = 0
vel_burned = np.zeros(shape=(dim, ))
vel_unburned = np.zeros(shape=(dim, ))
# {{{ Set up initial state using Cantera
# Use Cantera for initialization
# -- Pick up a CTI for the thermochemistry config
# --- Note: Users may add their own CTI file by dropping it into
# --- mirgecom/mechanisms alongside the other CTI files.
from mirgecom.mechanisms import get_mechanism_cti
# uiuc C2H4
#mech_cti = get_mechanism_cti("uiuc")
# sanDiego H2
mech_cti = get_mechanism_cti("sanDiego")
cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti)
nspecies = cantera_soln.n_species
# Initial temperature, pressure, and mixutre mole fractions are needed to
# set up the initial state in Cantera.
temp_unburned = 300.0
temp_ignition = 1500.0
# Parameters for calculating the amounts of fuel, oxidizer, and inert species
equiv_ratio = 1.0
ox_di_ratio = 0.21
# H2
stoich_ratio = 0.5
#C2H4
#stoich_ratio = 3.0
# Grab the array indices for the specific species, ethylene, oxygen, and nitrogen
# C2H4
#i_fu = cantera_soln.species_index("C2H4")
# H2
i_fu = cantera_soln.species_index("H2")
i_ox = cantera_soln.species_index("O2")
i_di = cantera_soln.species_index("N2")
x = np.zeros(nspecies)
# Set the species mole fractions according to our desired fuel/air mixture
x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio +
ox_di_ratio * equiv_ratio)
x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio
x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio
# Uncomment next line to make pylint fail when it can't find cantera.one_atm
one_atm = cantera.one_atm # pylint: disable=no-member
# one_atm = 101325.0
pres_unburned = one_atm
# Let the user know about how Cantera is being initilized
print(f"Input state (T,P,X) = ({temp_unburned}, {pres_unburned}, {x}")
# Set Cantera internal gas temperature, pressure, and mole fractios
cantera_soln.TPX = temp_unburned, pres_unburned, x
# Pull temperature, total density, mass fractions, and pressure from Cantera
# We need total density, and mass fractions to initialize the fluid/gas state.
y_unburned = np.zeros(nspecies)
can_t, rho_unburned, y_unburned = cantera_soln.TDY
can_p = cantera_soln.P
# *can_t*, *can_p* should not differ (significantly) from user's initial data,
# but we want to ensure that we use exactly the same starting point as Cantera,
# so we use Cantera's version of these data.
# now find the conditions for the burned gas
cantera_soln.equilibrate('TP')
temp_burned, rho_burned, y_burned = cantera_soln.TDY
pres_burned = cantera_soln.P
casename = "flame1d"
pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np)
# C2H4
mu = 1.e-5
kappa = 1.6e-5 # Pr = mu*rho/alpha = 0.75
# H2
mu = 1.e-5
kappa = mu * 0.08988 / 0.75 # Pr = mu*rho/alpha = 0.75
species_diffusivity = 1.e-5 * np.ones(nspecies)
transport_model = SimpleTransport(viscosity=mu,
thermal_conductivity=kappa,
species_diffusivity=species_diffusivity)
eos = PyrometheusMixture(pyrometheus_mechanism,
temperature_guess=temp_unburned,
transport_model=transport_model)
species_names = pyrometheus_mechanism.species_names
print(f"Pyrometheus mechanism species names {species_names}")
print(
f"Unburned state (T,P,Y) = ({temp_unburned}, {pres_unburned}, {y_unburned}"
)
print(f"Burned state (T,P,Y) = ({temp_burned}, {pres_burned}, {y_burned}")
flame_start_loc = 0.05
flame_speed = 1000
# use the burned conditions with a lower temperature
bulk_init = PlanarDiscontinuity(dim=dim,
disc_location=flame_start_loc,
sigma=0.01,
nspecies=nspecies,
temperature_left=temp_ignition,
temperature_right=temp_unburned,
pressure_left=pres_burned,
pressure_right=pres_unburned,
velocity_left=vel_burned,
velocity_right=vel_unburned,
species_mass_left=y_burned,
species_mass_right=y_unburned)
inflow_init = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=pres_burned,
temperature=temp_ignition,
massfractions=y_burned,
velocity=vel_burned)
outflow_init = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=pres_unburned,
temperature=temp_unburned,
massfractions=y_unburned,
velocity=vel_unburned)
inflow = PrescribedViscousBoundary(q_func=inflow_init)
outflow = PrescribedViscousBoundary(q_func=outflow_init)
wall = PrescribedViscousBoundary(
) # essentially a "dummy" use the interior solution for the exterior
from grudge import sym
boundaries = {
sym.DTAG_BOUNDARY("Inflow"): inflow,
sym.DTAG_BOUNDARY("Outflow"): outflow,
sym.DTAG_BOUNDARY("Wall"): wall
}
if restart_step is None:
char_len = 0.001
box_ll = (0.0, 0.0)
box_ur = (0.25, 0.01)
num_elements = (int((box_ur[0] - box_ll[0]) / char_len),
int((box_ur[1] - box_ll[1]) / char_len))
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh,
a=box_ll,
b=box_ur,
n=num_elements,
mesh_type="X",
boundary_tag_to_face={
"Inflow": ["-x"],
"Outflow": ["+x"],
"Wall": ["+y", "-y"]
})
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
else: # Restart
with open(snapshot_pattern.format(step=restart_step, rank=rank),
"rb") as f:
restart_data = pickle.load(f)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert comm.Get_size() == restart_data["num_parts"]
if rank == 0:
logging.info("Making discretization")
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
if restart_step is None:
if rank == 0:
logging.info("Initializing soln.")
# for Discontinuity initial conditions
current_state = bulk_init(t=0., x_vec=nodes, eos=eos)
# for uniform background initial condition
#current_state = bulk_init(nodes, eos=eos)
else:
current_t = restart_data["t"]
current_step = restart_step
current_state = unflatten(
actx, discr.discr_from_dd("vol"),
obj_array_vectorize(actx.from_numpy, restart_data["state"]))
vis_timer = None
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
logmgr.add_watches([
"step.max", "t_sim.max", "t_step.max", "t_log.max", "min_pressure",
"max_pressure", "min_temperature", "max_temperature"
])
try:
logmgr.add_watches(
["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["pyopencl_array_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
visualizer = make_visualizer(discr, order)
# initname = initializer.__class__.__name__
initname = "flame1d"
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
#timestepper = rk4_step
#timestepper = lsrk54_step
#timestepper = lsrk144_step
timestepper = euler_step
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
# check for some troublesome output types
inf_exists = not np.isfinite(discr.norm(state, np.inf))
if inf_exists:
if rank == 0:
logging.info(
"Non-finite values detected in simulation, exiting...")
# dump right now
sim_checkpoint(discr=discr,
visualizer=visualizer,
eos=eos,
q=state,
vizname=casename,
step=999999999,
t=t,
dt=current_dt,
nviz=1,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm,
vis_timer=vis_timer,
overwrite=True,
s0=s0_sc,
kappa=kappa_sc)
exit()
cv = split_conserved(dim=dim, q=state)
return (
ns_operator(discr, q=state, t=t, boundaries=boundaries, eos=eos) +
eos.get_species_source_terms(cv))
def my_checkpoint(step, t, dt, state):
write_restart = (check_step(step, nrestart)
if step != restart_step else False)
if write_restart is True:
with open(snapshot_pattern.format(step=step, rank=rank),
"wb") as f:
pickle.dump(
{
"local_mesh": local_mesh,
"state": obj_array_vectorize(actx.to_numpy,
flatten(state)),
"t": t,
"step": step,
"global_nelements": global_nelements,
"num_parts": nparts,
}, f)
def loc_fn(t):
return flame_start_loc + flame_speed * t
exact_soln = PlanarDiscontinuity(dim=dim,
disc_location=loc_fn,
sigma=0.0000001,
nspecies=nspecies,
temperature_left=temp_ignition,
temperature_right=temp_unburned,
pressure_left=pres_burned,
pressure_right=pres_unburned,
velocity_left=vel_burned,
velocity_right=vel_unburned,
species_mass_left=y_burned,
species_mass_right=y_unburned)
cv = split_conserved(dim, state)
reaction_rates = eos.get_production_rates(cv)
viz_fields = [("reaction_rates", reaction_rates)]
return sim_checkpoint(discr=discr,
visualizer=visualizer,
eos=eos,
q=state,
vizname=casename,
step=step,
t=t,
dt=dt,
nstatus=nstatus,
nviz=nviz,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm,
vis_timer=vis_timer,
overwrite=True,
exact_soln=exact_soln,
viz_fields=viz_fields)
if rank == 0:
logging.info("Stepping.")
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t_final=t_final, t=current_t, istep=current_step,
logmgr=logmgr,eos=eos,dim=dim)
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state)
if current_t - t_final < 0:
raise ValueError("Simulation exited abnormally")
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
exit()
def main(ctx_factory, dim=3, order=4, visualize=False):
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
force_device_scalars=True,
)
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(0.0, ) * dim,
b=(1.0, ) * dim,
nelements_per_axis=(4, ) * dim)
dcoll = DiscretizationCollection(actx, mesh, order=order)
if 0:
epsilon0 = 8.8541878176e-12 # C**2 / (N m**2)
mu0 = 4 * np.pi * 1e-7 # N/A**2.
epsilon = 1 * epsilon0
mu = 1 * mu0
else:
epsilon = 1
mu = 1
from grudge.models.em import MaxwellOperator
maxwell_operator = MaxwellOperator(dcoll,
epsilon,
mu,
flux_type=0.5,
dimensions=dim)
def cavity_mode(x, t=0):
if dim == 3:
return get_rectangular_cavity_mode(actx, x, t, 1, (1, 2, 2))
else:
return get_rectangular_cavity_mode(actx, x, t, 1, (2, 3))
fields = cavity_mode(thaw(dcoll.nodes(), actx), t=0)
maxwell_operator.check_bc_coverage(mesh)
def rhs(t, w):
return maxwell_operator.operator(t, w)
dt = maxwell_operator.estimate_rk4_timestep(actx, dcoll, fields=fields)
dt_stepper = set_up_rk4("w", dt, fields, rhs)
target_steps = 60
final_t = dt * target_steps
nsteps = int(final_t / dt) + 1
logger.info("dt = %g nsteps = %d", dt, nsteps)
from grudge.shortcuts import make_visualizer
vis = make_visualizer(dcoll)
step = 0
def norm(u):
return op.norm(dcoll, u, 2)
e, h = maxwell_operator.split_eh(fields)
if visualize:
vis.write_vtk_file(f"fld-cavities-{step:04d}.vtu", [
("e", e),
("h", h),
])
for event in dt_stepper.run(t_end=final_t):
if isinstance(event, dt_stepper.StateComputed):
assert event.component_id == "w"
step += 1
e, h = maxwell_operator.split_eh(event.state_component)
norm_e0 = actx.to_numpy(norm(u=e[0]))
norm_e1 = actx.to_numpy(norm(u=e[1]))
norm_h0 = actx.to_numpy(norm(u=h[0]))
norm_h1 = actx.to_numpy(norm(u=h[1]))
logger.info(
"[%04d] t = %.5f |e0| = %.5e, |e1| = %.5e, |h0| = %.5e, |h1| = %.5e",
step, event.t, norm_e0, norm_e1, norm_h0, norm_h1)
if step % 10 == 0:
if visualize:
vis.write_vtk_file(f"fld-cavities-{step:04d}.vtu", [
("e", e),
("h", h),
])
# NOTE: These are here to ensure the solution is bounded for the
# time interval specified
assert norm_e0 < 0.5
assert norm_e1 < 0.5
assert norm_h0 < 0.5
assert norm_h1 < 0.5
def main(ctx_factory=cl.create_some_context,
casename="flame1d",
user_input_file=None,
snapshot_pattern="{casename}-{step:06d}-{rank:04d}.pkl",
restart_step=None,
restart_name=None,
use_profiling=False,
use_logmgr=False,
use_lazy_eval=False):
"""Drive the 1D Flame example."""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = 0
rank = comm.Get_rank()
nparts = comm.Get_size()
if restart_name is None:
restart_name = casename
"""logging and profiling"""
logmgr = initialize_logmgr(use_logmgr,
filename=(f"{casename}.sqlite"),
mode="wo",
mpi_comm=comm)
cl_ctx = ctx_factory()
if use_profiling:
if use_lazy_eval:
raise RuntimeError("Cannot run lazy with profiling.")
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLProfilingArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
if use_lazy_eval:
actx = PytatoArrayContext(queue)
else:
actx = PyOpenCLArrayContext(
queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
# default input values that will be read from input (if they exist)
nviz = 100
nrestart = 100
nhealth = 100
nstatus = 1
current_dt = 1e-9
t_final = 1.e-3
order = 1
integrator = "rk4"
if user_input_file:
if rank == 0:
with open(user_input_file) as f:
input_data = yaml.load(f, Loader=yaml.FullLoader)
else:
input_data = None
input_data = comm.bcast(input_data, root=0)
#print(input_data)
try:
nviz = int(input_data["nviz"])
except KeyError:
pass
try:
nrestart = int(input_data["nrestart"])
except KeyError:
pass
try:
nhealth = int(input_data["nhealth"])
except KeyError:
pass
try:
nstatus = int(input_data["nstatus"])
except KeyError:
pass
try:
current_dt = float(input_data["current_dt"])
except KeyError:
pass
try:
t_final = float(input_data["t_final"])
except KeyError:
pass
try:
order = int(input_data["order"])
except KeyError:
pass
try:
integrator = input_data["integrator"]
except KeyError:
pass
# param sanity check
allowed_integrators = ["rk4", "euler", "lsrk54", "lsrk144"]
if (integrator not in allowed_integrators):
error_message = "Invalid time integrator: {}".format(integrator)
raise RuntimeError(error_message)
timestepper = rk4_step
if integrator == "euler":
timestepper = euler_step
if integrator == "lsrk54":
timestepper = lsrk54_step
if integrator == "lsrk144":
timestepper = lsrk144_step
if (rank == 0):
print(f'#### Simluation control data: ####')
print(f'\tnviz = {nviz}')
print(f'\tnrestart = {nrestart}')
print(f'\tnhealth = {nhealth}')
print(f'\tnstatus = {nstatus}')
print(f'\tcurrent_dt = {current_dt}')
print(f'\tt_final = {t_final}')
print(f'\torder = {order}')
print(f"\tTime integration {integrator}")
print(f'#### Simluation control data: ####')
restart_path = 'restart_data/'
viz_path = 'viz_data/'
#if(rank == 0):
#if not os.path.exists(restart_path):
#os.makedirs(restart_path)
#if not os.path.exists(viz_path):
#os.makedirs(viz_path)
dim = 2
exittol = .09
current_cfl = 1.0
current_t = 0
constant_cfl = False
checkpoint_t = current_t
current_step = 0
vel_burned = np.zeros(shape=(dim, ))
vel_unburned = np.zeros(shape=(dim, ))
# {{{ Set up initial state using Cantera
# Use Cantera for initialization
# -- Pick up a CTI for the thermochemistry config
# --- Note: Users may add their own CTI file by dropping it into
# --- mirgecom/mechanisms alongside the other CTI files.
fuel = "H2"
allowed_fuels = ["H2", "C2H4"]
if (fuel not in allowed_fuels):
error_message = "Invalid fuel selection: {}".format(fuel)
raise RuntimeError(error_message)
if rank == 0:
print(f"Fuel: {fuel}")
from mirgecom.mechanisms import get_mechanism_cti
if fuel == "C2H4":
mech_cti = get_mechanism_cti("uiuc")
elif fuel == "H2":
mech_cti = get_mechanism_cti("sanDiego")
cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti)
nspecies = cantera_soln.n_species
# Initial temperature, pressure, and mixutre mole fractions are needed to
# set up the initial state in Cantera.
temp_unburned = 300.0
temp_ignition = 1500.0
# Parameters for calculating the amounts of fuel, oxidizer, and inert species
if fuel == "C2H4":
stoich_ratio = 3.0
if fuel == "H2":
stoich_ratio = 0.5
equiv_ratio = 1.0
ox_di_ratio = 0.21
# Grab the array indices for the specific species, ethylene, oxygen, and nitrogen
i_fu = cantera_soln.species_index(fuel)
i_ox = cantera_soln.species_index("O2")
i_di = cantera_soln.species_index("N2")
x = np.zeros(nspecies)
# Set the species mole fractions according to our desired fuel/air mixture
x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio +
ox_di_ratio * equiv_ratio)
x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio
x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio
# Uncomment next line to make pylint fail when it can't find cantera.one_atm
one_atm = cantera.one_atm # pylint: disable=no-member
# one_atm = 101325.0
pres_unburned = one_atm
# Let the user know about how Cantera is being initilized
print(f"Input state (T,P,X) = ({temp_unburned}, {pres_unburned}, {x}")
# Set Cantera internal gas temperature, pressure, and mole fractios
cantera_soln.TPX = temp_unburned, pres_unburned, x
# Pull temperature, total density, mass fractions, and pressure from Cantera
# We need total density, and mass fractions to initialize the fluid/gas state.
y_unburned = np.zeros(nspecies)
can_t, rho_unburned, y_unburned = cantera_soln.TDY
can_p = cantera_soln.P
# *can_t*, *can_p* should not differ (significantly) from user's initial data,
# but we want to ensure that we use exactly the same starting point as Cantera,
# so we use Cantera's version of these data.
# now find the conditions for the burned gas
cantera_soln.equilibrate('TP')
temp_burned, rho_burned, y_burned = cantera_soln.TDY
pres_burned = cantera_soln.P
pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np)
kappa = 1.6e-5 # Pr = mu*rho/alpha = 0.75
mu = 1.e-5
species_diffusivity = 1.e-5 * np.ones(nspecies)
transport_model = SimpleTransport(viscosity=mu,
thermal_conductivity=kappa,
species_diffusivity=species_diffusivity)
eos = PyrometheusMixture(pyrometheus_mechanism,
temperature_guess=temp_unburned,
transport_model=transport_model)
species_names = pyrometheus_mechanism.species_names
print(f"Pyrometheus mechanism species names {species_names}")
print(
f"Unburned state (T,P,Y) = ({temp_unburned}, {pres_unburned}, {y_unburned}"
)
print(f"Burned state (T,P,Y) = ({temp_burned}, {pres_burned}, {y_burned}")
flame_start_loc = 0.10
flame_speed = 1000
# use the burned conditions with a lower temperature
#bulk_init = PlanarDiscontinuity(dim=dim, disc_location=flame_start_loc, sigma=0.01, nspecies=nspecies,
#temperature_left=temp_ignition, temperature_right=temp_unburned,
#pressure_left=pres_burned, pressure_right=pres_unburned,
#velocity_left=vel_burned, velocity_right=vel_unburned,
#species_mass_left=y_burned, species_mass_right=y_unburned)
bulk_init = PlanarDiscontinuity(dim=dim,
disc_location=flame_start_loc,
sigma=0.0005,
nspecies=nspecies,
temperature_right=temp_ignition,
temperature_left=temp_unburned,
pressure_right=pres_burned,
pressure_left=pres_unburned,
velocity_right=vel_burned,
velocity_left=vel_unburned,
species_mass_right=y_burned,
species_mass_left=y_unburned)
inflow_init = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=pres_burned,
temperature=temp_ignition,
massfractions=y_burned,
velocity=vel_burned)
outflow_init = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=pres_unburned,
temperature=temp_unburned,
massfractions=y_unburned,
velocity=vel_unburned)
def symmetry(nodes, eos, cv=None, **kwargs):
dim = len(nodes)
if cv is not None:
#cv = split_conserved(dim, q)
mass = cv.mass
momentum = cv.momentum
momentum[1] = -1.0 * momentum[1]
ke = .5 * np.dot(cv.momentum, cv.momentum) / cv.mass
energy = cv.energy
species_mass = cv.species_mass
return make_conserved(dim=dim,
mass=mass,
momentum=momentum,
energy=energy,
species_mass=species_mass)
def dummy(nodes, eos, cv=None, **kwargs):
dim = len(nodes)
if cv is not None:
#cv = split_conserved(dim, q)
mass = cv.mass
momentum = cv.momentum
ke = .5 * np.dot(cv.momentum, cv.momentum) / cv.mass
energy = cv.energy
species_mass = cv.species_mass
return make_conserved(dim=dim,
mass=mass,
momentum=momentum,
energy=energy,
species_mass=species_mass)
inflow = PrescribedViscousBoundary(q_func=inflow_init)
outflow = PrescribedViscousBoundary(q_func=outflow_init)
wall_symmetry = PrescribedViscousBoundary(q_func=symmetry)
wall_dummy = PrescribedViscousBoundary(q_func=dummy)
wall = PrescribedViscousBoundary(
) # essentially a "dummy" use the interior solution for the exterior
boundaries = {
DTAG_BOUNDARY("Inflow"): inflow,
DTAG_BOUNDARY("Outflow"): outflow,
#DTAG_BOUNDARY("Wall"): wall_dummy}
DTAG_BOUNDARY("Wall"): wall_symmetry
}
if restart_step is None:
char_len = 0.0001
box_ll = (0.0, 0.0)
box_ur = (0.2, 0.00125)
num_elements = (int((box_ur[0] - box_ll[0]) / char_len),
int((box_ur[1] - box_ll[1]) / char_len))
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh,
a=box_ll,
b=box_ur,
n=num_elements,
boundary_tag_to_face={
"Inflow": ["+x"],
"Outflow": ["-x"],
"Wall": ["+y", "-y"]
})
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
else: # Restart
from mirgecom.restart import read_restart_data
restart_file = restart_path + snapshot_pattern.format(
casename=restart_name, step=restart_step, rank=rank)
restart_data = read_restart_data(actx, restart_file)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert comm.Get_size() == restart_data["num_parts"]
if rank == 0:
logging.info("Making discretization")
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
if restart_step is None:
if rank == 0:
logging.info("Initializing soln.")
# for Discontinuity initial conditions
current_state = bulk_init(x_vec=nodes, eos=eos, time=0.)
# for uniform background initial condition
#current_state = bulk_init(nodes, eos=eos)
else:
current_t = restart_data["t"]
current_step = restart_step
#current_state = make_fluid_restart_state(actx, discr.discr_from_dd("vol"), restart_data["state"])
current_state = restart_data["state"]
vis_timer = None
log_cfl = LogUserQuantity(name="cfl", value=current_cfl)
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
logmgr_set_time(logmgr, current_step, current_t)
logmgr.add_quantity(log_cfl, interval=nstatus)
#logmgr_add_package_versions(logmgr)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s, "),
("cfl.max", "cfl = {value:1.4f}\n"),
("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("min_temperature", "------- T (min, max) (K) = ({value:7g}, "),
("max_temperature", "{value:7g})\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
try:
logmgr.add_watches(["memory_usage.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["pyopencl_array_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
visualizer = make_visualizer(discr)
initname = "flame1d"
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
return (
ns_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) +
eos.get_species_source_terms(cv=state))
def my_checkpoint(step, t, dt, state, force=False):
do_health = force or check_step(step, nhealth) and step > 0
do_viz = force or check_step(step, nviz)
do_restart = force or check_step(step, nrestart)
do_status = force or check_step(step, nstatus)
if do_viz or do_health:
dv = eos.dependent_vars(state)
errors = False
if do_health:
health_message = ""
if check_naninf_local(discr, "vol", dv.pressure):
errors = True
health_message += "Invalid pressure data found.\n"
elif check_range_local(discr,
"vol",
dv.pressure,
min_value=1,
max_value=2.e6):
errors = True
health_message += "Pressure data failed health check.\n"
errors = comm.allreduce(errors, MPI.LOR)
if errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
if health_message:
logger.info(f"{rank=}: {health_message}")
#if check_step(step, nrestart) and step != restart_step and not errors:
if do_restart or errors:
filename = restart_path + snapshot_pattern.format(
step=step, rank=rank, casename=casename)
restart_dictionary = {
"local_mesh": local_mesh,
"order": order,
"state": state,
"t": t,
"step": step,
"global_nelements": global_nelements,
"num_parts": nparts
}
write_restart_file(actx, restart_dictionary, filename, comm)
if do_status or do_viz or errors:
local_cfl = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state)
max_cfl = nodal_max(discr, "vol", local_cfl)
log_cfl.set_quantity(max_cfl)
#if ((check_step(step, nviz) and step != restart_step) or errors):
if do_viz or errors:
def loc_fn(t):
return flame_start_loc + flame_speed * t
#exact_soln = PlanarDiscontinuity(dim=dim, disc_location=loc_fn,
#sigma=0.0000001, nspecies=nspecies,
#temperature_left=temp_ignition, temperature_right=temp_unburned,
#pressure_left=pres_burned, pressure_right=pres_unburned,
#velocity_left=vel_burned, velocity_right=vel_unburned,
#species_mass_left=y_burned, species_mass_right=y_unburned)
reaction_rates = eos.get_production_rates(cv=state)
# conserved quantities
viz_fields = [
#("cv", state),
("CV_rho", state.mass),
("CV_rhoU", state.momentum[0]),
("CV_rhoV", state.momentum[1]),
("CV_rhoE", state.energy)
]
# species mass fractions
viz_fields.extend(
("Y_" + species_names[i], state.species_mass[i] / state.mass)
for i in range(nspecies))
# dependent variables
viz_fields.extend([
("DV", eos.dependent_vars(state)),
#("exact_soln", exact_soln),
("reaction_rates", reaction_rates),
("cfl", local_cfl)
])
write_visfile(discr,
viz_fields,
visualizer,
vizname=viz_path + casename,
step=step,
t=t,
overwrite=True,
vis_timer=vis_timer)
if errors:
raise RuntimeError("Error detected by user checkpoint, exiting.")
if rank == 0:
logging.info("Stepping.")
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t_final=t_final, t=current_t, istep=current_step,
logmgr=logmgr,eos=eos,dim=dim)
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state,
force=True)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
exit()
def main(ctx_factory=cl.create_some_context,
use_logmgr=True,
use_leap=False,
use_profiling=False,
casename=None,
rst_filename=None,
actx_class=PyOpenCLArrayContext):
"""Drive the example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
num_parts = comm.Get_size()
from mirgecom.simutil import global_reduce as _global_reduce
global_reduce = partial(_global_reduce, comm=comm)
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite",
mode="wu",
mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
# timestepping control
current_step = 0
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
t_final = 0.01
current_cfl = 1.0
current_dt = .001
current_t = 0
constant_cfl = False
# some i/o frequencies
nrestart = 10
nstatus = 1
nviz = 10
nhealth = 10
dim = 2
if dim != 2:
raise ValueError("This example must be run with dim = 2.")
rst_path = "restart_data/"
rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl")
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["num_parts"] == num_parts
else: # generate the grid from scratch
nel_1d = 16
box_ll = -5.0
box_ur = 5.0
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
order = 3
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(discr.nodes(), actx)
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
try:
logmgr.add_watches(
["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["multiply_time.max"])
# soln setup and init
eos = IdealSingleGas()
vel = np.zeros(shape=(dim, ))
orig = np.zeros(shape=(dim, ))
vel[:dim] = 1.0
initializer = Vortex2D(center=orig, velocity=vel)
gas_model = GasModel(eos=eos)
def boundary_solution(discr, btag, gas_model, state_minus, **kwargs):
actx = state_minus.array_context
bnd_discr = discr.discr_from_dd(btag)
nodes = thaw(bnd_discr.nodes(), actx)
return make_fluid_state(
initializer(x_vec=nodes, eos=gas_model.eos, **kwargs), gas_model)
boundaries = {
BTAG_ALL:
PrescribedFluidBoundary(boundary_state_func=boundary_solution)
}
if rst_filename:
current_t = restart_data["t"]
current_step = restart_data["step"]
current_cv = restart_data["cv"]
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
else:
# Set the current state from time 0
current_cv = initializer(nodes)
current_state = make_fluid_state(current_cv, gas_model)
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
def my_write_status(state, component_errors, cfl=None):
if cfl is None:
if constant_cfl:
cfl = current_cfl
else:
from grudge.op import nodal_max
from mirgecom.inviscid import get_inviscid_cfl
cfl = actx.to_numpy(
nodal_max(discr, "vol",
get_inviscid_cfl(discr, state, current_dt)))[()]
if rank == 0:
logger.info(f"------ {cfl=}\n"
"------- errors=" +
", ".join("%.3g" % en for en in component_errors))
def my_write_viz(step, t, state, dv=None, exact=None, resid=None):
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
if resid is None:
resid = state - exact
viz_fields = [("cv", state), ("dv", dv), ("exact", exact),
("residual", resid)]
from mirgecom.simutil import write_visfile
write_visfile(discr,
viz_fields,
visualizer,
vizname=casename,
step=step,
t=t,
overwrite=True,
vis_timer=vis_timer)
def my_write_restart(step, t, state):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname != rst_filename:
rst_data = {
"local_mesh": local_mesh,
"cv": state,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": num_parts
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(pressure, component_errors):
health_error = False
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", pressure) \
or check_range_local(discr, "vol", pressure, .2, 1.02):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
exittol = .1
if max(component_errors) > exittol:
health_error = True
if rank == 0:
logger.info("Solution diverged from exact soln.")
return health_error
def my_pre_step(step, t, dt, state):
fluid_state = make_fluid_state(state, gas_model)
cv = fluid_state.cv
dv = fluid_state.dv
try:
exact = None
component_errors = None
if logmgr:
logmgr.tick_before()
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
do_status = check_step(step=step, interval=nstatus)
if do_health:
exact = initializer(x_vec=nodes, eos=eos, time=t)
from mirgecom.simutil import compare_fluid_solutions
component_errors = compare_fluid_solutions(discr, cv, exact)
health_errors = global_reduce(my_health_check(
dv.pressure, component_errors),
op="lor")
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
if do_restart:
my_write_restart(step=step, t=t, state=cv)
if do_status:
if component_errors is None:
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
from mirgecom.simutil import compare_fluid_solutions
component_errors = compare_fluid_solutions(
discr, cv, exact)
my_write_status(fluid_state, component_errors)
if do_viz:
if exact is None:
exact = initializer(x_vec=nodes, eos=eos, time=t)
resid = state - exact
my_write_viz(step=step,
t=t,
state=cv,
dv=dv,
exact=exact,
resid=resid)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
my_write_viz(step=step, t=t, state=cv)
my_write_restart(step=step, t=t, state=cv)
raise
dt = get_sim_timestep(discr, fluid_state, t, dt, current_cfl, t_final,
constant_cfl)
return state, dt
def my_post_step(step, t, dt, state):
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, state, eos)
logmgr.tick_after()
return state, dt
def my_rhs(t, state):
fluid_state = make_fluid_state(state, gas_model)
return euler_operator(discr,
state=fluid_state,
time=t,
boundaries=boundaries,
gas_model=gas_model)
current_dt = get_sim_timestep(discr, current_state, current_t, current_dt,
current_cfl, t_final, constant_cfl)
current_step, current_t, current_cv = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step,
post_step_callback=my_post_step, dt=current_dt,
state=current_state.cv, t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
current_state = make_fluid_state(current_cv, gas_model)
final_dv = current_state.dv
final_exact = initializer(x_vec=nodes, eos=eos, time=current_t)
final_resid = current_state.cv - final_exact
my_write_viz(step=current_step,
t=current_t,
state=current_state.cv,
dv=final_dv,
exact=final_exact,
resid=final_resid)
my_write_restart(step=current_step, t=current_t, state=current_state.cv)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
assert np.abs(current_t - t_final) < finish_tol
def main(ctx_factory=cl.create_some_context, use_logmgr=True,
use_overintegration=False,
use_leap=False, use_profiling=False, casename=None,
rst_filename=None, actx_class=PyOpenCLArrayContext):
"""Drive the example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
num_parts = comm.Get_size()
from mirgecom.simutil import global_reduce as _global_reduce
global_reduce = partial(_global_reduce, comm=comm)
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
# timestepping control
current_step = 0
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
t_final = 0.1
current_cfl = 1.0
current_dt = .01
current_t = 0
constant_cfl = False
# some i/o frequencies
nstatus = 1
nrestart = 5
nviz = 10
nhealth = 1
dim = 3
rst_path = "restart_data/"
rst_pattern = (
rst_path + "{cname}-{step:04d}-{rank:04d}.pkl"
)
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["num_parts"] == num_parts
else: # generate the grid from scratch
from meshmode.mesh.generation import generate_regular_rect_mesh
box_ll = -1
box_ur = 1
nel_1d = 16
generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim,
b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim)
local_mesh, global_nelements = generate_and_distribute_mesh(comm,
generate_mesh)
local_nelements = local_mesh.nelements
from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD
from meshmode.discretization.poly_element import \
default_simplex_group_factory, QuadratureSimplexGroupFactory
order = 1
discr = EagerDGDiscretization(
actx, local_mesh,
discr_tag_to_group_factory={
DISCR_TAG_BASE: default_simplex_group_factory(
base_dim=local_mesh.dim, order=order),
DISCR_TAG_QUAD: QuadratureSimplexGroupFactory(2*order + 1)
},
mpi_communicator=comm
)
nodes = thaw(discr.nodes(), actx)
if use_overintegration:
quadrature_tag = DISCR_TAG_QUAD
else:
quadrature_tag = None
vis_timer = None
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging, units_for_logging)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
eos = IdealSingleGas()
gas_model = GasModel(eos=eos)
vel = np.zeros(shape=(dim,))
orig = np.zeros(shape=(dim,))
initializer = Lump(dim=dim, center=orig, velocity=vel, rhoamp=0.0)
wall = AdiabaticSlipBoundary()
boundaries = {BTAG_ALL: wall}
uniform_state = initializer(nodes)
acoustic_pulse = AcousticPulse(dim=dim, amplitude=1.0, width=.1,
center=orig)
if rst_filename:
current_t = restart_data["t"]
current_step = restart_data["step"]
current_cv = restart_data["cv"]
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
else:
# Set the current state from time 0
current_cv = acoustic_pulse(x_vec=nodes, cv=uniform_state, eos=eos)
current_state = make_fluid_state(current_cv, gas_model)
visualizer = make_visualizer(discr)
initname = "pulse"
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim, order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt, t_final=t_final, nstatus=nstatus,
nviz=nviz, cfl=current_cfl,
constant_cfl=constant_cfl, initname=initname,
eosname=eosname, casename=casename)
if rank == 0:
logger.info(init_message)
def my_write_viz(step, t, state, dv=None):
if dv is None:
dv = eos.dependent_vars(state)
viz_fields = [("cv", state),
("dv", dv)]
from mirgecom.simutil import write_visfile
write_visfile(discr, viz_fields, visualizer, vizname=casename,
step=step, t=t, overwrite=True, vis_timer=vis_timer)
def my_write_restart(step, t, state):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname != rst_filename:
rst_data = {
"local_mesh": local_mesh,
"cv": state,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": num_parts
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(pressure):
health_error = False
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", pressure) \
or check_range_local(discr, "vol", pressure, .8, 1.5):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
return health_error
def my_pre_step(step, t, dt, state):
fluid_state = make_fluid_state(state, gas_model)
dv = fluid_state.dv
try:
if logmgr:
logmgr.tick_before()
from mirgecom.simutil import check_step
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
if do_health:
health_errors = global_reduce(my_health_check(dv.pressure), op="lor")
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
if do_restart:
my_write_restart(step=step, t=t, state=state)
if do_viz:
my_write_viz(step=step, t=t, state=state, dv=dv)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
my_write_viz(step=step, t=t, state=state)
my_write_restart(step=step, t=t, state=state)
raise
dt = get_sim_timestep(discr, fluid_state, t, dt, current_cfl, t_final,
constant_cfl)
return state, dt
def my_post_step(step, t, dt, state):
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, state, eos)
logmgr.tick_after()
return state, dt
def my_rhs(t, state):
fluid_state = make_fluid_state(cv=state, gas_model=gas_model)
return euler_operator(discr, state=fluid_state, time=t,
boundaries=boundaries,
gas_model=gas_model,
quadrature_tag=quadrature_tag)
current_dt = get_sim_timestep(discr, current_state, current_t, current_dt,
current_cfl, t_final, constant_cfl)
current_step, current_t, current_cv = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step,
post_step_callback=my_post_step, dt=current_dt,
state=current_cv, t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
final_state = make_fluid_state(current_cv, gas_model)
final_dv = final_state.dv
my_write_viz(step=current_step, t=current_t, state=current_cv, dv=final_dv)
my_write_restart(step=current_step, t=current_t, state=current_cv)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
assert np.abs(current_t - t_final) < finish_tol
def main(ctx_factory=cl.create_some_context,
use_logmgr=True,
use_leap=False,
use_overintegration=False,
use_profiling=False,
casename=None,
rst_filename=None,
actx_class=PyOpenCLArrayContext,
log_dependent=True):
"""Drive example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nproc = comm.Get_size()
from mirgecom.simutil import global_reduce as _global_reduce
global_reduce = partial(_global_reduce, comm=comm)
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite",
mode="wu",
mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
# Some discretization parameters
dim = 2
nel_1d = 8
order = 1
# {{{ Time stepping control
# This example runs only 3 steps by default (to keep CI ~short)
# With the mixture defined below, equilibrium is achieved at ~40ms
# To run to equilibrium, set t_final >= 40ms.
# Time stepper selection
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
# Time loop control parameters
current_step = 0
t_final = 1e-8
current_cfl = 1.0
current_dt = 1e-9
current_t = 0
constant_cfl = False
# i.o frequencies
nstatus = 1
nviz = 5
nhealth = 1
nrestart = 5
# }}} Time stepping control
debug = False
rst_path = "restart_data/"
rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl")
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["num_parts"] == nproc
rst_time = restart_data["t"]
rst_step = restart_data["step"]
rst_order = restart_data["order"]
else: # generate the grid from scratch
from meshmode.mesh.generation import generate_regular_rect_mesh
box_ll = -0.005
box_ur = 0.005
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD
from meshmode.discretization.poly_element import \
default_simplex_group_factory, QuadratureSimplexGroupFactory
discr = EagerDGDiscretization(
actx,
local_mesh,
discr_tag_to_group_factory={
DISCR_TAG_BASE:
default_simplex_group_factory(base_dim=local_mesh.dim,
order=order),
DISCR_TAG_QUAD:
QuadratureSimplexGroupFactory(2 * order + 1)
},
mpi_communicator=comm)
nodes = thaw(discr.nodes(), actx)
ones = discr.zeros(actx) + 1.0
if use_overintegration:
quadrature_tag = DISCR_TAG_QUAD
else:
quadrature_tag = None
ones = discr.zeros(actx) + 1.0
vis_timer = None
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("t_step.max",
"------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")])
if log_dependent:
logmgr_add_many_discretization_quantities(
logmgr, discr, dim, extract_vars_for_logging,
units_for_logging)
logmgr.add_watches([
("min_pressure",
"\n------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("min_temperature",
"------- T (min, max) (K) = ({value:7g}, "),
("max_temperature", "{value:7g})\n")
])
# {{{ Set up initial state using Cantera
# Use Cantera for initialization
# -- Pick up a CTI for the thermochemistry config
# --- Note: Users may add their own CTI file by dropping it into
# --- mirgecom/mechanisms alongside the other CTI files.
from mirgecom.mechanisms import get_mechanism_cti
mech_cti = get_mechanism_cti("uiuc")
cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti)
nspecies = cantera_soln.n_species
# Initial temperature, pressure, and mixutre mole fractions are needed to
# set up the initial state in Cantera.
temperature_seed = 1500.0 # Initial temperature hot enough to burn
# Parameters for calculating the amounts of fuel, oxidizer, and inert species
equiv_ratio = 1.0
ox_di_ratio = 0.21
stoich_ratio = 3.0
# Grab the array indices for the specific species, ethylene, oxygen, and nitrogen
i_fu = cantera_soln.species_index("C2H4")
i_ox = cantera_soln.species_index("O2")
i_di = cantera_soln.species_index("N2")
x = np.zeros(nspecies)
# Set the species mole fractions according to our desired fuel/air mixture
x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio +
ox_di_ratio * equiv_ratio)
x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio
x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio
# Uncomment next line to make pylint fail when it can't find cantera.one_atm
one_atm = cantera.one_atm # pylint: disable=no-member
# one_atm = 101325.0
# Let the user know about how Cantera is being initilized
print(f"Input state (T,P,X) = ({temperature_seed}, {one_atm}, {x}")
# Set Cantera internal gas temperature, pressure, and mole fractios
cantera_soln.TPX = temperature_seed, one_atm, x
# Pull temperature, total density, mass fractions, and pressure from Cantera
# We need total density, and mass fractions to initialize the fluid/gas state.
can_t, can_rho, can_y = cantera_soln.TDY
can_p = cantera_soln.P
# *can_t*, *can_p* should not differ (significantly) from user's initial data,
# but we want to ensure that we use exactly the same starting point as Cantera,
# so we use Cantera's version of these data.
# }}}
# {{{ Create Pyrometheus thermochemistry object & EOS
# Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and
# generates a set of methods to calculate chemothermomechanical properties and
# states for this particular mechanism.
from mirgecom.thermochemistry import make_pyrometheus_mechanism_class
pyro_mechanism = make_pyrometheus_mechanism_class(cantera_soln)(actx.np)
eos = PyrometheusMixture(pyro_mechanism,
temperature_guess=temperature_seed)
gas_model = GasModel(eos=eos)
from pytools.obj_array import make_obj_array
def get_temperature_update(cv, temperature):
y = cv.species_mass_fractions
e = gas_model.eos.internal_energy(cv) / cv.mass
return pyro_mechanism.get_temperature_update_energy(e, temperature, y)
from mirgecom.gas_model import make_fluid_state
def get_fluid_state(cv, tseed):
return make_fluid_state(cv=cv,
gas_model=gas_model,
temperature_seed=tseed)
compute_temperature_update = actx.compile(get_temperature_update)
construct_fluid_state = actx.compile(get_fluid_state)
# }}}
# {{{ MIRGE-Com state initialization
# Initialize the fluid/gas state with Cantera-consistent data:
# (density, pressure, temperature, mass_fractions)
print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}")
velocity = np.zeros(shape=(dim, ))
initializer = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=can_p,
temperature=can_t,
massfractions=can_y,
velocity=velocity)
my_boundary = AdiabaticSlipBoundary()
boundaries = {BTAG_ALL: my_boundary}
if rst_filename:
current_step = rst_step
current_t = rst_time
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
if order == rst_order:
current_cv = restart_data["cv"]
temperature_seed = restart_data["temperature_seed"]
else:
rst_cv = restart_data["cv"]
old_discr = EagerDGDiscretization(actx,
local_mesh,
order=rst_order,
mpi_communicator=comm)
from meshmode.discretization.connection import make_same_mesh_connection
connection = make_same_mesh_connection(
actx, discr.discr_from_dd("vol"),
old_discr.discr_from_dd("vol"))
current_cv = connection(rst_cv)
temperature_seed = connection(restart_data["temperature_seed"])
else:
# Set the current state from time 0
current_cv = initializer(eos=gas_model.eos, x_vec=nodes)
temperature_seed = temperature_seed * ones
# The temperature_seed going into this function is:
# - At time 0: the initial temperature input data (maybe from Cantera)
# - On restart: the restarted temperature seed from restart file (saving
# the *seed* allows restarts to be deterministic
current_fluid_state = construct_fluid_state(current_cv, temperature_seed)
current_dv = current_fluid_state.dv
temperature_seed = current_dv.temperature
# Inspection at physics debugging time
if debug:
print("Initial MIRGE-Com state:")
print(f"Initial DV pressure: {current_fluid_state.pressure}")
print(f"Initial DV temperature: {current_fluid_state.temperature}")
# }}}
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = gas_model.eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
# Cantera equilibrate calculates the expected end state @ chemical equilibrium
# i.e. the expected state after all reactions
cantera_soln.equilibrate("UV")
eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY
eq_pressure = cantera_soln.P
# Report the expected final state to the user
if rank == 0:
logger.info(init_message)
logger.info(f"Expected equilibrium state:"
f" {eq_pressure=}, {eq_temperature=},"
f" {eq_density=}, {eq_mass_fractions=}")
def my_write_status(dt, cfl, dv=None):
status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}"
if ((dv is not None) and (not log_dependent)):
temp = dv.temperature
press = dv.pressure
from grudge.op import nodal_min_loc, nodal_max_loc
tmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", temp)),
comm=comm,
op=MPI.MIN)
tmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", temp)),
comm=comm,
op=MPI.MAX)
pmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", press)),
comm=comm,
op=MPI.MIN)
pmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", press)),
comm=comm,
op=MPI.MAX)
dv_status_msg = f"\nP({pmin}, {pmax}), T({tmin}, {tmax})"
status_msg = status_msg + dv_status_msg
if rank == 0:
logger.info(status_msg)
def my_write_viz(step, t, dt, state, ts_field, dv, production_rates, cfl):
viz_fields = [("cv", state), ("dv", dv),
("production_rates", production_rates),
("dt" if constant_cfl else "cfl", ts_field)]
write_visfile(discr,
viz_fields,
visualizer,
vizname=casename,
step=step,
t=t,
overwrite=True,
vis_timer=vis_timer)
def my_write_restart(step, t, state, temperature_seed):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname == rst_filename:
if rank == 0:
logger.info("Skipping overwrite of restart file.")
else:
rst_data = {
"local_mesh": local_mesh,
"cv": state.cv,
"temperature_seed": temperature_seed,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": nproc
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(cv, dv):
import grudge.op as op
health_error = False
pressure = dv.pressure
temperature = dv.temperature
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", pressure):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
if check_range_local(discr, "vol", pressure, 1e5, 2.6e5):
health_error = True
logger.info(f"{rank=}: Pressure range violation.")
if check_naninf_local(discr, "vol", temperature):
health_error = True
logger.info(f"{rank=}: Invalid temperature data found.")
if check_range_local(discr, "vol", temperature, 1.498e3, 1.6e3):
health_error = True
logger.info(f"{rank=}: Temperature range violation.")
# This check is the temperature convergence check
# The current *temperature* is what Pyrometheus gets
# after a fixed number of Newton iterations, *n_iter*.
# Calling `compute_temperature` here with *temperature*
# input as the guess returns the calculated gas temperature after
# yet another *n_iter*.
# The difference between those two temperatures is the
# temperature residual, which can be used as an indicator of
# convergence in Pyrometheus `get_temperature`.
# Note: The local max jig below works around a very long compile
# in lazy mode.
temp_resid = compute_temperature_update(cv, temperature) / temperature
temp_err = (actx.to_numpy(op.nodal_max_loc(discr, "vol", temp_resid)))
if temp_err > 1e-8:
health_error = True
logger.info(
f"{rank=}: Temperature is not converged {temp_resid=}.")
return health_error
from mirgecom.inviscid import get_inviscid_timestep
def get_dt(state):
return get_inviscid_timestep(discr, state=state)
compute_dt = actx.compile(get_dt)
from mirgecom.inviscid import get_inviscid_cfl
def get_cfl(state, dt):
return get_inviscid_cfl(discr, dt=dt, state=state)
compute_cfl = actx.compile(get_cfl)
def get_production_rates(cv, temperature):
return eos.get_production_rates(cv, temperature)
compute_production_rates = actx.compile(get_production_rates)
def my_get_timestep(t, dt, state):
# richer interface to calculate {dt,cfl} returns node-local estimates
t_remaining = max(0, t_final - t)
if constant_cfl:
ts_field = current_cfl * compute_dt(state)
from grudge.op import nodal_min_loc
dt = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", ts_field)),
comm=comm,
op=MPI.MIN)
cfl = current_cfl
else:
ts_field = compute_cfl(state, current_dt)
from grudge.op import nodal_max_loc
cfl = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", ts_field)),
comm=comm,
op=MPI.MAX)
return ts_field, cfl, min(t_remaining, dt)
def my_pre_step(step, t, dt, state):
cv, tseed = state
fluid_state = construct_fluid_state(cv, tseed)
dv = fluid_state.dv
try:
if logmgr:
logmgr.tick_before()
from mirgecom.simutil import check_step
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
do_status = check_step(step=step, interval=nstatus)
if do_health:
health_errors = global_reduce(my_health_check(cv, dv),
op="lor")
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=fluid_state)
if do_status:
my_write_status(dt=dt, cfl=cfl, dv=dv)
if do_restart:
my_write_restart(step=step,
t=t,
state=fluid_state,
temperature_seed=tseed)
if do_viz:
production_rates = compute_production_rates(
fluid_state.cv, fluid_state.temperature)
my_write_viz(step=step,
t=t,
dt=dt,
state=cv,
dv=dv,
production_rates=production_rates,
ts_field=ts_field,
cfl=cfl)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
# my_write_viz(step=step, t=t, dt=dt, state=cv)
# my_write_restart(step=step, t=t, state=fluid_state)
raise
return state, dt
def my_post_step(step, t, dt, state):
cv, tseed = state
fluid_state = construct_fluid_state(cv, tseed)
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, cv, gas_model.eos)
logmgr.tick_after()
return make_obj_array([cv, fluid_state.temperature]), dt
def my_rhs(t, state):
cv, tseed = state
from mirgecom.gas_model import make_fluid_state
fluid_state = make_fluid_state(cv=cv,
gas_model=gas_model,
temperature_seed=tseed)
return make_obj_array([
euler_operator(discr,
state=fluid_state,
time=t,
boundaries=boundaries,
gas_model=gas_model,
quadrature_tag=quadrature_tag) +
eos.get_species_source_terms(cv, fluid_state.temperature),
0 * tseed
])
current_dt = get_sim_timestep(discr, current_fluid_state, current_t,
current_dt, current_cfl, t_final,
constant_cfl)
current_step, current_t, current_state = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step,
post_step_callback=my_post_step, dt=current_dt,
state=make_obj_array([current_cv, temperature_seed]),
t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
final_cv, tseed = current_state
final_fluid_state = construct_fluid_state(final_cv, tseed)
final_dv = final_fluid_state.dv
final_dm = compute_production_rates(final_cv, final_dv.temperature)
ts_field, cfl, dt = my_get_timestep(t=current_t,
dt=current_dt,
state=final_fluid_state)
my_write_viz(step=current_step,
t=current_t,
dt=dt,
state=final_cv,
dv=final_dv,
production_rates=final_dm,
ts_field=ts_field,
cfl=cfl)
my_write_status(dt=dt, cfl=cfl, dv=final_dv)
my_write_restart(step=current_step,
t=current_t,
state=final_fluid_state,
temperature_seed=tseed)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
assert np.abs(current_t - t_final) < finish_tol
def main(ctx_factory=cl.create_some_context, use_leap=False):
"""Drive the example."""
cl_ctx = ctx_factory()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
dim = 1
nel_1d = 24
order = 1
# tolerate large errors; case is unstable
exittol = .2
t_final = 0.01
current_cfl = 1.0
current_dt = .0001
current_t = 0
eos = IdealSingleGas()
initializer = SodShock1D(dim=dim)
casename = "sod1d"
boundaries = {BTAG_ALL: PrescribedBoundary(initializer)}
constant_cfl = False
nstatus = 10
nviz = 10
rank = 0
checkpoint_t = current_t
current_step = 0
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
box_ll = -5.0
box_ur = 5.0
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
current_state = initializer(nodes)
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
return euler_operator(discr,
cv=state,
t=t,
boundaries=boundaries,
eos=eos)
def my_checkpoint(step, t, dt, state):
return sim_checkpoint(discr,
visualizer,
eos,
cv=state,
exact_soln=initializer,
vizname=casename,
step=step,
t=t,
dt=dt,
nstatus=nstatus,
nviz=nviz,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm)
try:
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t=current_t, t_final=t_final)
except ExactSolutionMismatch as ex:
current_step = ex.step
current_t = ex.t
current_state = ex.state
# if current_t != checkpoint_t:
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state)
if current_t - t_final < 0:
raise ValueError("Simulation exited abnormally")
def main(ctx_factory=cl.create_some_context,
use_profiling=False,
use_logmgr=False,
use_leap=False):
"""Drive the example."""
from mpi4py import MPI
comm = MPI.COMM_WORLD
logmgr = initialize_logmgr(use_logmgr,
filename="vortex.sqlite",
mode="wu",
mpi_comm=comm)
cl_ctx = ctx_factory()
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = PyOpenCLProfilingArrayContext(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
dim = 2
nel_1d = 16
order = 3
exittol = .1
t_final = 0.1
current_cfl = 1.0
vel = np.zeros(shape=(dim, ))
orig = np.zeros(shape=(dim, ))
vel[:dim] = 1.0
current_dt = .001
current_t = 0
eos = IdealSingleGas()
initializer = Vortex2D(center=orig, velocity=vel)
casename = "vortex"
boundaries = {BTAG_ALL: PrescribedBoundary(initializer)}
constant_cfl = False
nstatus = 10
nviz = 10
rank = 0
checkpoint_t = current_t
current_step = 0
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
box_ll = -5.0
box_ur = 5.0
rank = comm.Get_rank()
if dim != 2:
raise ValueError("This example must be run with dim = 2.")
from meshmode.mesh.generation import generate_regular_rect_mesh
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
current_state = initializer(nodes)
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
logmgr.add_watches([
"step.max", "t_step.max", "t_log.max", "min_temperature",
"L2_norm_momentum1"
])
try:
logmgr.add_watches(
["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["multiply_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
if rank == 0:
logger.info(init_message)
get_timestep = partial(inviscid_sim_timestep,
discr=discr,
t=current_t,
dt=current_dt,
cfl=current_cfl,
eos=eos,
t_final=t_final,
constant_cfl=constant_cfl)
def my_rhs(t, state):
return euler_operator(discr,
cv=state,
t=t,
boundaries=boundaries,
eos=eos)
def my_checkpoint(step, t, dt, state):
return sim_checkpoint(discr,
visualizer,
eos,
cv=state,
exact_soln=initializer,
vizname=casename,
step=step,
t=t,
dt=dt,
nstatus=nstatus,
nviz=nviz,
exittol=exittol,
constant_cfl=constant_cfl,
comm=comm,
vis_timer=vis_timer)
try:
(current_step, current_t, current_state) = \
advance_state(rhs=my_rhs, timestepper=timestepper,
checkpoint=my_checkpoint,
get_timestep=get_timestep, state=current_state,
t=current_t, t_final=t_final, logmgr=logmgr,
eos=eos, dim=dim)
except ExactSolutionMismatch as ex:
current_step = ex.step
current_t = ex.t
current_state = ex.state
# if current_t != checkpoint_t:
if rank == 0:
logger.info("Checkpointing final state ...")
my_checkpoint(current_step,
t=current_t,
dt=(current_t - checkpoint_t),
state=current_state)
if current_t - t_final < 0:
raise ValueError("Simulation exited abnormally")
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
def main(ctx_factory=cl.create_some_context, use_logmgr=True,
use_leap=False, use_profiling=False, casename=None,
rst_filename=None, actx_class=PyOpenCLArrayContext):
"""Run the example."""
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
from mpi4py import MPI
comm = MPI.COMM_WORLD
num_parts = comm.Get_size()
logmgr = initialize_logmgr(use_logmgr,
filename="heat-source.sqlite", mode="wu", mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(
queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
nel_1d = 16
t = 0
t_final = 0.0002
istep = 0
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(
a=(-0.5,)*dim,
b=(0.5,)*dim,
nelements_per_axis=(nel_1d,)*dim,
boundary_tag_to_face={
"dirichlet": ["+x", "-x"],
"neumann": ["+y", "-y"]
}
)
print("%d elements" % mesh.nelements)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts)
del mesh
else:
local_mesh = mesh_dist.receive_mesh_part()
order = 3
discr = EagerDGDiscretization(actx, local_mesh, order=order,
mpi_communicator=comm)
if dim == 2:
# no deep meaning here, just a fudge factor
dt = 0.0025/(nel_1d*order**2)
else:
raise ValueError("don't have a stable time step guesstimate")
source_width = 0.2
from arraycontext import thaw
nodes = thaw(discr.nodes(), actx)
boundaries = {
DTAG_BOUNDARY("dirichlet"): DirichletDiffusionBoundary(0.),
DTAG_BOUNDARY("neumann"): NeumannDiffusionBoundary(0.)
}
u = discr.zeros(actx)
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr.add_watches(["step.max", "t_step.max", "t_log.max"])
try:
logmgr.add_watches(["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["multiply_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
vis = make_visualizer(discr)
def rhs(t, u):
return (
diffusion_operator(
discr, quad_tag=DISCR_TAG_BASE,
alpha=1, boundaries=boundaries, u=u)
+ actx.np.exp(-np.dot(nodes, nodes)/source_width**2))
compiled_rhs = actx.compile(rhs)
rank = comm.Get_rank()
while t < t_final:
if logmgr:
logmgr.tick_before()
if istep % 10 == 0:
print(istep, t, actx.to_numpy(actx.np.linalg.norm(u[0])))
vis.write_vtk_file("fld-heat-source-mpi-%03d-%04d.vtu" % (rank, istep),
[
("u", u)
], overwrite=True)
u = rk4_step(u, t, dt, compiled_rhs)
t += dt
istep += 1
if logmgr:
set_dt(logmgr, dt)
logmgr.tick_after()
final_answer = discr.norm(u, np.inf)
resid = abs(final_answer - 0.00020620711665201585)
if resid > 1e-15:
raise ValueError(f"Run did not produce the expected result {resid=}")
def main():
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
comm = MPI.COMM_WORLD
num_parts = comm.Get_size()
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
nel_1d = 16
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim,
b=(0.5, ) * dim,
n=(nel_1d, ) * dim)
print("%d elements" % mesh.nelements)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element,
num_parts)
del mesh
else:
local_mesh = mesh_dist.receive_mesh_part()
order = 3
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
if dim == 2:
# no deep meaning here, just a fudge factor
dt = 0.75 / (nel_1d * order**2)
elif dim == 3:
# no deep meaning here, just a fudge factor
dt = 0.45 / (nel_1d * order**2)
else:
raise ValueError("don't have a stable time step guesstimate")
fields = flat_obj_array(bump(actx, discr),
[discr.zeros(actx) for i in range(discr.dim)])
vis = make_visualizer(discr, order + 3 if dim == 2 else order)
def rhs(t, w):
return wave_operator(discr, c=1, w=w)
rank = comm.Get_rank()
t = 0
t_final = 3
istep = 0
while t < t_final:
fields = rk4_step(fields, t, dt, rhs)
if istep % 10 == 0:
print(istep, t, discr.norm(fields[0]))
vis.write_vtk_file(
"fld-wave-eager-mpi-%03d-%04d.vtu" % (rank, istep), [
("u", fields[0]),
("v", fields[1:]),
])
t += dt
istep += 1