def _refine_qbx_quad_stage2(lpot_source, stage2_density_discr):
from meshmode.discretization.connection import make_same_mesh_connection
discr = _make_quad_stage2_discr(lpot_source, stage2_density_discr)
conn = make_same_mesh_connection(lpot_source._setup_actx, discr,
stage2_density_discr)
return discr, conn
def __init__(self, density_discr, fine_order, qbx_order, fmm_order,
# FIXME set debug=False once everything works
expansion_getter=None, real_dtype=np.float64, debug=True):
"""
:arg fine_order: The total degree to which the (upsampled)
underlying quadrature is exact.
:arg fmm_order: `False` for direct calculation. ``None`` will set
a reasonable(-ish?) default.
"""
self.fine_density_discr = Discretization(
density_discr.cl_context, density_discr.mesh,
QuadratureSimplexGroupFactory(fine_order), real_dtype)
from meshmode.discretization.connection import make_same_mesh_connection
with cl.CommandQueue(density_discr.cl_context) as queue:
self.resampler = make_same_mesh_connection(
queue,
self.fine_density_discr, density_discr)
if fmm_order is None:
fmm_order = qbx_order + 1
self.qbx_order = qbx_order
self.density_discr = density_discr
self.fmm_order = fmm_order
self.debug = debug
# only used in non-FMM case
if expansion_getter is None:
from sumpy.expansion.local import LineTaylorLocalExpansion
expansion_getter = LineTaylorLocalExpansion
self.expansion_getter = expansion_getter
def make_visualizer(queue, discr, vis_order):
from meshmode.discretization import Discretization
from meshmode.discretization.poly_element import \
PolynomialWarpAndBlendGroupFactory
vis_discr = Discretization(
discr.cl_context, discr.mesh,
PolynomialWarpAndBlendGroupFactory(vis_order),
real_dtype=discr.real_dtype)
from meshmode.discretization.connection import \
make_same_mesh_connection
return Visualizer(make_same_mesh_connection(vis_discr, discr))
def make_visualizer(queue, discr, vis_order):
from meshmode.discretization import Discretization
from meshmode.discretization.poly_element import \
PolynomialWarpAndBlendGroupFactory
vis_discr = Discretization(discr.cl_context,
discr.mesh,
PolynomialWarpAndBlendGroupFactory(vis_order),
real_dtype=discr.real_dtype)
from meshmode.discretization.connection import \
make_same_mesh_connection
return Visualizer(make_same_mesh_connection(vis_discr, discr))
def make_visualizer(actx, discr, vis_order=None,
element_shrink_factor=None, force_equidistant=False):
"""
:arg vis_order: order of the visualization DOFs.
:arg element_shrink_factor: number in :math:`(0, 1]`.
:arg force_equidistant: if *True*, the visualization is done on
equidistant nodes. If plotting high-order Lagrange VTK elements, this
needs to be set to *True*.
"""
from meshmode.discretization.poly_element import OrderAndTypeBasedGroupFactory
vis_discr = None
if (element_shrink_factor is None
and not force_equidistant
and vis_order is None):
vis_discr = discr
else:
if force_equidistant:
from meshmode.discretization.poly_element import (
PolynomialEquidistantSimplexElementGroup as SimplexElementGroup,
EquidistantTensorProductElementGroup as TensorElementGroup)
else:
from meshmode.discretization.poly_element import (
PolynomialWarpAndBlendElementGroup as SimplexElementGroup,
LegendreGaussLobattoTensorProductElementGroup as TensorElementGroup)
vis_discr = discr.copy(
actx=actx,
group_factory=OrderAndTypeBasedGroupFactory(
vis_order,
simplex_group_class=SimplexElementGroup,
tensor_product_group_class=TensorElementGroup),
)
if all(grp.discretization_key() == vgrp.discretization_key()
for grp, vgrp in zip(discr.groups, vis_discr.groups)):
from warnings import warn
warn("Visualization discretization is identical to base discretization. "
"To avoid the creation of a separate discretization for "
"visualization, avoid passing vis_order unless needed.",
stacklevel=2)
vis_discr = discr
from meshmode.discretization.connection import make_same_mesh_connection
return Visualizer(
make_same_mesh_connection(actx, vis_discr, discr),
element_shrink_factor=element_shrink_factor,
is_equidistant=force_equidistant)
def test_sanity_qhull_nd(ctx_getter, dim, order):
pytest.importorskip("scipy")
logging.basicConfig(level=logging.INFO)
ctx = ctx_getter()
queue = cl.CommandQueue(ctx)
from scipy.spatial import Delaunay
verts = np.random.rand(1000, dim)
dtri = Delaunay(verts)
from meshmode.mesh.io import from_vertices_and_simplices
mesh = from_vertices_and_simplices(dtri.points.T,
dtri.simplices,
fix_orientation=True)
from meshmode.discretization import Discretization
low_discr = Discretization(ctx, mesh,
PolynomialEquidistantGroupFactory(order))
high_discr = Discretization(ctx, mesh,
PolynomialEquidistantGroupFactory(order + 1))
from meshmode.discretization.connection import make_same_mesh_connection
cnx = make_same_mesh_connection(high_discr, low_discr)
def f(x):
return 0.1 * cl.clmath.sin(x)
x_low = low_discr.nodes()[0].with_queue(queue)
f_low = f(x_low)
x_high = high_discr.nodes()[0].with_queue(queue)
f_high_ref = f(x_high)
f_high_num = cnx(queue, f_low)
err = (f_high_ref - f_high_num).get()
err = la.norm(err, np.inf) / la.norm(f_high_ref.get(), np.inf)
print(err)
assert err < 1e-2
def test_sanity_qhull_nd(ctx_getter, dim, order):
pytest.importorskip("scipy")
logging.basicConfig(level=logging.INFO)
ctx = ctx_getter()
queue = cl.CommandQueue(ctx)
from scipy.spatial import Delaunay
verts = np.random.rand(1000, dim)
dtri = Delaunay(verts)
from meshmode.mesh.io import from_vertices_and_simplices
mesh = from_vertices_and_simplices(dtri.points.T, dtri.simplices,
fix_orientation=True)
from meshmode.discretization import Discretization
low_discr = Discretization(ctx, mesh,
PolynomialEquidistantGroupFactory(order))
high_discr = Discretization(ctx, mesh,
PolynomialEquidistantGroupFactory(order+1))
from meshmode.discretization.connection import make_same_mesh_connection
cnx = make_same_mesh_connection(high_discr, low_discr)
def f(x):
return 0.1*cl.clmath.sin(x)
x_low = low_discr.nodes()[0].with_queue(queue)
f_low = f(x_low)
x_high = high_discr.nodes()[0].with_queue(queue)
f_high_ref = f(x_high)
f_high_num = cnx(queue, f_low)
err = (f_high_ref-f_high_num).get()
err = la.norm(err, np.inf)/la.norm(f_high_ref.get(), np.inf)
print(err)
assert err < 1e-2
def make_visualizer(queue, discr, vis_order, element_shrink_factor=None):
from meshmode.discretization import Discretization
from meshmode.discretization.poly_element import (
PolynomialWarpAndBlendElementGroup,
LegendreGaussLobattoTensorProductElementGroup,
OrderAndTypeBasedGroupFactory)
vis_discr = Discretization(
discr.cl_context,
discr.mesh,
OrderAndTypeBasedGroupFactory(
vis_order,
simplex_group_class=PolynomialWarpAndBlendElementGroup,
tensor_product_group_class=(
LegendreGaussLobattoTensorProductElementGroup)),
real_dtype=discr.real_dtype)
from meshmode.discretization.connection import \
make_same_mesh_connection
return Visualizer(make_same_mesh_connection(vis_discr, discr),
element_shrink_factor=element_shrink_factor)
def make_visualizer(queue, discr, vis_order, element_shrink_factor=None):
from meshmode.discretization import Discretization
from meshmode.discretization.poly_element import (
PolynomialWarpAndBlendElementGroup,
LegendreGaussLobattoTensorProductElementGroup,
OrderAndTypeBasedGroupFactory)
vis_discr = Discretization(
discr.cl_context, discr.mesh,
OrderAndTypeBasedGroupFactory(
vis_order,
simplex_group_class=PolynomialWarpAndBlendElementGroup,
tensor_product_group_class=(
LegendreGaussLobattoTensorProductElementGroup)),
real_dtype=discr.real_dtype)
from meshmode.discretization.connection import \
make_same_mesh_connection
return Visualizer(
make_same_mesh_connection(vis_discr, discr),
element_shrink_factor=element_shrink_factor)
def test_sanity_qhull_nd(actx_factory, dim, order):
pytest.importorskip("scipy")
logging.basicConfig(level=logging.INFO)
actx = actx_factory()
from scipy.spatial import Delaunay # pylint: disable=no-name-in-module
verts = np.random.rand(1000, dim)
dtri = Delaunay(verts)
# pylint: disable=no-member
from meshmode.mesh.io import from_vertices_and_simplices
mesh = from_vertices_and_simplices(dtri.points.T,
dtri.simplices,
fix_orientation=True)
from meshmode.discretization import Discretization
low_discr = Discretization(actx, mesh,
PolynomialEquidistantSimplexGroupFactory(order))
high_discr = Discretization(
actx, mesh, PolynomialEquidistantSimplexGroupFactory(order + 1))
from meshmode.discretization.connection import make_same_mesh_connection
cnx = make_same_mesh_connection(actx, high_discr, low_discr)
def f(x):
return 0.1 * actx.np.sin(x)
x_low = thaw(low_discr.nodes()[0], actx)
f_low = f(x_low)
x_high = thaw(high_discr.nodes()[0], actx)
f_high_ref = f(x_high)
f_high_num = cnx(f_low)
err = (flat_norm(f_high_ref - f_high_num, np.inf) /
flat_norm(f_high_ref, np.inf))
print(err)
assert err < 1e-2
def make_visualizer(actx,
discr,
vis_order,
element_shrink_factor=None,
force_equidistant=False):
"""
:arg vis_order: order of the visualization DOFs.
:arg element_shrink_factor: number in :math:`(0, 1]`.
:arg force_equidistant: if *True*, the visualization is done on
equidistant nodes. If plotting high-order Lagrange VTK elements, this
needs to be set to *True*.
"""
from meshmode.discretization import Discretization
if force_equidistant:
from meshmode.discretization.poly_element import (
PolynomialEquidistantSimplexElementGroup as SimplexElementGroup,
EquidistantTensorProductElementGroup as TensorElementGroup)
else:
from meshmode.discretization.poly_element import (
PolynomialWarpAndBlendElementGroup as SimplexElementGroup,
LegendreGaussLobattoTensorProductElementGroup as
TensorElementGroup)
from meshmode.discretization.poly_element import OrderAndTypeBasedGroupFactory
vis_discr = Discretization(
actx,
discr.mesh,
OrderAndTypeBasedGroupFactory(
vis_order,
simplex_group_class=SimplexElementGroup,
tensor_product_group_class=TensorElementGroup),
real_dtype=discr.real_dtype)
from meshmode.discretization.connection import \
make_same_mesh_connection
return Visualizer(make_same_mesh_connection(actx, vis_discr, discr),
element_shrink_factor=element_shrink_factor,
is_equidistant=force_equidistant)
def refined_interp_to_ovsmp_quad_connection(self):
from meshmode.discretization.connection import make_same_mesh_connection
return make_same_mesh_connection(self.quad_stage2_density_discr,
self.stage2_density_discr)
def main(ctx_factory=cl.create_some_context,
use_logmgr=True,
use_leap=False,
use_profiling=False,
casename=None,
rst_filename=None,
actx_class=PyOpenCLArrayContext):
"""Drive example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nproc = comm.Get_size()
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite",
mode="wu",
mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
# Some discretization parameters
dim = 2
nel_1d = 8
order = 1
# {{{ Time stepping control
# This example runs only 3 steps by default (to keep CI ~short)
# With the mixture defined below, equilibrium is achieved at ~40ms
# To run to equlibrium, set t_final >= 40ms.
# Time stepper selection
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
# Time loop control parameters
current_step = 0
t_final = 1e-8
current_cfl = 1.0
current_dt = 1e-9
current_t = 0
constant_cfl = False
# i.o frequencies
nstatus = 1
nviz = 5
nhealth = 1
nrestart = 5
# }}} Time stepping control
debug = False
rst_path = "restart_data/"
rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl")
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["num_parts"] == nproc
rst_time = restart_data["t"]
rst_step = restart_data["step"]
rst_order = restart_data["order"]
else: # generate the grid from scratch
from meshmode.mesh.generation import generate_regular_rect_mesh
box_ll = -0.005
box_ur = 0.005
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
nodes = thaw(actx, discr.nodes())
vis_timer = None
if logmgr:
logmgr_add_device_name(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr_add_many_discretization_quantities(logmgr, discr, dim,
extract_vars_for_logging,
units_for_logging)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([
("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("min_temperature", "------- T (min, max) (K) = ({value:7g}, "),
("max_temperature", "{value:7g})\n"),
("t_step.max", "------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")
])
# {{{ Set up initial state using Cantera
# Use Cantera for initialization
# -- Pick up a CTI for the thermochemistry config
# --- Note: Users may add their own CTI file by dropping it into
# --- mirgecom/mechanisms alongside the other CTI files.
from mirgecom.mechanisms import get_mechanism_cti
mech_cti = get_mechanism_cti("uiuc")
cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti)
nspecies = cantera_soln.n_species
# Initial temperature, pressure, and mixutre mole fractions are needed to
# set up the initial state in Cantera.
init_temperature = 1500.0 # Initial temperature hot enough to burn
# Parameters for calculating the amounts of fuel, oxidizer, and inert species
equiv_ratio = 1.0
ox_di_ratio = 0.21
stoich_ratio = 3.0
# Grab the array indices for the specific species, ethylene, oxygen, and nitrogen
i_fu = cantera_soln.species_index("C2H4")
i_ox = cantera_soln.species_index("O2")
i_di = cantera_soln.species_index("N2")
x = np.zeros(nspecies)
# Set the species mole fractions according to our desired fuel/air mixture
x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio +
ox_di_ratio * equiv_ratio)
x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio
x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio
# Uncomment next line to make pylint fail when it can't find cantera.one_atm
one_atm = cantera.one_atm # pylint: disable=no-member
# one_atm = 101325.0
# Let the user know about how Cantera is being initilized
print(f"Input state (T,P,X) = ({init_temperature}, {one_atm}, {x}")
# Set Cantera internal gas temperature, pressure, and mole fractios
cantera_soln.TPX = init_temperature, one_atm, x
# Pull temperature, total density, mass fractions, and pressure from Cantera
# We need total density, and mass fractions to initialize the fluid/gas state.
can_t, can_rho, can_y = cantera_soln.TDY
can_p = cantera_soln.P
# *can_t*, *can_p* should not differ (significantly) from user's initial data,
# but we want to ensure that we use exactly the same starting point as Cantera,
# so we use Cantera's version of these data.
# }}}
# {{{ Create Pyrometheus thermochemistry object & EOS
# Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and
# generates a set of methods to calculate chemothermomechanical properties and
# states for this particular mechanism.
pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np)
eos = PyrometheusMixture(pyrometheus_mechanism,
temperature_guess=init_temperature)
# }}}
# {{{ MIRGE-Com state initialization
# Initialize the fluid/gas state with Cantera-consistent data:
# (density, pressure, temperature, mass_fractions)
print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}")
velocity = np.zeros(shape=(dim, ))
initializer = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=can_p,
temperature=can_t,
massfractions=can_y,
velocity=velocity)
my_boundary = AdiabaticSlipBoundary()
boundaries = {BTAG_ALL: my_boundary}
if rst_filename:
current_step = rst_step
current_t = rst_time
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
if order == rst_order:
current_state = restart_data["state"]
else:
rst_state = restart_data["state"]
old_discr = EagerDGDiscretization(actx,
local_mesh,
order=rst_order,
mpi_communicator=comm)
from meshmode.discretization.connection import make_same_mesh_connection
connection = make_same_mesh_connection(
actx, discr.discr_from_dd("vol"),
old_discr.discr_from_dd("vol"))
current_state = connection(rst_state)
else:
# Set the current state from time 0
current_state = initializer(eos=eos, x_vec=nodes)
# Inspection at physics debugging time
if debug:
print("Initial MIRGE-Com state:")
print(f"{current_state=}")
print(f"Initial DV pressure: {eos.pressure(current_state)}")
print(f"Initial DV temperature: {eos.temperature(current_state)}")
# }}}
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
# Cantera equilibrate calculates the expected end state @ chemical equilibrium
# i.e. the expected state after all reactions
cantera_soln.equilibrate("UV")
eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY
eq_pressure = cantera_soln.P
# Report the expected final state to the user
if rank == 0:
logger.info(init_message)
logger.info(f"Expected equilibrium state:"
f" {eq_pressure=}, {eq_temperature=},"
f" {eq_density=}, {eq_mass_fractions=}")
def my_write_status(dt, cfl):
status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}"
if rank == 0:
logger.info(status_msg)
def my_write_viz(step,
t,
dt,
state,
ts_field=None,
dv=None,
production_rates=None,
cfl=None):
if dv is None:
dv = eos.dependent_vars(state)
if production_rates is None:
production_rates = eos.get_production_rates(state)
if ts_field is None:
ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state)
viz_fields = [("cv", state), ("dv", dv),
("production_rates", production_rates),
("dt" if constant_cfl else "cfl", ts_field)]
write_visfile(discr,
viz_fields,
visualizer,
vizname=casename,
step=step,
t=t,
overwrite=True,
vis_timer=vis_timer)
def my_write_restart(step, t, state):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname == rst_filename:
if rank == 0:
logger.info("Skipping overwrite of restart file.")
else:
rst_data = {
"local_mesh": local_mesh,
"state": state,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": nproc
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(dv):
health_error = False
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", dv.pressure) \
or check_range_local(discr, "vol", dv.pressure, 1e5, 2.4e5):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
if check_range_local(discr, "vol", dv.temperature, 1.498e3, 1.52e3):
health_error = True
logger.info(f"{rank=}: Invalid temperature data found.")
return health_error
def my_get_timestep(t, dt, state):
# richer interface to calculate {dt,cfl} returns node-local estimates
t_remaining = max(0, t_final - t)
if constant_cfl:
from mirgecom.inviscid import get_inviscid_timestep
ts_field = current_cfl * get_inviscid_timestep(
discr, eos=eos, cv=state)
from grudge.op import nodal_min
dt = nodal_min(discr, "vol", ts_field)
cfl = current_cfl
else:
from mirgecom.inviscid import get_inviscid_cfl
ts_field = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state)
from grudge.op import nodal_max
cfl = nodal_max(discr, "vol", ts_field)
return ts_field, cfl, min(t_remaining, dt)
def my_pre_step(step, t, dt, state):
try:
dv = None
if logmgr:
logmgr.tick_before()
from mirgecom.simutil import check_step
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
do_status = check_step(step=step, interval=nstatus)
if do_health:
dv = eos.dependent_vars(state)
from mirgecom.simutil import allsync
health_errors = allsync(my_health_check(dv), comm, op=MPI.LOR)
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state)
if do_status:
my_write_status(dt, cfl)
if do_restart:
my_write_restart(step=step, t=t, state=state)
if do_viz:
production_rates = eos.get_production_rates(state)
if dv is None:
dv = eos.dependent_vars(state)
my_write_viz(step=step,
t=t,
dt=dt,
state=state,
dv=dv,
production_rates=production_rates,
ts_field=ts_field,
cfl=cfl)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
my_write_viz(step=step, t=t, dt=dt, state=state)
my_write_restart(step=step, t=t, state=state)
raise
return state, dt
def my_post_step(step, t, dt, state):
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, state, eos)
logmgr.tick_after()
return state, dt
def my_rhs(t, state):
return (euler_operator(
discr, cv=state, time=t, boundaries=boundaries, eos=eos) +
eos.get_species_source_terms(state))
current_dt = get_sim_timestep(discr, current_state, current_t, current_dt,
current_cfl, eos, t_final, constant_cfl)
current_step, current_t, current_state = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step,
post_step_callback=my_post_step, dt=current_dt,
state=current_state, t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
final_dv = eos.dependent_vars(current_state)
final_dm = eos.get_production_rates(current_state)
ts_field, cfl, dt = my_get_timestep(t=current_t,
dt=current_dt,
state=current_state)
my_write_viz(step=current_step,
t=current_t,
dt=dt,
state=current_state,
dv=final_dv,
production_rates=final_dm,
ts_field=ts_field,
cfl=cfl)
my_write_status(dt=dt, cfl=cfl)
my_write_restart(step=current_step, t=current_t, state=current_state)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
assert np.abs(current_t - t_final) < finish_tol
def main():
import logging
logging.basicConfig(level=logging.INFO)
ctx = cl.create_some_context()
queue = cl.CommandQueue(ctx)
mesh = generate_gmsh(
FileSource("circle.step"), 2, order=mesh_order,
force_ambient_dim=2,
other_options=["-string", "Mesh.CharacteristicLengthMax = %g;" % h]
)
logger.info("%d elements" % mesh.nelements)
# {{{ discretizations and connections
vol_discr = Discretization(ctx, mesh,
InterpolatoryQuadratureSimplexGroupFactory(vol_quad_order))
ovsmp_vol_discr = Discretization(ctx, mesh,
InterpolatoryQuadratureSimplexGroupFactory(vol_ovsmp_quad_order))
from meshmode.discretization.connection import (
make_boundary_restriction, make_same_mesh_connection)
bdry_mesh, bdry_discr, bdry_connection = make_boundary_restriction(
queue, vol_discr,
InterpolatoryQuadratureSimplexGroupFactory(bdry_quad_order))
vol_to_ovsmp_vol = make_same_mesh_connection(
queue, ovsmp_vol_discr, vol_discr)
# }}}
# {{{ visualizers
vol_vis = make_visualizer(queue, vol_discr, 20)
bdry_vis = make_visualizer(queue, bdry_discr, 20)
# }}}
vol_x = vol_discr.nodes().with_queue(queue)
ovsmp_vol_x = ovsmp_vol_discr.nodes().with_queue(queue)
rhs = rhs_func(vol_x[0], vol_x[1])
poisson_true_sol = sol_func(vol_x[0], vol_x[1])
vol_vis.write_vtk_file("volume.vtu", [("f", rhs)])
bdry_normals = bind(bdry_discr, p.normal())(queue).as_vector(dtype=object)
bdry_vis.write_vtk_file("boundary.vtu", [
("normals", bdry_normals)
])
bdry_nodes = bdry_discr.nodes().with_queue(queue)
bdry_f = rhs_func(bdry_nodes[0], bdry_nodes[1])
bdry_f_2 = bdry_connection(queue, rhs)
bdry_vis.write_vtk_file("y.vtu", [("f", bdry_f_2)])
if 0:
vol_vis.show_scalar_in_mayavi(rhs, do_show=False)
bdry_vis.show_scalar_in_mayavi(bdry_f - bdry_f_2, line_width=10,
do_show=False)
import mayavi.mlab as mlab
mlab.colorbar()
mlab.show()
# {{{ compute volume potential
from sumpy.qbx import LayerPotential
from sumpy.expansion.local import LineTaylorLocalExpansion
def get_kernel():
from sumpy.symbolic import pymbolic_real_norm_2
from pymbolic.primitives import (make_sym_vector, Variable as var)
r = pymbolic_real_norm_2(make_sym_vector("d", 3))
expr = var("log")(r)
scaling = 1/(2*var("pi"))
from sumpy.kernel import ExpressionKernel
return ExpressionKernel(
dim=3,
expression=expr,
scaling=scaling,
is_complex_valued=False)
laplace_2d_in_3d_kernel = get_kernel()
layer_pot = LayerPotential(ctx, [
LineTaylorLocalExpansion(laplace_2d_in_3d_kernel,
order=vol_qbx_order)])
targets = cl.array.zeros(queue, (3,) + vol_x.shape[1:], vol_x.dtype)
targets[:2] = vol_x
center_dist = np.min(
cl.clmath.sqrt(
bind(vol_discr, p.area_element())(queue)).get())
centers = make_obj_array([ci.copy().reshape(vol_discr.nnodes) for ci in targets])
centers[2][:] = center_dist
sources = cl.array.zeros(queue, (3,) + ovsmp_vol_x.shape[1:], ovsmp_vol_x.dtype)
sources[:2] = ovsmp_vol_x
ovsmp_rhs = vol_to_ovsmp_vol(queue, rhs)
ovsmp_vol_weights = bind(ovsmp_vol_discr, p.area_element() * p.QWeight())(queue)
evt, (vol_pot,) = layer_pot(
queue,
targets=targets.reshape(3, vol_discr.nnodes),
centers=centers,
sources=sources.reshape(3, ovsmp_vol_discr.nnodes),
strengths=(
(ovsmp_vol_weights*ovsmp_rhs).reshape(ovsmp_vol_discr.nnodes),)
)
vol_pot_bdry = bdry_connection(queue, vol_pot)
# }}}
# {{{ solve bvp
from sumpy.kernel import LaplaceKernel
from pytential.symbolic.pde.scalar import DirichletOperator
op = DirichletOperator(LaplaceKernel(2), -1, use_l2_weighting=True)
sym_sigma = sym.var("sigma")
op_sigma = op.operator(sym_sigma)
from pytential.qbx import QBXLayerPotentialSource
qbx = QBXLayerPotentialSource(
bdry_discr, fine_order=bdry_ovsmp_quad_order, qbx_order=qbx_order,
fmm_order=fmm_order
)
bound_op = bind(qbx, op_sigma)
poisson_bc = poisson_bc_func(bdry_nodes[0], bdry_nodes[1])
bvp_bc = poisson_bc - vol_pot_bdry
bdry_f = rhs_func(bdry_nodes[0], bdry_nodes[1])
bvp_rhs = bind(bdry_discr, op.prepare_rhs(sym.var("bc")))(queue, bc=bvp_bc)
from pytential.solve import gmres
gmres_result = gmres(
bound_op.scipy_op(queue, "sigma"),
bvp_rhs, tol=1e-14, progress=True,
hard_failure=False)
sigma = gmres_result.solution
print("gmres state:", gmres_result.state)
# }}}
bvp_sol = bind(
(qbx, vol_discr),
op.representation(sym_sigma))(queue, sigma=sigma)
poisson_sol = bvp_sol + vol_pot
poisson_err = poisson_sol-poisson_true_sol
rel_err = (
norm(vol_discr, queue, poisson_err)
/
norm(vol_discr, queue, poisson_true_sol))
bdry_vis.write_vtk_file("poisson-boundary.vtu", [
("vol_pot_bdry", vol_pot_bdry),
("sigma", sigma),
])
vol_vis.write_vtk_file("poisson-volume.vtu", [
("bvp_sol", bvp_sol),
("poisson_sol", poisson_sol),
("poisson_true_sol", poisson_true_sol),
("poisson_err", poisson_err),
("vol_pot", vol_pot),
("rhs", rhs),
])
print("h = %s" % h)
print("mesh_order = %s" % mesh_order)
print("vol_quad_order = %s" % vol_quad_order)
print("vol_ovsmp_quad_order = %s" % vol_ovsmp_quad_order)
print("bdry_quad_order = %s" % bdry_quad_order)
print("bdry_ovsmp_quad_order = %s" % bdry_ovsmp_quad_order)
print("qbx_order = %s" % qbx_order)
print("vol_qbx_order = %s" % vol_qbx_order)
print("fmm_order = %s" % fmm_order)
print()
print("rel err: %g" % rel_err)
def refined_interp_to_ovsmp_quad_connection(self):
from meshmode.discretization.connection import make_same_mesh_connection
return make_same_mesh_connection(
self.quad_stage2_density_discr,
self.stage2_density_discr)
def main(ctx_factory=cl.create_some_context,
use_logmgr=True,
use_leap=False,
use_overintegration=False,
use_profiling=False,
casename=None,
rst_filename=None,
actx_class=PyOpenCLArrayContext,
log_dependent=True):
"""Drive example."""
cl_ctx = ctx_factory()
if casename is None:
casename = "mirgecom"
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nproc = comm.Get_size()
from mirgecom.simutil import global_reduce as _global_reduce
global_reduce = partial(_global_reduce, comm=comm)
logmgr = initialize_logmgr(use_logmgr,
filename=f"{casename}.sqlite",
mode="wu",
mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(
cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
# Some discretization parameters
dim = 2
nel_1d = 8
order = 1
# {{{ Time stepping control
# This example runs only 3 steps by default (to keep CI ~short)
# With the mixture defined below, equilibrium is achieved at ~40ms
# To run to equilibrium, set t_final >= 40ms.
# Time stepper selection
if use_leap:
from leap.rk import RK4MethodBuilder
timestepper = RK4MethodBuilder("state")
else:
timestepper = rk4_step
# Time loop control parameters
current_step = 0
t_final = 1e-8
current_cfl = 1.0
current_dt = 1e-9
current_t = 0
constant_cfl = False
# i.o frequencies
nstatus = 1
nviz = 5
nhealth = 1
nrestart = 5
# }}} Time stepping control
debug = False
rst_path = "restart_data/"
rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl")
if rst_filename: # read the grid from restart data
rst_filename = f"{rst_filename}-{rank:04d}.pkl"
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(actx, rst_filename)
local_mesh = restart_data["local_mesh"]
local_nelements = local_mesh.nelements
global_nelements = restart_data["global_nelements"]
assert restart_data["num_parts"] == nproc
rst_time = restart_data["t"]
rst_step = restart_data["step"]
rst_order = restart_data["order"]
else: # generate the grid from scratch
from meshmode.mesh.generation import generate_regular_rect_mesh
box_ll = -0.005
box_ur = 0.005
generate_mesh = partial(generate_regular_rect_mesh,
a=(box_ll, ) * dim,
b=(box_ur, ) * dim,
nelements_per_axis=(nel_1d, ) * dim)
local_mesh, global_nelements = generate_and_distribute_mesh(
comm, generate_mesh)
local_nelements = local_mesh.nelements
from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD
from meshmode.discretization.poly_element import \
default_simplex_group_factory, QuadratureSimplexGroupFactory
discr = EagerDGDiscretization(
actx,
local_mesh,
discr_tag_to_group_factory={
DISCR_TAG_BASE:
default_simplex_group_factory(base_dim=local_mesh.dim,
order=order),
DISCR_TAG_QUAD:
QuadratureSimplexGroupFactory(2 * order + 1)
},
mpi_communicator=comm)
nodes = thaw(discr.nodes(), actx)
ones = discr.zeros(actx) + 1.0
if use_overintegration:
quadrature_tag = DISCR_TAG_QUAD
else:
quadrature_tag = None
ones = discr.zeros(actx) + 1.0
vis_timer = None
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
logmgr.add_watches([("step.max", "step = {value}, "),
("t_sim.max", "sim time: {value:1.6e} s\n"),
("t_step.max",
"------- step walltime: {value:6g} s, "),
("t_log.max", "log walltime: {value:6g} s")])
if log_dependent:
logmgr_add_many_discretization_quantities(
logmgr, discr, dim, extract_vars_for_logging,
units_for_logging)
logmgr.add_watches([
("min_pressure",
"\n------- P (min, max) (Pa) = ({value:1.9e}, "),
("max_pressure", "{value:1.9e})\n"),
("min_temperature",
"------- T (min, max) (K) = ({value:7g}, "),
("max_temperature", "{value:7g})\n")
])
# {{{ Set up initial state using Cantera
# Use Cantera for initialization
# -- Pick up a CTI for the thermochemistry config
# --- Note: Users may add their own CTI file by dropping it into
# --- mirgecom/mechanisms alongside the other CTI files.
from mirgecom.mechanisms import get_mechanism_cti
mech_cti = get_mechanism_cti("uiuc")
cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti)
nspecies = cantera_soln.n_species
# Initial temperature, pressure, and mixutre mole fractions are needed to
# set up the initial state in Cantera.
temperature_seed = 1500.0 # Initial temperature hot enough to burn
# Parameters for calculating the amounts of fuel, oxidizer, and inert species
equiv_ratio = 1.0
ox_di_ratio = 0.21
stoich_ratio = 3.0
# Grab the array indices for the specific species, ethylene, oxygen, and nitrogen
i_fu = cantera_soln.species_index("C2H4")
i_ox = cantera_soln.species_index("O2")
i_di = cantera_soln.species_index("N2")
x = np.zeros(nspecies)
# Set the species mole fractions according to our desired fuel/air mixture
x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio +
ox_di_ratio * equiv_ratio)
x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio
x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio
# Uncomment next line to make pylint fail when it can't find cantera.one_atm
one_atm = cantera.one_atm # pylint: disable=no-member
# one_atm = 101325.0
# Let the user know about how Cantera is being initilized
print(f"Input state (T,P,X) = ({temperature_seed}, {one_atm}, {x}")
# Set Cantera internal gas temperature, pressure, and mole fractios
cantera_soln.TPX = temperature_seed, one_atm, x
# Pull temperature, total density, mass fractions, and pressure from Cantera
# We need total density, and mass fractions to initialize the fluid/gas state.
can_t, can_rho, can_y = cantera_soln.TDY
can_p = cantera_soln.P
# *can_t*, *can_p* should not differ (significantly) from user's initial data,
# but we want to ensure that we use exactly the same starting point as Cantera,
# so we use Cantera's version of these data.
# }}}
# {{{ Create Pyrometheus thermochemistry object & EOS
# Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and
# generates a set of methods to calculate chemothermomechanical properties and
# states for this particular mechanism.
from mirgecom.thermochemistry import make_pyrometheus_mechanism_class
pyro_mechanism = make_pyrometheus_mechanism_class(cantera_soln)(actx.np)
eos = PyrometheusMixture(pyro_mechanism,
temperature_guess=temperature_seed)
gas_model = GasModel(eos=eos)
from pytools.obj_array import make_obj_array
def get_temperature_update(cv, temperature):
y = cv.species_mass_fractions
e = gas_model.eos.internal_energy(cv) / cv.mass
return pyro_mechanism.get_temperature_update_energy(e, temperature, y)
from mirgecom.gas_model import make_fluid_state
def get_fluid_state(cv, tseed):
return make_fluid_state(cv=cv,
gas_model=gas_model,
temperature_seed=tseed)
compute_temperature_update = actx.compile(get_temperature_update)
construct_fluid_state = actx.compile(get_fluid_state)
# }}}
# {{{ MIRGE-Com state initialization
# Initialize the fluid/gas state with Cantera-consistent data:
# (density, pressure, temperature, mass_fractions)
print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}")
velocity = np.zeros(shape=(dim, ))
initializer = MixtureInitializer(dim=dim,
nspecies=nspecies,
pressure=can_p,
temperature=can_t,
massfractions=can_y,
velocity=velocity)
my_boundary = AdiabaticSlipBoundary()
boundaries = {BTAG_ALL: my_boundary}
if rst_filename:
current_step = rst_step
current_t = rst_time
if logmgr:
from mirgecom.logging_quantities import logmgr_set_time
logmgr_set_time(logmgr, current_step, current_t)
if order == rst_order:
current_cv = restart_data["cv"]
temperature_seed = restart_data["temperature_seed"]
else:
rst_cv = restart_data["cv"]
old_discr = EagerDGDiscretization(actx,
local_mesh,
order=rst_order,
mpi_communicator=comm)
from meshmode.discretization.connection import make_same_mesh_connection
connection = make_same_mesh_connection(
actx, discr.discr_from_dd("vol"),
old_discr.discr_from_dd("vol"))
current_cv = connection(rst_cv)
temperature_seed = connection(restart_data["temperature_seed"])
else:
# Set the current state from time 0
current_cv = initializer(eos=gas_model.eos, x_vec=nodes)
temperature_seed = temperature_seed * ones
# The temperature_seed going into this function is:
# - At time 0: the initial temperature input data (maybe from Cantera)
# - On restart: the restarted temperature seed from restart file (saving
# the *seed* allows restarts to be deterministic
current_fluid_state = construct_fluid_state(current_cv, temperature_seed)
current_dv = current_fluid_state.dv
temperature_seed = current_dv.temperature
# Inspection at physics debugging time
if debug:
print("Initial MIRGE-Com state:")
print(f"Initial DV pressure: {current_fluid_state.pressure}")
print(f"Initial DV temperature: {current_fluid_state.temperature}")
# }}}
visualizer = make_visualizer(discr)
initname = initializer.__class__.__name__
eosname = gas_model.eos.__class__.__name__
init_message = make_init_message(dim=dim,
order=order,
nelements=local_nelements,
global_nelements=global_nelements,
dt=current_dt,
t_final=t_final,
nstatus=nstatus,
nviz=nviz,
cfl=current_cfl,
constant_cfl=constant_cfl,
initname=initname,
eosname=eosname,
casename=casename)
# Cantera equilibrate calculates the expected end state @ chemical equilibrium
# i.e. the expected state after all reactions
cantera_soln.equilibrate("UV")
eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY
eq_pressure = cantera_soln.P
# Report the expected final state to the user
if rank == 0:
logger.info(init_message)
logger.info(f"Expected equilibrium state:"
f" {eq_pressure=}, {eq_temperature=},"
f" {eq_density=}, {eq_mass_fractions=}")
def my_write_status(dt, cfl, dv=None):
status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}"
if ((dv is not None) and (not log_dependent)):
temp = dv.temperature
press = dv.pressure
from grudge.op import nodal_min_loc, nodal_max_loc
tmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", temp)),
comm=comm,
op=MPI.MIN)
tmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", temp)),
comm=comm,
op=MPI.MAX)
pmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", press)),
comm=comm,
op=MPI.MIN)
pmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", press)),
comm=comm,
op=MPI.MAX)
dv_status_msg = f"\nP({pmin}, {pmax}), T({tmin}, {tmax})"
status_msg = status_msg + dv_status_msg
if rank == 0:
logger.info(status_msg)
def my_write_viz(step, t, dt, state, ts_field, dv, production_rates, cfl):
viz_fields = [("cv", state), ("dv", dv),
("production_rates", production_rates),
("dt" if constant_cfl else "cfl", ts_field)]
write_visfile(discr,
viz_fields,
visualizer,
vizname=casename,
step=step,
t=t,
overwrite=True,
vis_timer=vis_timer)
def my_write_restart(step, t, state, temperature_seed):
rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank)
if rst_fname == rst_filename:
if rank == 0:
logger.info("Skipping overwrite of restart file.")
else:
rst_data = {
"local_mesh": local_mesh,
"cv": state.cv,
"temperature_seed": temperature_seed,
"t": t,
"step": step,
"order": order,
"global_nelements": global_nelements,
"num_parts": nproc
}
from mirgecom.restart import write_restart_file
write_restart_file(actx, rst_data, rst_fname, comm)
def my_health_check(cv, dv):
import grudge.op as op
health_error = False
pressure = dv.pressure
temperature = dv.temperature
from mirgecom.simutil import check_naninf_local, check_range_local
if check_naninf_local(discr, "vol", pressure):
health_error = True
logger.info(f"{rank=}: Invalid pressure data found.")
if check_range_local(discr, "vol", pressure, 1e5, 2.6e5):
health_error = True
logger.info(f"{rank=}: Pressure range violation.")
if check_naninf_local(discr, "vol", temperature):
health_error = True
logger.info(f"{rank=}: Invalid temperature data found.")
if check_range_local(discr, "vol", temperature, 1.498e3, 1.6e3):
health_error = True
logger.info(f"{rank=}: Temperature range violation.")
# This check is the temperature convergence check
# The current *temperature* is what Pyrometheus gets
# after a fixed number of Newton iterations, *n_iter*.
# Calling `compute_temperature` here with *temperature*
# input as the guess returns the calculated gas temperature after
# yet another *n_iter*.
# The difference between those two temperatures is the
# temperature residual, which can be used as an indicator of
# convergence in Pyrometheus `get_temperature`.
# Note: The local max jig below works around a very long compile
# in lazy mode.
temp_resid = compute_temperature_update(cv, temperature) / temperature
temp_err = (actx.to_numpy(op.nodal_max_loc(discr, "vol", temp_resid)))
if temp_err > 1e-8:
health_error = True
logger.info(
f"{rank=}: Temperature is not converged {temp_resid=}.")
return health_error
from mirgecom.inviscid import get_inviscid_timestep
def get_dt(state):
return get_inviscid_timestep(discr, state=state)
compute_dt = actx.compile(get_dt)
from mirgecom.inviscid import get_inviscid_cfl
def get_cfl(state, dt):
return get_inviscid_cfl(discr, dt=dt, state=state)
compute_cfl = actx.compile(get_cfl)
def get_production_rates(cv, temperature):
return eos.get_production_rates(cv, temperature)
compute_production_rates = actx.compile(get_production_rates)
def my_get_timestep(t, dt, state):
# richer interface to calculate {dt,cfl} returns node-local estimates
t_remaining = max(0, t_final - t)
if constant_cfl:
ts_field = current_cfl * compute_dt(state)
from grudge.op import nodal_min_loc
dt = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", ts_field)),
comm=comm,
op=MPI.MIN)
cfl = current_cfl
else:
ts_field = compute_cfl(state, current_dt)
from grudge.op import nodal_max_loc
cfl = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", ts_field)),
comm=comm,
op=MPI.MAX)
return ts_field, cfl, min(t_remaining, dt)
def my_pre_step(step, t, dt, state):
cv, tseed = state
fluid_state = construct_fluid_state(cv, tseed)
dv = fluid_state.dv
try:
if logmgr:
logmgr.tick_before()
from mirgecom.simutil import check_step
do_viz = check_step(step=step, interval=nviz)
do_restart = check_step(step=step, interval=nrestart)
do_health = check_step(step=step, interval=nhealth)
do_status = check_step(step=step, interval=nstatus)
if do_health:
health_errors = global_reduce(my_health_check(cv, dv),
op="lor")
if health_errors:
if rank == 0:
logger.info("Fluid solution failed health check.")
raise MyRuntimeError("Failed simulation health check.")
ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=fluid_state)
if do_status:
my_write_status(dt=dt, cfl=cfl, dv=dv)
if do_restart:
my_write_restart(step=step,
t=t,
state=fluid_state,
temperature_seed=tseed)
if do_viz:
production_rates = compute_production_rates(
fluid_state.cv, fluid_state.temperature)
my_write_viz(step=step,
t=t,
dt=dt,
state=cv,
dv=dv,
production_rates=production_rates,
ts_field=ts_field,
cfl=cfl)
except MyRuntimeError:
if rank == 0:
logger.info("Errors detected; attempting graceful exit.")
# my_write_viz(step=step, t=t, dt=dt, state=cv)
# my_write_restart(step=step, t=t, state=fluid_state)
raise
return state, dt
def my_post_step(step, t, dt, state):
cv, tseed = state
fluid_state = construct_fluid_state(cv, tseed)
# Logmgr needs to know about EOS, dt, dim?
# imo this is a design/scope flaw
if logmgr:
set_dt(logmgr, dt)
set_sim_state(logmgr, dim, cv, gas_model.eos)
logmgr.tick_after()
return make_obj_array([cv, fluid_state.temperature]), dt
def my_rhs(t, state):
cv, tseed = state
from mirgecom.gas_model import make_fluid_state
fluid_state = make_fluid_state(cv=cv,
gas_model=gas_model,
temperature_seed=tseed)
return make_obj_array([
euler_operator(discr,
state=fluid_state,
time=t,
boundaries=boundaries,
gas_model=gas_model,
quadrature_tag=quadrature_tag) +
eos.get_species_source_terms(cv, fluid_state.temperature),
0 * tseed
])
current_dt = get_sim_timestep(discr, current_fluid_state, current_t,
current_dt, current_cfl, t_final,
constant_cfl)
current_step, current_t, current_state = \
advance_state(rhs=my_rhs, timestepper=timestepper,
pre_step_callback=my_pre_step,
post_step_callback=my_post_step, dt=current_dt,
state=make_obj_array([current_cv, temperature_seed]),
t=current_t, t_final=t_final)
# Dump the final data
if rank == 0:
logger.info("Checkpointing final state ...")
final_cv, tseed = current_state
final_fluid_state = construct_fluid_state(final_cv, tseed)
final_dv = final_fluid_state.dv
final_dm = compute_production_rates(final_cv, final_dv.temperature)
ts_field, cfl, dt = my_get_timestep(t=current_t,
dt=current_dt,
state=final_fluid_state)
my_write_viz(step=current_step,
t=current_t,
dt=dt,
state=final_cv,
dv=final_dv,
production_rates=final_dm,
ts_field=ts_field,
cfl=cfl)
my_write_status(dt=dt, cfl=cfl, dv=final_dv)
my_write_restart(step=current_step,
t=current_t,
state=final_fluid_state,
temperature_seed=tseed)
if logmgr:
logmgr.close()
elif use_profiling:
print(actx.tabulate_profiling_data())
finish_tol = 1e-16
assert np.abs(current_t - t_final) < finish_tol
def main(snapshot_pattern="wave-mpi-{step:04d}-{rank:04d}.pkl", restart_step=None,
use_profiling=False, use_logmgr=False, actx_class=PyOpenCLArrayContext):
"""Drive the example."""
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
num_parts = comm.Get_size()
logmgr = initialize_logmgr(use_logmgr,
filename="wave-mpi.sqlite", mode="wu", mpi_comm=comm)
if use_profiling:
queue = cl.CommandQueue(cl_ctx,
properties=cl.command_queue_properties.PROFILING_ENABLE)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)),
logmgr=logmgr)
else:
queue = cl.CommandQueue(cl_ctx)
actx = actx_class(queue,
allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)))
if restart_step is None:
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
nel_1d = 16
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(
a=(-0.5,)*dim, b=(0.5,)*dim,
nelements_per_axis=(nel_1d,)*dim)
print("%d elements" % mesh.nelements)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts)
del mesh
else:
local_mesh = mesh_dist.receive_mesh_part()
fields = None
else:
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(
actx, snapshot_pattern.format(step=restart_step, rank=rank)
)
local_mesh = restart_data["local_mesh"]
nel_1d = restart_data["nel_1d"]
assert comm.Get_size() == restart_data["num_parts"]
order = 3
discr = EagerDGDiscretization(actx, local_mesh, order=order,
mpi_communicator=comm)
current_cfl = 0.485
wave_speed = 1.0
from grudge.dt_utils import characteristic_lengthscales
dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed
from grudge.op import nodal_min
dt = nodal_min(discr, "vol", dt)
t_final = 1
if restart_step is None:
t = 0
istep = 0
fields = flat_obj_array(
bump(actx, discr),
[discr.zeros(actx) for i in range(discr.dim)]
)
else:
t = restart_data["t"]
istep = restart_step
assert istep == restart_step
restart_fields = restart_data["fields"]
old_order = restart_data["order"]
if old_order != order:
old_discr = EagerDGDiscretization(actx, local_mesh, order=old_order,
mpi_communicator=comm)
from meshmode.discretization.connection import make_same_mesh_connection
connection = make_same_mesh_connection(actx, discr.discr_from_dd("vol"),
old_discr.discr_from_dd("vol"))
fields = connection(restart_fields)
else:
fields = restart_fields
if logmgr:
logmgr_add_cl_device_info(logmgr, queue)
logmgr_add_device_memory_usage(logmgr, queue)
logmgr.add_watches(["step.max", "t_step.max", "t_log.max"])
try:
logmgr.add_watches(["memory_usage_python.max", "memory_usage_gpu.max"])
except KeyError:
pass
if use_profiling:
logmgr.add_watches(["multiply_time.max"])
vis_timer = IntervalTimer("t_vis", "Time spent visualizing")
logmgr.add_quantity(vis_timer)
vis = make_visualizer(discr)
def rhs(t, w):
return wave_operator(discr, c=wave_speed, w=w)
compiled_rhs = actx.compile(rhs)
while t < t_final:
if logmgr:
logmgr.tick_before()
# restart must happen at beginning of step
if istep % 100 == 0 and (
# Do not overwrite the restart file that we just read.
istep != restart_step):
from mirgecom.restart import write_restart_file
write_restart_file(
actx, restart_data={
"local_mesh": local_mesh,
"order": order,
"fields": fields,
"t": t,
"step": istep,
"nel_1d": nel_1d,
"num_parts": num_parts},
filename=snapshot_pattern.format(step=istep, rank=rank),
comm=comm
)
if istep % 10 == 0:
print(istep, t, discr.norm(fields[0]))
vis.write_parallel_vtk_file(
comm,
"fld-wave-mpi-%03d-%04d.vtu" % (rank, istep),
[
("u", fields[0]),
("v", fields[1:]),
], overwrite=True
)
fields = thaw(freeze(fields, actx), actx)
fields = rk4_step(fields, t, dt, compiled_rhs)
t += dt
istep += 1
if logmgr:
set_dt(logmgr, dt)
logmgr.tick_after()
final_soln = discr.norm(fields[0])
assert np.abs(final_soln - 0.04409852463947439) < 1e-14
def refine_for_global_qbx(lpot_source,
wrangler,
group_factory,
kernel_length_scale=None,
force_stage2_uniform_refinement_rounds=None,
scaled_max_curvature_threshold=None,
debug=None,
maxiter=None,
visualize=None,
expansion_disturbance_tolerance=None,
refiner=None):
"""
Entry point for calling the refiner.
:arg lpot_source: An instance of :class:`QBXLayerPotentialSource`.
:arg wrangler: An instance of :class:`RefinerWrangler`.
:arg group_factory: An instance of
:class:`meshmode.mesh.discretization.ElementGroupFactory`. Used for
discretizing the coarse refined mesh.
:arg kernel_length_scale: The kernel length scale, or *None* if not
applicable. All panels are refined to below this size.
:arg maxiter: The maximum number of refiner iterations.
:returns: A tuple ``(lpot_source, *conn*)`` where ``lpot_source`` is the
refined layer potential source, and ``conn`` is a
:class:`meshmode.discretization.connection.DiscretizationConnection`
going from the original mesh to the refined mesh.
"""
if maxiter is None:
maxiter = 10
if debug is None:
# FIXME: Set debug=False by default once everything works.
debug = True
if expansion_disturbance_tolerance is None:
expansion_disturbance_tolerance = 0.025
if force_stage2_uniform_refinement_rounds is None:
force_stage2_uniform_refinement_rounds = 0
# TODO: Stop doing redundant checks by avoiding panels which no longer need
# refinement.
from meshmode.mesh.refinement import RefinerWithoutAdjacency
from meshmode.discretization.connection import (
ChainedDiscretizationConnection, make_same_mesh_connection)
if refiner is not None:
assert refiner.get_current_mesh() == lpot_source.density_discr.mesh
else:
# We may be handed a mesh that's already non-conforming, we don't rely
# on adjacency, and the no-adjacency refiner is faster.
refiner = RefinerWithoutAdjacency(lpot_source.density_discr.mesh)
connections = []
# {{{ first stage refinement
def visualize_refinement(niter, stage_nr, stage_name, flags):
if not visualize:
return
if stage_nr == 1:
discr = lpot_source.density_discr
elif stage_nr == 2:
discr = lpot_source.stage2_density_discr
else:
raise ValueError("unexpected stage number")
flags = flags.get()
logger.info("for stage %s: splitting %d/%d stage-%d elements",
stage_name, np.sum(flags), discr.mesh.nelements, stage_nr)
from meshmode.discretization.visualization import make_visualizer
vis = make_visualizer(wrangler.queue, discr, 3)
assert len(flags) == discr.mesh.nelements
flags = flags.astype(np.bool)
nodes_flags = np.zeros(discr.nnodes)
for grp in discr.groups:
meg = grp.mesh_el_group
grp.view(nodes_flags)[flags[meg.element_nr_base:meg.nelements +
meg.element_nr_base]] = 1
nodes_flags = cl.array.to_device(wrangler.queue, nodes_flags)
vis_data = [
("refine_flags", nodes_flags),
]
if 0:
from pytential import sym, bind
bdry_normals = bind(discr, sym.normal(discr.ambient_dim))(
wrangler.queue).as_vector(dtype=object)
vis_data.append(("bdry_normals", bdry_normals), )
vis.write_vtk_file("refinement-%s-%03d.vtu" % (stage_name, niter),
vis_data)
def warn_max_iterations():
from warnings import warn
warn(
"QBX layer potential source refiner did not terminate "
"after %d iterations (the maximum). "
"You may pass 'visualize=True' to with_refinement() "
"to see what area of the geometry is causing trouble. "
"If the issue is disturbance of expansion disks, you may "
"pass a slightly increased value (currently: %g) for "
"_expansion_disturbance_tolerance in with_refinement(). "
"As a last resort, "
"you may use Python's warning filtering mechanism to "
"not treat this warning as an error. "
"The criteria triggering refinement in each iteration "
"were: %s. " %
(len(violated_criteria), expansion_disturbance_tolerance,
", ".join("%d: %s" % (i + 1, vc_text)
for i, vc_text in enumerate(violated_criteria))),
RefinerNotConvergedWarning)
violated_criteria = []
iter_violated_criteria = ["start"]
niter = 0
while iter_violated_criteria:
iter_violated_criteria = []
niter += 1
if niter > maxiter:
warn_max_iterations()
break
refine_flags = make_empty_refine_flags(wrangler.queue, lpot_source)
if kernel_length_scale is not None:
with ProcessLogger(
logger,
"checking kernel length scale to panel size ratio"):
from pytential import bind, sym
quad_resolution = bind(
lpot_source,
sym._quad_resolution(lpot_source.ambient_dim,
dofdesc=sym.GRANULARITY_ELEMENT))(
wrangler.queue)
violates_kernel_length_scale = \
wrangler.check_element_prop_threshold(
element_property=quad_resolution,
threshold=kernel_length_scale,
refine_flags=refine_flags, debug=debug)
if violates_kernel_length_scale:
iter_violated_criteria.append("kernel length scale")
visualize_refinement(niter, 1, "kernel-length-scale",
refine_flags)
if scaled_max_curvature_threshold is not None:
with ProcessLogger(logger,
"checking scaled max curvature threshold"):
from pytential import sym, bind
scaled_max_curv = bind(
lpot_source,
sym.ElementwiseMax(
sym._scaled_max_curvature(lpot_source.ambient_dim),
dofdesc=sym.GRANULARITY_ELEMENT))(wrangler.queue)
violates_scaled_max_curv = \
wrangler.check_element_prop_threshold(
element_property=scaled_max_curv,
threshold=scaled_max_curvature_threshold,
refine_flags=refine_flags, debug=debug)
if violates_scaled_max_curv:
iter_violated_criteria.append("curvature")
visualize_refinement(niter, 1, "curvature", refine_flags)
if not iter_violated_criteria:
# Only start building trees once the simple length-based criteria
# are happy.
# Build tree and auxiliary data.
# FIXME: The tree should not have to be rebuilt at each iteration.
tree = wrangler.build_tree(lpot_source)
peer_lists = wrangler.find_peer_lists(tree)
has_disturbed_expansions = \
wrangler.check_expansion_disks_undisturbed_by_sources(
lpot_source, tree, peer_lists,
expansion_disturbance_tolerance,
refine_flags, debug)
if has_disturbed_expansions:
iter_violated_criteria.append("disturbed expansions")
visualize_refinement(niter, 1, "disturbed-expansions",
refine_flags)
del tree
del peer_lists
if iter_violated_criteria:
violated_criteria.append(" and ".join(iter_violated_criteria))
conn = wrangler.refine(lpot_source.density_discr, refiner,
refine_flags, group_factory, debug)
connections.append(conn)
lpot_source = lpot_source.copy(density_discr=conn.to_discr)
del refine_flags
# }}}
# {{{ second stage refinement
iter_violated_criteria = ["start"]
niter = 0
fine_connections = []
stage2_density_discr = lpot_source.density_discr
while iter_violated_criteria:
iter_violated_criteria = []
niter += 1
if niter > maxiter:
warn_max_iterations()
break
# Build tree and auxiliary data.
# FIXME: The tree should not have to be rebuilt at each iteration.
tree = wrangler.build_tree(lpot_source, use_stage2_discr=True)
peer_lists = wrangler.find_peer_lists(tree)
refine_flags = make_empty_refine_flags(wrangler.queue,
lpot_source,
use_stage2_discr=True)
has_insufficient_quad_res = \
wrangler.check_sufficient_source_quadrature_resolution(
lpot_source, tree, peer_lists, refine_flags, debug)
if has_insufficient_quad_res:
iter_violated_criteria.append("insufficient quadrature resolution")
visualize_refinement(niter, 2, "quad-resolution", refine_flags)
if iter_violated_criteria:
violated_criteria.append(" and ".join(iter_violated_criteria))
conn = wrangler.refine(stage2_density_discr, refiner, refine_flags,
group_factory, debug)
stage2_density_discr = conn.to_discr
fine_connections.append(conn)
lpot_source = lpot_source.copy(
to_refined_connection=ChainedDiscretizationConnection(
fine_connections))
del tree
del refine_flags
del peer_lists
for round in range(force_stage2_uniform_refinement_rounds):
conn = wrangler.refine(
stage2_density_discr, refiner,
np.ones(stage2_density_discr.mesh.nelements, dtype=np.bool),
group_factory, debug)
stage2_density_discr = conn.to_discr
fine_connections.append(conn)
lpot_source = lpot_source.copy(
to_refined_connection=ChainedDiscretizationConnection(
fine_connections))
# }}}
lpot_source = lpot_source.copy(debug=debug, _refined_for_global_qbx=True)
if len(connections) == 0:
# FIXME: This is inefficient
connection = make_same_mesh_connection(lpot_source.density_discr,
lpot_source.density_discr)
else:
connection = ChainedDiscretizationConnection(connections)
return lpot_source, connection
def flatten_chained_connection(queue, connection):
"""Collapse a connection into a direct connection.
If the given connection is already a
:class:`~meshmode.discretization.connection.DirectDiscretizationConnection`
nothing is done. However, if the connection is a
:class:`~meshmode.discretization.connection.ChainedDiscretizationConnection`,
a new direct connection is constructed that transports from
:attr:`connection.from_discr` to :attr:`connection.to_discr`.
The new direct connection will have a number of groups and batches that
is, at worse, the product of all the connections in the chain. For
example, if we consider a connection between a discretization and a
two-level refinement, both levels will have :math:`n` groups and
:math:`m + 1` batches per group, where :math:`m` is the number of
subdivisions of an element (exact number depends on implementation
details in
:func:`~meshmode.discretization.connection.make_refinement_connection`).
However, a direct connection from level :math:`0` to level :math:`2`
will have at worst :math:`n^2` groups and each group will have
:math:`(m + 1)^2` batches.
.. warning::
If a large number of connections is chained, the number of groups and
batches can become very large.
:arg queue: An instance of :class:`pyopencl.CommandQueue`.
:arg connection: An instance of
:class:`~meshmode.discretization.connection.DiscretizationConnection`.
:return: An instance of
:class:`~meshmode.discretization.connection.DirectDiscretizationConnection`.
"""
from meshmode.discretization.connection import (
DirectDiscretizationConnection, DiscretizationConnectionElementGroup,
make_same_mesh_connection)
if not hasattr(connection, 'connections'):
return connection
if not connection.connections:
return make_same_mesh_connection(connection.to_discr,
connection.from_discr)
# recursively build direct connections
connections = connection.connections
direct_connections = []
for conn in connections:
direct_connections.append(flatten_chained_connection(queue, conn))
# merge all the direct connections
from_conn = direct_connections[0]
for to_conn in direct_connections[1:]:
el_table = _build_element_lookup_table(queue, from_conn)
grp_to_grp, batch_info = _build_new_group_table(from_conn, to_conn)
# distribute the indices to new groups and batches
from_bins = [[np.empty(0, dtype=np.int) for _ in g]
for g in batch_info]
to_bins = [[np.empty(0, dtype=np.int) for _ in g] for g in batch_info]
for (igrp, ibatch), (_, from_batch) in _iterbatches(from_conn.groups):
from_to_element_indices = from_batch.to_element_indices.get(queue)
for (jgrp, jbatch), (_, to_batch) in _iterbatches(to_conn.groups):
igrp_new, ibatch_new = grp_to_grp[igrp, ibatch, jgrp, jbatch]
jfrom = to_batch.from_element_indices.get(queue)
jto = to_batch.to_element_indices.get(queue)
mask = np.isin(jfrom, from_to_element_indices)
from_bins[igrp_new][ibatch_new] = \
np.hstack([from_bins[igrp_new][ibatch_new],
el_table[igrp][jfrom[mask]]])
to_bins[igrp_new][ibatch_new] = \
np.hstack([to_bins[igrp_new][ibatch_new],
jto[mask]])
# build new groups
groups = []
for igrp, (from_bin, to_bin) in enumerate(zip(from_bins, to_bins)):
groups.append(
DiscretizationConnectionElementGroup(
list(
_build_batches(queue, from_bin, to_bin,
batch_info[igrp]))))
from_conn = DirectDiscretizationConnection(
from_discr=from_conn.from_discr,
to_discr=to_conn.to_discr,
groups=groups,
is_surjective=connection.is_surjective)
return from_conn
def main(snapshot_pattern="wave-eager-{step:04d}-{rank:04d}.pkl",
restart_step=None):
"""Drive the example."""
cl_ctx = cl.create_some_context()
queue = cl.CommandQueue(cl_ctx)
actx = PyOpenCLArrayContext(queue,
allocator=cl_tools.MemoryPool(
cl_tools.ImmediateAllocator(queue)))
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
num_parts = comm.Get_size()
if restart_step is None:
from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis
mesh_dist = MPIMeshDistributor(comm)
dim = 2
nel_1d = 16
if mesh_dist.is_mananger_rank():
from meshmode.mesh.generation import generate_regular_rect_mesh
mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim,
b=(0.5, ) * dim,
nelements_per_axis=(nel_1d, ) *
dim)
print("%d elements" % mesh.nelements)
part_per_element = get_partition_by_pymetis(mesh, num_parts)
local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element,
num_parts)
del mesh
else:
local_mesh = mesh_dist.receive_mesh_part()
fields = None
else:
from mirgecom.restart import read_restart_data
restart_data = read_restart_data(
actx, snapshot_pattern.format(step=restart_step, rank=rank))
local_mesh = restart_data["local_mesh"]
nel_1d = restart_data["nel_1d"]
assert comm.Get_size() == restart_data["num_parts"]
order = 3
discr = EagerDGDiscretization(actx,
local_mesh,
order=order,
mpi_communicator=comm)
if local_mesh.dim == 2:
# no deep meaning here, just a fudge factor
dt = 0.7 / (nel_1d * order**2)
elif dim == 3:
# no deep meaning here, just a fudge factor
dt = 0.4 / (nel_1d * order**2)
else:
raise ValueError("don't have a stable time step guesstimate")
t_final = 3
if restart_step is None:
t = 0
istep = 0
fields = flat_obj_array(bump(actx, discr),
[discr.zeros(actx) for i in range(discr.dim)])
else:
t = restart_data["t"]
istep = restart_step
assert istep == restart_step
restart_fields = restart_data["fields"]
old_order = restart_data["order"]
if old_order != order:
old_discr = EagerDGDiscretization(actx,
local_mesh,
order=old_order,
mpi_communicator=comm)
from meshmode.discretization.connection import make_same_mesh_connection
connection = make_same_mesh_connection(
actx, discr.discr_from_dd("vol"),
old_discr.discr_from_dd("vol"))
fields = connection(restart_fields)
else:
fields = restart_fields
vis = make_visualizer(discr)
def rhs(t, w):
return wave_operator(discr, c=1, w=w)
while t < t_final:
# restart must happen at beginning of step
if istep % 100 == 0 and (
# Do not overwrite the restart file that we just read.
istep != restart_step):
from mirgecom.restart import write_restart_file
write_restart_file(actx,
restart_data={
"local_mesh": local_mesh,
"order": order,
"fields": fields,
"t": t,
"step": istep,
"nel_1d": nel_1d,
"num_parts": num_parts
},
filename=snapshot_pattern.format(step=istep,
rank=rank),
comm=comm)
if istep % 10 == 0:
print(istep, t, discr.norm(fields[0]))
vis.write_parallel_vtk_file(
comm, "fld-wave-eager-mpi-%03d-%04d.vtu" % (rank, istep), [
("u", fields[0]),
("v", fields[1:]),
])
fields = rk4_step(fields, t, dt, rhs)
t += dt
istep += 1
def main():
import logging
logging.basicConfig(level=logging.INFO)
ctx = cl.create_some_context()
queue = cl.CommandQueue(ctx)
if 1:
ext = 0.5
mesh = generate_regular_rect_mesh(a=(-ext / 2, -ext / 2),
b=(ext / 2, ext / 2),
n=(int(ext / h), int(ext / h)))
else:
mesh = generate_gmsh(FileSource("circle.step"),
2,
order=mesh_order,
force_ambient_dim=2,
other_options=[
"-string",
"Mesh.CharacteristicLengthMax = %g;" % h
])
logger.info("%d elements" % mesh.nelements)
# {{{ discretizations and connections
vol_discr = Discretization(
ctx, mesh, InterpolatoryQuadratureSimplexGroupFactory(vol_quad_order))
ovsmp_vol_discr = Discretization(
ctx, mesh,
InterpolatoryQuadratureSimplexGroupFactory(vol_ovsmp_quad_order))
from meshmode.mesh import BTAG_ALL
from meshmode.discretization.connection import (make_face_restriction,
make_same_mesh_connection)
bdry_connection = make_face_restriction(
vol_discr, InterpolatoryQuadratureSimplexGroupFactory(bdry_quad_order),
BTAG_ALL)
bdry_discr = bdry_connection.to_discr
vol_to_ovsmp_vol = make_same_mesh_connection(ovsmp_vol_discr, vol_discr)
# }}}
# {{{ visualizers
vol_vis = make_visualizer(queue, vol_discr, 20)
bdry_vis = make_visualizer(queue, bdry_discr, 20)
# }}}
vol_x = vol_discr.nodes().with_queue(queue)
ovsmp_vol_x = ovsmp_vol_discr.nodes().with_queue(queue)
rhs = rhs_func(vol_x[0], vol_x[1])
poisson_true_sol = sol_func(vol_x[0], vol_x[1])
vol_vis.write_vtk_file("volume.vtu", [("f", rhs)])
bdry_normals = bind(bdry_discr, p.normal(
mesh.ambient_dim))(queue).as_vector(dtype=object)
bdry_vis.write_vtk_file("boundary.vtu", [("normals", bdry_normals)])
bdry_nodes = bdry_discr.nodes().with_queue(queue)
bdry_f = rhs_func(bdry_nodes[0], bdry_nodes[1])
bdry_f_2 = bdry_connection(queue, rhs)
bdry_vis.write_vtk_file("y.vtu", [("f", bdry_f_2)])
if 0:
vol_vis.show_scalar_in_mayavi(rhs, do_show=False)
bdry_vis.show_scalar_in_mayavi(bdry_f - bdry_f_2,
line_width=10,
do_show=False)
import mayavi.mlab as mlab
mlab.colorbar()
mlab.show()
# {{{ compute volume potential
from sumpy.qbx import LayerPotential
from sumpy.expansion.local import LineTaylorLocalExpansion
def get_kernel():
from sumpy.symbolic import pymbolic_real_norm_2
from pymbolic.primitives import make_sym_vector
from pymbolic import var
d = make_sym_vector("d", 3)
r = pymbolic_real_norm_2(d[:-1])
# r3d = pymbolic_real_norm_2(d)
#expr = var("log")(r3d)
log = var("log")
sqrt = var("sqrt")
a = d[-1]
expr = log(r)
expr = log(sqrt(r**2 + a**2))
expr = log(sqrt(r + a**2))
#expr = log(sqrt(r**2 + a**2))-a**2/2/(r**2+a**2)
#expr = 2*log(sqrt(r**2 + a**2))
scaling = 1 / (2 * var("pi"))
from sumpy.kernel import ExpressionKernel
return ExpressionKernel(dim=3,
expression=expr,
global_scaling_const=scaling,
is_complex_valued=False)
laplace_2d_in_3d_kernel = get_kernel()
layer_pot = LayerPotential(
ctx, [LineTaylorLocalExpansion(laplace_2d_in_3d_kernel, order=0)])
targets = cl.array.zeros(queue, (3, ) + vol_x.shape[1:], vol_x.dtype)
targets[:2] = vol_x
center_dist = 0.125 * np.min(
cl.clmath.sqrt(
bind(vol_discr, p.area_element(mesh.ambient_dim,
mesh.dim))(queue)).get())
centers = make_obj_array(
[ci.copy().reshape(vol_discr.nnodes) for ci in targets])
centers[2][:] = center_dist
print(center_dist)
sources = cl.array.zeros(queue, (3, ) + ovsmp_vol_x.shape[1:],
ovsmp_vol_x.dtype)
sources[:2] = ovsmp_vol_x
ovsmp_rhs = vol_to_ovsmp_vol(queue, rhs)
ovsmp_vol_weights = bind(
ovsmp_vol_discr,
p.area_element(mesh.ambient_dim, mesh.dim) * p.QWeight())(queue)
print("volume: %d source nodes, %d target nodes" %
(ovsmp_vol_discr.nnodes, vol_discr.nnodes))
evt, (vol_pot, ) = layer_pot(
queue,
targets=targets.reshape(3, vol_discr.nnodes),
centers=centers,
sources=sources.reshape(3, ovsmp_vol_discr.nnodes),
strengths=((ovsmp_vol_weights * ovsmp_rhs).reshape(
ovsmp_vol_discr.nnodes), ),
expansion_radii=np.zeros(vol_discr.nnodes),
)
vol_pot_bdry = bdry_connection(queue, vol_pot)
# }}}
# {{{ solve bvp
from sumpy.kernel import LaplaceKernel
from pytential.symbolic.pde.scalar import DirichletOperator
op = DirichletOperator(LaplaceKernel(2), -1, use_l2_weighting=True)
sym_sigma = sym.var("sigma")
op_sigma = op.operator(sym_sigma)
from pytential.qbx import QBXLayerPotentialSource
qbx = QBXLayerPotentialSource(
bdry_discr,
fine_order=bdry_ovsmp_quad_order,
qbx_order=qbx_order,
fmm_order=fmm_order,
)
bound_op = bind(qbx, op_sigma)
poisson_bc = poisson_bc_func(bdry_nodes[0], bdry_nodes[1])
bvp_bc = poisson_bc - vol_pot_bdry
bdry_f = rhs_func(bdry_nodes[0], bdry_nodes[1])
bvp_rhs = bind(bdry_discr, op.prepare_rhs(sym.var("bc")))(queue, bc=bvp_bc)
from pytential.solve import gmres
gmres_result = gmres(bound_op.scipy_op(queue, "sigma", dtype=np.float64),
bvp_rhs,
tol=1e-14,
progress=True,
hard_failure=False)
sigma = gmres_result.solution
print("gmres state:", gmres_result.state)
# }}}
bvp_sol = bind((qbx, vol_discr), op.representation(sym_sigma))(queue,
sigma=sigma)
poisson_sol = bvp_sol + vol_pot
poisson_err = poisson_sol - poisson_true_sol
rel_err = (norm(vol_discr, queue, poisson_err) /
norm(vol_discr, queue, poisson_true_sol))
bdry_vis.write_vtk_file("poisson-boundary.vtu", [
("vol_pot_bdry", vol_pot_bdry),
("sigma", sigma),
])
vol_vis.write_vtk_file("poisson-volume.vtu", [
("bvp_sol", bvp_sol),
("poisson_sol", poisson_sol),
("poisson_true_sol", poisson_true_sol),
("poisson_err", poisson_err),
("vol_pot", vol_pot),
("rhs", rhs),
])
print("h = %s" % h)
print("mesh_order = %s" % mesh_order)
print("vol_quad_order = %s" % vol_quad_order)
print("vol_ovsmp_quad_order = %s" % vol_ovsmp_quad_order)
print("bdry_quad_order = %s" % bdry_quad_order)
print("bdry_ovsmp_quad_order = %s" % bdry_ovsmp_quad_order)
print("qbx_order = %s" % qbx_order)
#print("vol_qbx_order = %s" % vol_qbx_order)
print("fmm_order = %s" % fmm_order)
print()
print("rel err: %g" % rel_err)
def refine_for_global_qbx(lpot_source, wrangler,
group_factory, kernel_length_scale=None,
force_stage2_uniform_refinement_rounds=None,
scaled_max_curvature_threshold=None,
debug=None, maxiter=None,
visualize=None, expansion_disturbance_tolerance=None,
refiner=None):
"""
Entry point for calling the refiner.
:arg lpot_source: An instance of :class:`QBXLayerPotentialSource`.
:arg wrangler: An instance of :class:`RefinerWrangler`.
:arg group_factory: An instance of
:class:`meshmode.mesh.discretization.ElementGroupFactory`. Used for
discretizing the coarse refined mesh.
:arg kernel_length_scale: The kernel length scale, or *None* if not
applicable. All panels are refined to below this size.
:arg maxiter: The maximum number of refiner iterations.
:returns: A tuple ``(lpot_source, *conn*)`` where ``lpot_source`` is the
refined layer potential source, and ``conn`` is a
:class:`meshmode.discretization.connection.DiscretizationConnection`
going from the original mesh to the refined mesh.
"""
if maxiter is None:
maxiter = 10
if debug is None:
# FIXME: Set debug=False by default once everything works.
debug = True
if expansion_disturbance_tolerance is None:
expansion_disturbance_tolerance = 0.025
if force_stage2_uniform_refinement_rounds is None:
force_stage2_uniform_refinement_rounds = 0
# TODO: Stop doing redundant checks by avoiding panels which no longer need
# refinement.
from meshmode.mesh.refinement import RefinerWithoutAdjacency
from meshmode.discretization.connection import (
ChainedDiscretizationConnection, make_same_mesh_connection)
if refiner is not None:
assert refiner.get_current_mesh() == lpot_source.density_discr.mesh
else:
# We may be handed a mesh that's already non-conforming, we don't rely
# on adjacency, and the no-adjacency refiner is faster.
refiner = RefinerWithoutAdjacency(lpot_source.density_discr.mesh)
connections = []
# {{{ first stage refinement
def visualize_refinement(niter, stage_nr, stage_name, flags):
if not visualize:
return
if stage_nr == 1:
discr = lpot_source.density_discr
elif stage_nr == 2:
discr = lpot_source.stage2_density_discr
else:
raise ValueError("unexpected stage number")
flags = flags.get()
logger.info("for stage %s: splitting %d/%d stage-%d elements",
stage_name, np.sum(flags), discr.mesh.nelements, stage_nr)
from meshmode.discretization.visualization import make_visualizer
vis = make_visualizer(wrangler.queue, discr, 3)
assert len(flags) == discr.mesh.nelements
flags = flags.astype(np.bool)
nodes_flags = np.zeros(discr.nnodes)
for grp in discr.groups:
meg = grp.mesh_el_group
grp.view(nodes_flags)[
flags[meg.element_nr_base:meg.nelements+meg.element_nr_base]] = 1
nodes_flags = cl.array.to_device(wrangler.queue, nodes_flags)
vis_data = [
("refine_flags", nodes_flags),
]
if 0:
from pytential import sym, bind
bdry_normals = bind(discr, sym.normal(discr.ambient_dim))(
wrangler.queue).as_vector(dtype=object)
vis_data.append(("bdry_normals", bdry_normals),)
vis.write_vtk_file("refinement-%s-%03d.vtu" % (stage_name, niter), vis_data)
def warn_max_iterations():
from warnings import warn
warn(
"QBX layer potential source refiner did not terminate "
"after %d iterations (the maximum). "
"You may pass 'visualize=True' to with_refinement() "
"to see what area of the geometry is causing trouble. "
"If the issue is disturbance of expansion disks, you may "
"pass a slightly increased value (currently: %g) for "
"_expansion_disturbance_tolerance in with_refinement(). "
"As a last resort, "
"you may use Python's warning filtering mechanism to "
"not treat this warning as an error. "
"The criteria triggering refinement in each iteration "
"were: %s. " % (
len(violated_criteria),
expansion_disturbance_tolerance,
", ".join(
"%d: %s" % (i+1, vc_text)
for i, vc_text in enumerate(violated_criteria))),
RefinerNotConvergedWarning)
violated_criteria = []
iter_violated_criteria = ["start"]
niter = 0
while iter_violated_criteria:
iter_violated_criteria = []
niter += 1
if niter > maxiter:
warn_max_iterations()
break
refine_flags = make_empty_refine_flags(wrangler.queue, lpot_source)
if kernel_length_scale is not None:
with ProcessLogger(logger,
"checking kernel length scale to panel size ratio"):
violates_kernel_length_scale = \
wrangler.check_element_prop_threshold(
element_property=(
lpot_source._coarsest_quad_resolution(
"npanels")),
threshold=kernel_length_scale,
refine_flags=refine_flags, debug=debug)
if violates_kernel_length_scale:
iter_violated_criteria.append("kernel length scale")
visualize_refinement(
niter, 1, "kernel-length-scale", refine_flags)
if scaled_max_curvature_threshold is not None:
with ProcessLogger(logger,
"checking scaled max curvature threshold"):
from pytential.qbx.utils import to_last_dim_length
from pytential import sym, bind
scaled_max_curv = to_last_dim_length(
lpot_source.density_discr,
bind(lpot_source,
sym.ElementwiseMax(
sym._scaled_max_curvature(
lpot_source.density_discr.ambient_dim)))
(wrangler.queue), "npanels")
violates_scaled_max_curv = \
wrangler.check_element_prop_threshold(
element_property=scaled_max_curv,
threshold=scaled_max_curvature_threshold,
refine_flags=refine_flags, debug=debug)
if violates_scaled_max_curv:
iter_violated_criteria.append("curvature")
visualize_refinement(niter, 1, "curvature", refine_flags)
if not iter_violated_criteria:
# Only start building trees once the simple length-based criteria
# are happy.
# Build tree and auxiliary data.
# FIXME: The tree should not have to be rebuilt at each iteration.
tree = wrangler.build_tree(lpot_source)
peer_lists = wrangler.find_peer_lists(tree)
has_disturbed_expansions = \
wrangler.check_expansion_disks_undisturbed_by_sources(
lpot_source, tree, peer_lists,
expansion_disturbance_tolerance,
refine_flags, debug)
if has_disturbed_expansions:
iter_violated_criteria.append("disturbed expansions")
visualize_refinement(niter, 1, "disturbed-expansions", refine_flags)
del tree
del peer_lists
if iter_violated_criteria:
violated_criteria.append(" and ".join(iter_violated_criteria))
conn = wrangler.refine(
lpot_source.density_discr, refiner, refine_flags,
group_factory, debug)
connections.append(conn)
lpot_source = lpot_source.copy(density_discr=conn.to_discr)
del refine_flags
# }}}
# {{{ second stage refinement
iter_violated_criteria = ["start"]
niter = 0
fine_connections = []
stage2_density_discr = lpot_source.density_discr
while iter_violated_criteria:
iter_violated_criteria = []
niter += 1
if niter > maxiter:
warn_max_iterations()
break
# Build tree and auxiliary data.
# FIXME: The tree should not have to be rebuilt at each iteration.
tree = wrangler.build_tree(lpot_source, use_stage2_discr=True)
peer_lists = wrangler.find_peer_lists(tree)
refine_flags = make_empty_refine_flags(
wrangler.queue, lpot_source, use_stage2_discr=True)
has_insufficient_quad_res = \
wrangler.check_sufficient_source_quadrature_resolution(
lpot_source, tree, peer_lists, refine_flags, debug)
if has_insufficient_quad_res:
iter_violated_criteria.append("insufficient quadrature resolution")
visualize_refinement(niter, 2, "quad-resolution", refine_flags)
if iter_violated_criteria:
violated_criteria.append(" and ".join(iter_violated_criteria))
conn = wrangler.refine(
stage2_density_discr,
refiner, refine_flags, group_factory, debug)
stage2_density_discr = conn.to_discr
fine_connections.append(conn)
lpot_source = lpot_source.copy(
to_refined_connection=ChainedDiscretizationConnection(
fine_connections))
del tree
del refine_flags
del peer_lists
for round in range(force_stage2_uniform_refinement_rounds):
conn = wrangler.refine(
stage2_density_discr,
refiner,
np.ones(stage2_density_discr.mesh.nelements, dtype=np.bool),
group_factory, debug)
stage2_density_discr = conn.to_discr
fine_connections.append(conn)
lpot_source = lpot_source.copy(
to_refined_connection=ChainedDiscretizationConnection(
fine_connections))
# }}}
lpot_source = lpot_source.copy(debug=debug, _refined_for_global_qbx=True)
if len(connections) == 0:
# FIXME: This is inefficient
connection = make_same_mesh_connection(
lpot_source.density_discr,
lpot_source.density_discr)
else:
connection = ChainedDiscretizationConnection(connections)
return lpot_source, connection
def flatten_chained_connection(queue, connection):
"""Collapse a connection into a direct connection.
If the given connection is already a
:class:`~meshmode.discretization.connection.DirectDiscretizationConnection`
nothing is done. However, if the connection is a
:class:`~meshmode.discretization.connection.ChainedDiscretizationConnection`,
a new direct connection is constructed that transports from
:attr:`connection.from_discr` to :attr:`connection.to_discr`.
The new direct connection will have a number of groups and batches that
is, at worse, the product of all the connections in the chain. For
example, if we consider a connection between a discretization and a
two-level refinement, both levels will have :math:`n` groups and
:math:`m + 1` batches per group, where :math:`m` is the number of
subdivisions of an element (exact number depends on implementation
details in
:func:`~meshmode.discretization.connection.make_refinement_connection`).
However, a direct connection from level :math:`0` to level :math:`2`
will have at worst :math:`n^2` groups and each group will have
:math:`(m + 1)^2` batches.
.. warning::
If a large number of connections is chained, the number of groups and
batches can become very large.
:arg queue: An instance of :class:`pyopencl.CommandQueue`.
:arg connection: An instance of
:class:`~meshmode.discretization.connection.DiscretizationConnection`.
:return: An instance of
:class:`~meshmode.discretization.connection.DirectDiscretizationConnection`.
"""
from meshmode.discretization.connection import (
DirectDiscretizationConnection,
DiscretizationConnectionElementGroup,
make_same_mesh_connection)
if not hasattr(connection, 'connections'):
return connection
if not connection.connections:
return make_same_mesh_connection(connection.to_discr,
connection.from_discr)
# recursively build direct connections
connections = connection.connections
direct_connections = []
for conn in connections:
direct_connections.append(flatten_chained_connection(queue, conn))
# merge all the direct connections
from_conn = direct_connections[0]
for to_conn in direct_connections[1:]:
el_table = _build_element_lookup_table(queue, from_conn)
grp_to_grp, batch_info = _build_new_group_table(from_conn, to_conn)
# distribute the indices to new groups and batches
from_bins = [[np.empty(0, dtype=np.int) for _ in g] for g in batch_info]
to_bins = [[np.empty(0, dtype=np.int) for _ in g] for g in batch_info]
for (igrp, ibatch), (_, from_batch) in _iterbatches(from_conn.groups):
from_to_element_indices = from_batch.to_element_indices.get(queue)
for (jgrp, jbatch), (_, to_batch) in _iterbatches(to_conn.groups):
igrp_new, ibatch_new = grp_to_grp[igrp, ibatch, jgrp, jbatch]
jfrom = to_batch.from_element_indices.get(queue)
jto = to_batch.to_element_indices.get(queue)
mask = np.isin(jfrom, from_to_element_indices)
from_bins[igrp_new][ibatch_new] = \
np.hstack([from_bins[igrp_new][ibatch_new],
el_table[igrp][jfrom[mask]]])
to_bins[igrp_new][ibatch_new] = \
np.hstack([to_bins[igrp_new][ibatch_new],
jto[mask]])
# build new groups
groups = []
for igrp, (from_bin, to_bin) in enumerate(zip(from_bins, to_bins)):
groups.append(DiscretizationConnectionElementGroup(
list(_build_batches(queue, from_bin, to_bin,
batch_info[igrp]))))
from_conn = DirectDiscretizationConnection(
from_discr=from_conn.from_discr,
to_discr=to_conn.to_discr,
groups=groups,
is_surjective=connection.is_surjective)
return from_conn
