def _refine_qbx_quad_stage2(lpot_source, stage2_density_discr): from meshmode.discretization.connection import make_same_mesh_connection discr = _make_quad_stage2_discr(lpot_source, stage2_density_discr) conn = make_same_mesh_connection(lpot_source._setup_actx, discr, stage2_density_discr) return discr, conn
def __init__(self, density_discr, fine_order, qbx_order, fmm_order, # FIXME set debug=False once everything works expansion_getter=None, real_dtype=np.float64, debug=True): """ :arg fine_order: The total degree to which the (upsampled) underlying quadrature is exact. :arg fmm_order: `False` for direct calculation. ``None`` will set a reasonable(-ish?) default. """ self.fine_density_discr = Discretization( density_discr.cl_context, density_discr.mesh, QuadratureSimplexGroupFactory(fine_order), real_dtype) from meshmode.discretization.connection import make_same_mesh_connection with cl.CommandQueue(density_discr.cl_context) as queue: self.resampler = make_same_mesh_connection( queue, self.fine_density_discr, density_discr) if fmm_order is None: fmm_order = qbx_order + 1 self.qbx_order = qbx_order self.density_discr = density_discr self.fmm_order = fmm_order self.debug = debug # only used in non-FMM case if expansion_getter is None: from sumpy.expansion.local import LineTaylorLocalExpansion expansion_getter = LineTaylorLocalExpansion self.expansion_getter = expansion_getter
def make_visualizer(queue, discr, vis_order): from meshmode.discretization import Discretization from meshmode.discretization.poly_element import \ PolynomialWarpAndBlendGroupFactory vis_discr = Discretization( discr.cl_context, discr.mesh, PolynomialWarpAndBlendGroupFactory(vis_order), real_dtype=discr.real_dtype) from meshmode.discretization.connection import \ make_same_mesh_connection return Visualizer(make_same_mesh_connection(vis_discr, discr))
def make_visualizer(queue, discr, vis_order): from meshmode.discretization import Discretization from meshmode.discretization.poly_element import \ PolynomialWarpAndBlendGroupFactory vis_discr = Discretization(discr.cl_context, discr.mesh, PolynomialWarpAndBlendGroupFactory(vis_order), real_dtype=discr.real_dtype) from meshmode.discretization.connection import \ make_same_mesh_connection return Visualizer(make_same_mesh_connection(vis_discr, discr))
def make_visualizer(actx, discr, vis_order=None, element_shrink_factor=None, force_equidistant=False): """ :arg vis_order: order of the visualization DOFs. :arg element_shrink_factor: number in :math:`(0, 1]`. :arg force_equidistant: if *True*, the visualization is done on equidistant nodes. If plotting high-order Lagrange VTK elements, this needs to be set to *True*. """ from meshmode.discretization.poly_element import OrderAndTypeBasedGroupFactory vis_discr = None if (element_shrink_factor is None and not force_equidistant and vis_order is None): vis_discr = discr else: if force_equidistant: from meshmode.discretization.poly_element import ( PolynomialEquidistantSimplexElementGroup as SimplexElementGroup, EquidistantTensorProductElementGroup as TensorElementGroup) else: from meshmode.discretization.poly_element import ( PolynomialWarpAndBlendElementGroup as SimplexElementGroup, LegendreGaussLobattoTensorProductElementGroup as TensorElementGroup) vis_discr = discr.copy( actx=actx, group_factory=OrderAndTypeBasedGroupFactory( vis_order, simplex_group_class=SimplexElementGroup, tensor_product_group_class=TensorElementGroup), ) if all(grp.discretization_key() == vgrp.discretization_key() for grp, vgrp in zip(discr.groups, vis_discr.groups)): from warnings import warn warn("Visualization discretization is identical to base discretization. " "To avoid the creation of a separate discretization for " "visualization, avoid passing vis_order unless needed.", stacklevel=2) vis_discr = discr from meshmode.discretization.connection import make_same_mesh_connection return Visualizer( make_same_mesh_connection(actx, vis_discr, discr), element_shrink_factor=element_shrink_factor, is_equidistant=force_equidistant)
def test_sanity_qhull_nd(ctx_getter, dim, order): pytest.importorskip("scipy") logging.basicConfig(level=logging.INFO) ctx = ctx_getter() queue = cl.CommandQueue(ctx) from scipy.spatial import Delaunay verts = np.random.rand(1000, dim) dtri = Delaunay(verts) from meshmode.mesh.io import from_vertices_and_simplices mesh = from_vertices_and_simplices(dtri.points.T, dtri.simplices, fix_orientation=True) from meshmode.discretization import Discretization low_discr = Discretization(ctx, mesh, PolynomialEquidistantGroupFactory(order)) high_discr = Discretization(ctx, mesh, PolynomialEquidistantGroupFactory(order + 1)) from meshmode.discretization.connection import make_same_mesh_connection cnx = make_same_mesh_connection(high_discr, low_discr) def f(x): return 0.1 * cl.clmath.sin(x) x_low = low_discr.nodes()[0].with_queue(queue) f_low = f(x_low) x_high = high_discr.nodes()[0].with_queue(queue) f_high_ref = f(x_high) f_high_num = cnx(queue, f_low) err = (f_high_ref - f_high_num).get() err = la.norm(err, np.inf) / la.norm(f_high_ref.get(), np.inf) print(err) assert err < 1e-2
def test_sanity_qhull_nd(ctx_getter, dim, order): pytest.importorskip("scipy") logging.basicConfig(level=logging.INFO) ctx = ctx_getter() queue = cl.CommandQueue(ctx) from scipy.spatial import Delaunay verts = np.random.rand(1000, dim) dtri = Delaunay(verts) from meshmode.mesh.io import from_vertices_and_simplices mesh = from_vertices_and_simplices(dtri.points.T, dtri.simplices, fix_orientation=True) from meshmode.discretization import Discretization low_discr = Discretization(ctx, mesh, PolynomialEquidistantGroupFactory(order)) high_discr = Discretization(ctx, mesh, PolynomialEquidistantGroupFactory(order+1)) from meshmode.discretization.connection import make_same_mesh_connection cnx = make_same_mesh_connection(high_discr, low_discr) def f(x): return 0.1*cl.clmath.sin(x) x_low = low_discr.nodes()[0].with_queue(queue) f_low = f(x_low) x_high = high_discr.nodes()[0].with_queue(queue) f_high_ref = f(x_high) f_high_num = cnx(queue, f_low) err = (f_high_ref-f_high_num).get() err = la.norm(err, np.inf)/la.norm(f_high_ref.get(), np.inf) print(err) assert err < 1e-2
def make_visualizer(queue, discr, vis_order, element_shrink_factor=None): from meshmode.discretization import Discretization from meshmode.discretization.poly_element import ( PolynomialWarpAndBlendElementGroup, LegendreGaussLobattoTensorProductElementGroup, OrderAndTypeBasedGroupFactory) vis_discr = Discretization( discr.cl_context, discr.mesh, OrderAndTypeBasedGroupFactory( vis_order, simplex_group_class=PolynomialWarpAndBlendElementGroup, tensor_product_group_class=( LegendreGaussLobattoTensorProductElementGroup)), real_dtype=discr.real_dtype) from meshmode.discretization.connection import \ make_same_mesh_connection return Visualizer(make_same_mesh_connection(vis_discr, discr), element_shrink_factor=element_shrink_factor)
def make_visualizer(queue, discr, vis_order, element_shrink_factor=None): from meshmode.discretization import Discretization from meshmode.discretization.poly_element import ( PolynomialWarpAndBlendElementGroup, LegendreGaussLobattoTensorProductElementGroup, OrderAndTypeBasedGroupFactory) vis_discr = Discretization( discr.cl_context, discr.mesh, OrderAndTypeBasedGroupFactory( vis_order, simplex_group_class=PolynomialWarpAndBlendElementGroup, tensor_product_group_class=( LegendreGaussLobattoTensorProductElementGroup)), real_dtype=discr.real_dtype) from meshmode.discretization.connection import \ make_same_mesh_connection return Visualizer( make_same_mesh_connection(vis_discr, discr), element_shrink_factor=element_shrink_factor)
def test_sanity_qhull_nd(actx_factory, dim, order): pytest.importorskip("scipy") logging.basicConfig(level=logging.INFO) actx = actx_factory() from scipy.spatial import Delaunay # pylint: disable=no-name-in-module verts = np.random.rand(1000, dim) dtri = Delaunay(verts) # pylint: disable=no-member from meshmode.mesh.io import from_vertices_and_simplices mesh = from_vertices_and_simplices(dtri.points.T, dtri.simplices, fix_orientation=True) from meshmode.discretization import Discretization low_discr = Discretization(actx, mesh, PolynomialEquidistantSimplexGroupFactory(order)) high_discr = Discretization( actx, mesh, PolynomialEquidistantSimplexGroupFactory(order + 1)) from meshmode.discretization.connection import make_same_mesh_connection cnx = make_same_mesh_connection(actx, high_discr, low_discr) def f(x): return 0.1 * actx.np.sin(x) x_low = thaw(low_discr.nodes()[0], actx) f_low = f(x_low) x_high = thaw(high_discr.nodes()[0], actx) f_high_ref = f(x_high) f_high_num = cnx(f_low) err = (flat_norm(f_high_ref - f_high_num, np.inf) / flat_norm(f_high_ref, np.inf)) print(err) assert err < 1e-2
def make_visualizer(actx, discr, vis_order, element_shrink_factor=None, force_equidistant=False): """ :arg vis_order: order of the visualization DOFs. :arg element_shrink_factor: number in :math:`(0, 1]`. :arg force_equidistant: if *True*, the visualization is done on equidistant nodes. If plotting high-order Lagrange VTK elements, this needs to be set to *True*. """ from meshmode.discretization import Discretization if force_equidistant: from meshmode.discretization.poly_element import ( PolynomialEquidistantSimplexElementGroup as SimplexElementGroup, EquidistantTensorProductElementGroup as TensorElementGroup) else: from meshmode.discretization.poly_element import ( PolynomialWarpAndBlendElementGroup as SimplexElementGroup, LegendreGaussLobattoTensorProductElementGroup as TensorElementGroup) from meshmode.discretization.poly_element import OrderAndTypeBasedGroupFactory vis_discr = Discretization( actx, discr.mesh, OrderAndTypeBasedGroupFactory( vis_order, simplex_group_class=SimplexElementGroup, tensor_product_group_class=TensorElementGroup), real_dtype=discr.real_dtype) from meshmode.discretization.connection import \ make_same_mesh_connection return Visualizer(make_same_mesh_connection(actx, vis_discr, discr), element_shrink_factor=element_shrink_factor, is_equidistant=force_equidistant)
def refined_interp_to_ovsmp_quad_connection(self): from meshmode.discretization.connection import make_same_mesh_connection return make_same_mesh_connection(self.quad_stage2_density_discr, self.stage2_density_discr)
def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename=None, rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive example.""" cl_ctx = ctx_factory() if casename is None: casename = "mirgecom" from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() nproc = comm.Get_size() logmgr = initialize_logmgr(use_logmgr, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: queue = cl.CommandQueue( cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) actx = actx_class(queue, allocator=cl_tools.MemoryPool( cl_tools.ImmediateAllocator(queue))) # Some discretization parameters dim = 2 nel_1d = 8 order = 1 # {{{ Time stepping control # This example runs only 3 steps by default (to keep CI ~short) # With the mixture defined below, equilibrium is achieved at ~40ms # To run to equlibrium, set t_final >= 40ms. # Time stepper selection if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step # Time loop control parameters current_step = 0 t_final = 1e-8 current_cfl = 1.0 current_dt = 1e-9 current_t = 0 constant_cfl = False # i.o frequencies nstatus = 1 nviz = 5 nhealth = 1 nrestart = 5 # }}} Time stepping control debug = False rst_path = "restart_data/" rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl") if rst_filename: # read the grid from restart data rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["num_parts"] == nproc rst_time = restart_data["t"] rst_step = restart_data["step"] rst_order = restart_data["order"] else: # generate the grid from scratch from meshmode.mesh.generation import generate_regular_rect_mesh box_ll = -0.005 box_ur = 0.005 generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll, ) * dim, b=(box_ur, ) * dim, nelements_per_axis=(nel_1d, ) * dim) local_mesh, global_nelements = generate_and_distribute_mesh( comm, generate_mesh) local_nelements = local_mesh.nelements discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) nodes = thaw(actx, discr.nodes()) vis_timer = None if logmgr: logmgr_add_device_name(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) logmgr.add_watches([ ("step.max", "step = {value}, "), ("t_sim.max", "sim time: {value:1.6e} s\n"), ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), ("max_pressure", "{value:1.9e})\n"), ("min_temperature", "------- T (min, max) (K) = ({value:7g}, "), ("max_temperature", "{value:7g})\n"), ("t_step.max", "------- step walltime: {value:6g} s, "), ("t_log.max", "log walltime: {value:6g} s") ]) # {{{ Set up initial state using Cantera # Use Cantera for initialization # -- Pick up a CTI for the thermochemistry config # --- Note: Users may add their own CTI file by dropping it into # --- mirgecom/mechanisms alongside the other CTI files. from mirgecom.mechanisms import get_mechanism_cti mech_cti = get_mechanism_cti("uiuc") cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti) nspecies = cantera_soln.n_species # Initial temperature, pressure, and mixutre mole fractions are needed to # set up the initial state in Cantera. init_temperature = 1500.0 # Initial temperature hot enough to burn # Parameters for calculating the amounts of fuel, oxidizer, and inert species equiv_ratio = 1.0 ox_di_ratio = 0.21 stoich_ratio = 3.0 # Grab the array indices for the specific species, ethylene, oxygen, and nitrogen i_fu = cantera_soln.species_index("C2H4") i_ox = cantera_soln.species_index("O2") i_di = cantera_soln.species_index("N2") x = np.zeros(nspecies) # Set the species mole fractions according to our desired fuel/air mixture x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio + ox_di_ratio * equiv_ratio) x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio # Uncomment next line to make pylint fail when it can't find cantera.one_atm one_atm = cantera.one_atm # pylint: disable=no-member # one_atm = 101325.0 # Let the user know about how Cantera is being initilized print(f"Input state (T,P,X) = ({init_temperature}, {one_atm}, {x}") # Set Cantera internal gas temperature, pressure, and mole fractios cantera_soln.TPX = init_temperature, one_atm, x # Pull temperature, total density, mass fractions, and pressure from Cantera # We need total density, and mass fractions to initialize the fluid/gas state. can_t, can_rho, can_y = cantera_soln.TDY can_p = cantera_soln.P # *can_t*, *can_p* should not differ (significantly) from user's initial data, # but we want to ensure that we use exactly the same starting point as Cantera, # so we use Cantera's version of these data. # }}} # {{{ Create Pyrometheus thermochemistry object & EOS # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and # generates a set of methods to calculate chemothermomechanical properties and # states for this particular mechanism. pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np) eos = PyrometheusMixture(pyrometheus_mechanism, temperature_guess=init_temperature) # }}} # {{{ MIRGE-Com state initialization # Initialize the fluid/gas state with Cantera-consistent data: # (density, pressure, temperature, mass_fractions) print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}") velocity = np.zeros(shape=(dim, )) initializer = MixtureInitializer(dim=dim, nspecies=nspecies, pressure=can_p, temperature=can_t, massfractions=can_y, velocity=velocity) my_boundary = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: my_boundary} if rst_filename: current_step = rst_step current_t = rst_time if logmgr: from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) if order == rst_order: current_state = restart_data["state"] else: rst_state = restart_data["state"] old_discr = EagerDGDiscretization(actx, local_mesh, order=rst_order, mpi_communicator=comm) from meshmode.discretization.connection import make_same_mesh_connection connection = make_same_mesh_connection( actx, discr.discr_from_dd("vol"), old_discr.discr_from_dd("vol")) current_state = connection(rst_state) else: # Set the current state from time 0 current_state = initializer(eos=eos, x_vec=nodes) # Inspection at physics debugging time if debug: print("Initial MIRGE-Com state:") print(f"{current_state=}") print(f"Initial DV pressure: {eos.pressure(current_state)}") print(f"Initial DV temperature: {eos.temperature(current_state)}") # }}} visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ eosname = eos.__class__.__name__ init_message = make_init_message(dim=dim, order=order, nelements=local_nelements, global_nelements=global_nelements, dt=current_dt, t_final=t_final, nstatus=nstatus, nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, initname=initname, eosname=eosname, casename=casename) # Cantera equilibrate calculates the expected end state @ chemical equilibrium # i.e. the expected state after all reactions cantera_soln.equilibrate("UV") eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY eq_pressure = cantera_soln.P # Report the expected final state to the user if rank == 0: logger.info(init_message) logger.info(f"Expected equilibrium state:" f" {eq_pressure=}, {eq_temperature=}," f" {eq_density=}, {eq_mass_fractions=}") def my_write_status(dt, cfl): status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}" if rank == 0: logger.info(status_msg) def my_write_viz(step, t, dt, state, ts_field=None, dv=None, production_rates=None, cfl=None): if dv is None: dv = eos.dependent_vars(state) if production_rates is None: production_rates = eos.get_production_rates(state) if ts_field is None: ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state) viz_fields = [("cv", state), ("dv", dv), ("production_rates", production_rates), ("dt" if constant_cfl else "cfl", ts_field)] write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True, vis_timer=vis_timer) def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) if rst_fname == rst_filename: if rank == 0: logger.info("Skipping overwrite of restart file.") else: rst_data = { "local_mesh": local_mesh, "state": state, "t": t, "step": step, "order": order, "global_nelements": global_nelements, "num_parts": nproc } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(dv): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ or check_range_local(discr, "vol", dv.pressure, 1e5, 2.4e5): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") if check_range_local(discr, "vol", dv.temperature, 1.498e3, 1.52e3): health_error = True logger.info(f"{rank=}: Invalid temperature data found.") return health_error def my_get_timestep(t, dt, state): # richer interface to calculate {dt,cfl} returns node-local estimates t_remaining = max(0, t_final - t) if constant_cfl: from mirgecom.inviscid import get_inviscid_timestep ts_field = current_cfl * get_inviscid_timestep( discr, eos=eos, cv=state) from grudge.op import nodal_min dt = nodal_min(discr, "vol", ts_field) cfl = current_cfl else: from mirgecom.inviscid import get_inviscid_cfl ts_field = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) from grudge.op import nodal_max cfl = nodal_max(discr, "vol", ts_field) return ts_field, cfl, min(t_remaining, dt) def my_pre_step(step, t, dt, state): try: dv = None if logmgr: logmgr.tick_before() from mirgecom.simutil import check_step do_viz = check_step(step=step, interval=nviz) do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) do_status = check_step(step=step, interval=nstatus) if do_health: dv = eos.dependent_vars(state) from mirgecom.simutil import allsync health_errors = allsync(my_health_check(dv), comm, op=MPI.LOR) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state) if do_status: my_write_status(dt, cfl) if do_restart: my_write_restart(step=step, t=t, state=state) if do_viz: production_rates = eos.get_production_rates(state) if dv is None: dv = eos.dependent_vars(state) my_write_viz(step=step, t=t, dt=dt, state=state, dv=dv, production_rates=production_rates, ts_field=ts_field, cfl=cfl) except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, dt=dt, state=state) my_write_restart(step=step, t=t, state=state) raise return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() return state, dt def my_rhs(t, state): return (euler_operator( discr, cv=state, time=t, boundaries=boundaries, eos=eos) + eos.get_species_source_terms(state)) current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, current_cfl, eos, t_final, constant_cfl) current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, state=current_state, t=current_t, t_final=t_final) # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") final_dv = eos.dependent_vars(current_state) final_dm = eos.get_production_rates(current_state) ts_field, cfl, dt = my_get_timestep(t=current_t, dt=current_dt, state=current_state) my_write_viz(step=current_step, t=current_t, dt=dt, state=current_state, dv=final_dv, production_rates=final_dm, ts_field=ts_field, cfl=cfl) my_write_status(dt=dt, cfl=cfl) my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: logmgr.close() elif use_profiling: print(actx.tabulate_profiling_data()) finish_tol = 1e-16 assert np.abs(current_t - t_final) < finish_tol
def main(): import logging logging.basicConfig(level=logging.INFO) ctx = cl.create_some_context() queue = cl.CommandQueue(ctx) mesh = generate_gmsh( FileSource("circle.step"), 2, order=mesh_order, force_ambient_dim=2, other_options=["-string", "Mesh.CharacteristicLengthMax = %g;" % h] ) logger.info("%d elements" % mesh.nelements) # {{{ discretizations and connections vol_discr = Discretization(ctx, mesh, InterpolatoryQuadratureSimplexGroupFactory(vol_quad_order)) ovsmp_vol_discr = Discretization(ctx, mesh, InterpolatoryQuadratureSimplexGroupFactory(vol_ovsmp_quad_order)) from meshmode.discretization.connection import ( make_boundary_restriction, make_same_mesh_connection) bdry_mesh, bdry_discr, bdry_connection = make_boundary_restriction( queue, vol_discr, InterpolatoryQuadratureSimplexGroupFactory(bdry_quad_order)) vol_to_ovsmp_vol = make_same_mesh_connection( queue, ovsmp_vol_discr, vol_discr) # }}} # {{{ visualizers vol_vis = make_visualizer(queue, vol_discr, 20) bdry_vis = make_visualizer(queue, bdry_discr, 20) # }}} vol_x = vol_discr.nodes().with_queue(queue) ovsmp_vol_x = ovsmp_vol_discr.nodes().with_queue(queue) rhs = rhs_func(vol_x[0], vol_x[1]) poisson_true_sol = sol_func(vol_x[0], vol_x[1]) vol_vis.write_vtk_file("volume.vtu", [("f", rhs)]) bdry_normals = bind(bdry_discr, p.normal())(queue).as_vector(dtype=object) bdry_vis.write_vtk_file("boundary.vtu", [ ("normals", bdry_normals) ]) bdry_nodes = bdry_discr.nodes().with_queue(queue) bdry_f = rhs_func(bdry_nodes[0], bdry_nodes[1]) bdry_f_2 = bdry_connection(queue, rhs) bdry_vis.write_vtk_file("y.vtu", [("f", bdry_f_2)]) if 0: vol_vis.show_scalar_in_mayavi(rhs, do_show=False) bdry_vis.show_scalar_in_mayavi(bdry_f - bdry_f_2, line_width=10, do_show=False) import mayavi.mlab as mlab mlab.colorbar() mlab.show() # {{{ compute volume potential from sumpy.qbx import LayerPotential from sumpy.expansion.local import LineTaylorLocalExpansion def get_kernel(): from sumpy.symbolic import pymbolic_real_norm_2 from pymbolic.primitives import (make_sym_vector, Variable as var) r = pymbolic_real_norm_2(make_sym_vector("d", 3)) expr = var("log")(r) scaling = 1/(2*var("pi")) from sumpy.kernel import ExpressionKernel return ExpressionKernel( dim=3, expression=expr, scaling=scaling, is_complex_valued=False) laplace_2d_in_3d_kernel = get_kernel() layer_pot = LayerPotential(ctx, [ LineTaylorLocalExpansion(laplace_2d_in_3d_kernel, order=vol_qbx_order)]) targets = cl.array.zeros(queue, (3,) + vol_x.shape[1:], vol_x.dtype) targets[:2] = vol_x center_dist = np.min( cl.clmath.sqrt( bind(vol_discr, p.area_element())(queue)).get()) centers = make_obj_array([ci.copy().reshape(vol_discr.nnodes) for ci in targets]) centers[2][:] = center_dist sources = cl.array.zeros(queue, (3,) + ovsmp_vol_x.shape[1:], ovsmp_vol_x.dtype) sources[:2] = ovsmp_vol_x ovsmp_rhs = vol_to_ovsmp_vol(queue, rhs) ovsmp_vol_weights = bind(ovsmp_vol_discr, p.area_element() * p.QWeight())(queue) evt, (vol_pot,) = layer_pot( queue, targets=targets.reshape(3, vol_discr.nnodes), centers=centers, sources=sources.reshape(3, ovsmp_vol_discr.nnodes), strengths=( (ovsmp_vol_weights*ovsmp_rhs).reshape(ovsmp_vol_discr.nnodes),) ) vol_pot_bdry = bdry_connection(queue, vol_pot) # }}} # {{{ solve bvp from sumpy.kernel import LaplaceKernel from pytential.symbolic.pde.scalar import DirichletOperator op = DirichletOperator(LaplaceKernel(2), -1, use_l2_weighting=True) sym_sigma = sym.var("sigma") op_sigma = op.operator(sym_sigma) from pytential.qbx import QBXLayerPotentialSource qbx = QBXLayerPotentialSource( bdry_discr, fine_order=bdry_ovsmp_quad_order, qbx_order=qbx_order, fmm_order=fmm_order ) bound_op = bind(qbx, op_sigma) poisson_bc = poisson_bc_func(bdry_nodes[0], bdry_nodes[1]) bvp_bc = poisson_bc - vol_pot_bdry bdry_f = rhs_func(bdry_nodes[0], bdry_nodes[1]) bvp_rhs = bind(bdry_discr, op.prepare_rhs(sym.var("bc")))(queue, bc=bvp_bc) from pytential.solve import gmres gmres_result = gmres( bound_op.scipy_op(queue, "sigma"), bvp_rhs, tol=1e-14, progress=True, hard_failure=False) sigma = gmres_result.solution print("gmres state:", gmres_result.state) # }}} bvp_sol = bind( (qbx, vol_discr), op.representation(sym_sigma))(queue, sigma=sigma) poisson_sol = bvp_sol + vol_pot poisson_err = poisson_sol-poisson_true_sol rel_err = ( norm(vol_discr, queue, poisson_err) / norm(vol_discr, queue, poisson_true_sol)) bdry_vis.write_vtk_file("poisson-boundary.vtu", [ ("vol_pot_bdry", vol_pot_bdry), ("sigma", sigma), ]) vol_vis.write_vtk_file("poisson-volume.vtu", [ ("bvp_sol", bvp_sol), ("poisson_sol", poisson_sol), ("poisson_true_sol", poisson_true_sol), ("poisson_err", poisson_err), ("vol_pot", vol_pot), ("rhs", rhs), ]) print("h = %s" % h) print("mesh_order = %s" % mesh_order) print("vol_quad_order = %s" % vol_quad_order) print("vol_ovsmp_quad_order = %s" % vol_ovsmp_quad_order) print("bdry_quad_order = %s" % bdry_quad_order) print("bdry_ovsmp_quad_order = %s" % bdry_ovsmp_quad_order) print("qbx_order = %s" % qbx_order) print("vol_qbx_order = %s" % vol_qbx_order) print("fmm_order = %s" % fmm_order) print() print("rel err: %g" % rel_err)
def refined_interp_to_ovsmp_quad_connection(self): from meshmode.discretization.connection import make_same_mesh_connection return make_same_mesh_connection( self.quad_stage2_density_discr, self.stage2_density_discr)
def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_overintegration=False, use_profiling=False, casename=None, rst_filename=None, actx_class=PyOpenCLArrayContext, log_dependent=True): """Drive example.""" cl_ctx = ctx_factory() if casename is None: casename = "mirgecom" from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() nproc = comm.Get_size() from mirgecom.simutil import global_reduce as _global_reduce global_reduce = partial(_global_reduce, comm=comm) logmgr = initialize_logmgr(use_logmgr, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: queue = cl.CommandQueue( cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) actx = actx_class(queue, allocator=cl_tools.MemoryPool( cl_tools.ImmediateAllocator(queue))) # Some discretization parameters dim = 2 nel_1d = 8 order = 1 # {{{ Time stepping control # This example runs only 3 steps by default (to keep CI ~short) # With the mixture defined below, equilibrium is achieved at ~40ms # To run to equilibrium, set t_final >= 40ms. # Time stepper selection if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step # Time loop control parameters current_step = 0 t_final = 1e-8 current_cfl = 1.0 current_dt = 1e-9 current_t = 0 constant_cfl = False # i.o frequencies nstatus = 1 nviz = 5 nhealth = 1 nrestart = 5 # }}} Time stepping control debug = False rst_path = "restart_data/" rst_pattern = (rst_path + "{cname}-{step:04d}-{rank:04d}.pkl") if rst_filename: # read the grid from restart data rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["num_parts"] == nproc rst_time = restart_data["t"] rst_step = restart_data["step"] rst_order = restart_data["order"] else: # generate the grid from scratch from meshmode.mesh.generation import generate_regular_rect_mesh box_ll = -0.005 box_ur = 0.005 generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll, ) * dim, b=(box_ur, ) * dim, nelements_per_axis=(nel_1d, ) * dim) local_mesh, global_nelements = generate_and_distribute_mesh( comm, generate_mesh) local_nelements = local_mesh.nelements from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD from meshmode.discretization.poly_element import \ default_simplex_group_factory, QuadratureSimplexGroupFactory discr = EagerDGDiscretization( actx, local_mesh, discr_tag_to_group_factory={ DISCR_TAG_BASE: default_simplex_group_factory(base_dim=local_mesh.dim, order=order), DISCR_TAG_QUAD: QuadratureSimplexGroupFactory(2 * order + 1) }, mpi_communicator=comm) nodes = thaw(discr.nodes(), actx) ones = discr.zeros(actx) + 1.0 if use_overintegration: quadrature_tag = DISCR_TAG_QUAD else: quadrature_tag = None ones = discr.zeros(actx) + 1.0 vis_timer = None if logmgr: logmgr_add_cl_device_info(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) logmgr.add_watches([("step.max", "step = {value}, "), ("t_sim.max", "sim time: {value:1.6e} s\n"), ("t_step.max", "------- step walltime: {value:6g} s, "), ("t_log.max", "log walltime: {value:6g} s")]) if log_dependent: logmgr_add_many_discretization_quantities( logmgr, discr, dim, extract_vars_for_logging, units_for_logging) logmgr.add_watches([ ("min_pressure", "\n------- P (min, max) (Pa) = ({value:1.9e}, "), ("max_pressure", "{value:1.9e})\n"), ("min_temperature", "------- T (min, max) (K) = ({value:7g}, "), ("max_temperature", "{value:7g})\n") ]) # {{{ Set up initial state using Cantera # Use Cantera for initialization # -- Pick up a CTI for the thermochemistry config # --- Note: Users may add their own CTI file by dropping it into # --- mirgecom/mechanisms alongside the other CTI files. from mirgecom.mechanisms import get_mechanism_cti mech_cti = get_mechanism_cti("uiuc") cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti) nspecies = cantera_soln.n_species # Initial temperature, pressure, and mixutre mole fractions are needed to # set up the initial state in Cantera. temperature_seed = 1500.0 # Initial temperature hot enough to burn # Parameters for calculating the amounts of fuel, oxidizer, and inert species equiv_ratio = 1.0 ox_di_ratio = 0.21 stoich_ratio = 3.0 # Grab the array indices for the specific species, ethylene, oxygen, and nitrogen i_fu = cantera_soln.species_index("C2H4") i_ox = cantera_soln.species_index("O2") i_di = cantera_soln.species_index("N2") x = np.zeros(nspecies) # Set the species mole fractions according to our desired fuel/air mixture x[i_fu] = (ox_di_ratio * equiv_ratio) / (stoich_ratio + ox_di_ratio * equiv_ratio) x[i_ox] = stoich_ratio * x[i_fu] / equiv_ratio x[i_di] = (1.0 - ox_di_ratio) * x[i_ox] / ox_di_ratio # Uncomment next line to make pylint fail when it can't find cantera.one_atm one_atm = cantera.one_atm # pylint: disable=no-member # one_atm = 101325.0 # Let the user know about how Cantera is being initilized print(f"Input state (T,P,X) = ({temperature_seed}, {one_atm}, {x}") # Set Cantera internal gas temperature, pressure, and mole fractios cantera_soln.TPX = temperature_seed, one_atm, x # Pull temperature, total density, mass fractions, and pressure from Cantera # We need total density, and mass fractions to initialize the fluid/gas state. can_t, can_rho, can_y = cantera_soln.TDY can_p = cantera_soln.P # *can_t*, *can_p* should not differ (significantly) from user's initial data, # but we want to ensure that we use exactly the same starting point as Cantera, # so we use Cantera's version of these data. # }}} # {{{ Create Pyrometheus thermochemistry object & EOS # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and # generates a set of methods to calculate chemothermomechanical properties and # states for this particular mechanism. from mirgecom.thermochemistry import make_pyrometheus_mechanism_class pyro_mechanism = make_pyrometheus_mechanism_class(cantera_soln)(actx.np) eos = PyrometheusMixture(pyro_mechanism, temperature_guess=temperature_seed) gas_model = GasModel(eos=eos) from pytools.obj_array import make_obj_array def get_temperature_update(cv, temperature): y = cv.species_mass_fractions e = gas_model.eos.internal_energy(cv) / cv.mass return pyro_mechanism.get_temperature_update_energy(e, temperature, y) from mirgecom.gas_model import make_fluid_state def get_fluid_state(cv, tseed): return make_fluid_state(cv=cv, gas_model=gas_model, temperature_seed=tseed) compute_temperature_update = actx.compile(get_temperature_update) construct_fluid_state = actx.compile(get_fluid_state) # }}} # {{{ MIRGE-Com state initialization # Initialize the fluid/gas state with Cantera-consistent data: # (density, pressure, temperature, mass_fractions) print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}") velocity = np.zeros(shape=(dim, )) initializer = MixtureInitializer(dim=dim, nspecies=nspecies, pressure=can_p, temperature=can_t, massfractions=can_y, velocity=velocity) my_boundary = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: my_boundary} if rst_filename: current_step = rst_step current_t = rst_time if logmgr: from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) if order == rst_order: current_cv = restart_data["cv"] temperature_seed = restart_data["temperature_seed"] else: rst_cv = restart_data["cv"] old_discr = EagerDGDiscretization(actx, local_mesh, order=rst_order, mpi_communicator=comm) from meshmode.discretization.connection import make_same_mesh_connection connection = make_same_mesh_connection( actx, discr.discr_from_dd("vol"), old_discr.discr_from_dd("vol")) current_cv = connection(rst_cv) temperature_seed = connection(restart_data["temperature_seed"]) else: # Set the current state from time 0 current_cv = initializer(eos=gas_model.eos, x_vec=nodes) temperature_seed = temperature_seed * ones # The temperature_seed going into this function is: # - At time 0: the initial temperature input data (maybe from Cantera) # - On restart: the restarted temperature seed from restart file (saving # the *seed* allows restarts to be deterministic current_fluid_state = construct_fluid_state(current_cv, temperature_seed) current_dv = current_fluid_state.dv temperature_seed = current_dv.temperature # Inspection at physics debugging time if debug: print("Initial MIRGE-Com state:") print(f"Initial DV pressure: {current_fluid_state.pressure}") print(f"Initial DV temperature: {current_fluid_state.temperature}") # }}} visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ eosname = gas_model.eos.__class__.__name__ init_message = make_init_message(dim=dim, order=order, nelements=local_nelements, global_nelements=global_nelements, dt=current_dt, t_final=t_final, nstatus=nstatus, nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, initname=initname, eosname=eosname, casename=casename) # Cantera equilibrate calculates the expected end state @ chemical equilibrium # i.e. the expected state after all reactions cantera_soln.equilibrate("UV") eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY eq_pressure = cantera_soln.P # Report the expected final state to the user if rank == 0: logger.info(init_message) logger.info(f"Expected equilibrium state:" f" {eq_pressure=}, {eq_temperature=}," f" {eq_density=}, {eq_mass_fractions=}") def my_write_status(dt, cfl, dv=None): status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}" if ((dv is not None) and (not log_dependent)): temp = dv.temperature press = dv.pressure from grudge.op import nodal_min_loc, nodal_max_loc tmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", temp)), comm=comm, op=MPI.MIN) tmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", temp)), comm=comm, op=MPI.MAX) pmin = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", press)), comm=comm, op=MPI.MIN) pmax = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", press)), comm=comm, op=MPI.MAX) dv_status_msg = f"\nP({pmin}, {pmax}), T({tmin}, {tmax})" status_msg = status_msg + dv_status_msg if rank == 0: logger.info(status_msg) def my_write_viz(step, t, dt, state, ts_field, dv, production_rates, cfl): viz_fields = [("cv", state), ("dv", dv), ("production_rates", production_rates), ("dt" if constant_cfl else "cfl", ts_field)] write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True, vis_timer=vis_timer) def my_write_restart(step, t, state, temperature_seed): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) if rst_fname == rst_filename: if rank == 0: logger.info("Skipping overwrite of restart file.") else: rst_data = { "local_mesh": local_mesh, "cv": state.cv, "temperature_seed": temperature_seed, "t": t, "step": step, "order": order, "global_nelements": global_nelements, "num_parts": nproc } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(cv, dv): import grudge.op as op health_error = False pressure = dv.pressure temperature = dv.temperature from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") if check_range_local(discr, "vol", pressure, 1e5, 2.6e5): health_error = True logger.info(f"{rank=}: Pressure range violation.") if check_naninf_local(discr, "vol", temperature): health_error = True logger.info(f"{rank=}: Invalid temperature data found.") if check_range_local(discr, "vol", temperature, 1.498e3, 1.6e3): health_error = True logger.info(f"{rank=}: Temperature range violation.") # This check is the temperature convergence check # The current *temperature* is what Pyrometheus gets # after a fixed number of Newton iterations, *n_iter*. # Calling `compute_temperature` here with *temperature* # input as the guess returns the calculated gas temperature after # yet another *n_iter*. # The difference between those two temperatures is the # temperature residual, which can be used as an indicator of # convergence in Pyrometheus `get_temperature`. # Note: The local max jig below works around a very long compile # in lazy mode. temp_resid = compute_temperature_update(cv, temperature) / temperature temp_err = (actx.to_numpy(op.nodal_max_loc(discr, "vol", temp_resid))) if temp_err > 1e-8: health_error = True logger.info( f"{rank=}: Temperature is not converged {temp_resid=}.") return health_error from mirgecom.inviscid import get_inviscid_timestep def get_dt(state): return get_inviscid_timestep(discr, state=state) compute_dt = actx.compile(get_dt) from mirgecom.inviscid import get_inviscid_cfl def get_cfl(state, dt): return get_inviscid_cfl(discr, dt=dt, state=state) compute_cfl = actx.compile(get_cfl) def get_production_rates(cv, temperature): return eos.get_production_rates(cv, temperature) compute_production_rates = actx.compile(get_production_rates) def my_get_timestep(t, dt, state): # richer interface to calculate {dt,cfl} returns node-local estimates t_remaining = max(0, t_final - t) if constant_cfl: ts_field = current_cfl * compute_dt(state) from grudge.op import nodal_min_loc dt = allsync(actx.to_numpy(nodal_min_loc(discr, "vol", ts_field)), comm=comm, op=MPI.MIN) cfl = current_cfl else: ts_field = compute_cfl(state, current_dt) from grudge.op import nodal_max_loc cfl = allsync(actx.to_numpy(nodal_max_loc(discr, "vol", ts_field)), comm=comm, op=MPI.MAX) return ts_field, cfl, min(t_remaining, dt) def my_pre_step(step, t, dt, state): cv, tseed = state fluid_state = construct_fluid_state(cv, tseed) dv = fluid_state.dv try: if logmgr: logmgr.tick_before() from mirgecom.simutil import check_step do_viz = check_step(step=step, interval=nviz) do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) do_status = check_step(step=step, interval=nstatus) if do_health: health_errors = global_reduce(my_health_check(cv, dv), op="lor") if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=fluid_state) if do_status: my_write_status(dt=dt, cfl=cfl, dv=dv) if do_restart: my_write_restart(step=step, t=t, state=fluid_state, temperature_seed=tseed) if do_viz: production_rates = compute_production_rates( fluid_state.cv, fluid_state.temperature) my_write_viz(step=step, t=t, dt=dt, state=cv, dv=dv, production_rates=production_rates, ts_field=ts_field, cfl=cfl) except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") # my_write_viz(step=step, t=t, dt=dt, state=cv) # my_write_restart(step=step, t=t, state=fluid_state) raise return state, dt def my_post_step(step, t, dt, state): cv, tseed = state fluid_state = construct_fluid_state(cv, tseed) # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, cv, gas_model.eos) logmgr.tick_after() return make_obj_array([cv, fluid_state.temperature]), dt def my_rhs(t, state): cv, tseed = state from mirgecom.gas_model import make_fluid_state fluid_state = make_fluid_state(cv=cv, gas_model=gas_model, temperature_seed=tseed) return make_obj_array([ euler_operator(discr, state=fluid_state, time=t, boundaries=boundaries, gas_model=gas_model, quadrature_tag=quadrature_tag) + eos.get_species_source_terms(cv, fluid_state.temperature), 0 * tseed ]) current_dt = get_sim_timestep(discr, current_fluid_state, current_t, current_dt, current_cfl, t_final, constant_cfl) current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, state=make_obj_array([current_cv, temperature_seed]), t=current_t, t_final=t_final) # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") final_cv, tseed = current_state final_fluid_state = construct_fluid_state(final_cv, tseed) final_dv = final_fluid_state.dv final_dm = compute_production_rates(final_cv, final_dv.temperature) ts_field, cfl, dt = my_get_timestep(t=current_t, dt=current_dt, state=final_fluid_state) my_write_viz(step=current_step, t=current_t, dt=dt, state=final_cv, dv=final_dv, production_rates=final_dm, ts_field=ts_field, cfl=cfl) my_write_status(dt=dt, cfl=cfl, dv=final_dv) my_write_restart(step=current_step, t=current_t, state=final_fluid_state, temperature_seed=tseed) if logmgr: logmgr.close() elif use_profiling: print(actx.tabulate_profiling_data()) finish_tol = 1e-16 assert np.abs(current_t - t_final) < finish_tol
def main(snapshot_pattern="wave-mpi-{step:04d}-{rank:04d}.pkl", restart_step=None, use_profiling=False, use_logmgr=False, actx_class=PyOpenCLArrayContext): """Drive the example.""" cl_ctx = cl.create_some_context() queue = cl.CommandQueue(cl_ctx) from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() num_parts = comm.Get_size() logmgr = initialize_logmgr(use_logmgr, filename="wave-mpi.sqlite", mode="wu", mpi_comm=comm) if use_profiling: queue = cl.CommandQueue(cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) actx = actx_class(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), logmgr=logmgr) else: queue = cl.CommandQueue(cl_ctx) actx = actx_class(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) if restart_step is None: from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis mesh_dist = MPIMeshDistributor(comm) dim = 2 nel_1d = 16 if mesh_dist.is_mananger_rank(): from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh( a=(-0.5,)*dim, b=(0.5,)*dim, nelements_per_axis=(nel_1d,)*dim) print("%d elements" % mesh.nelements) part_per_element = get_partition_by_pymetis(mesh, num_parts) local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts) del mesh else: local_mesh = mesh_dist.receive_mesh_part() fields = None else: from mirgecom.restart import read_restart_data restart_data = read_restart_data( actx, snapshot_pattern.format(step=restart_step, rank=rank) ) local_mesh = restart_data["local_mesh"] nel_1d = restart_data["nel_1d"] assert comm.Get_size() == restart_data["num_parts"] order = 3 discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) current_cfl = 0.485 wave_speed = 1.0 from grudge.dt_utils import characteristic_lengthscales dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed from grudge.op import nodal_min dt = nodal_min(discr, "vol", dt) t_final = 1 if restart_step is None: t = 0 istep = 0 fields = flat_obj_array( bump(actx, discr), [discr.zeros(actx) for i in range(discr.dim)] ) else: t = restart_data["t"] istep = restart_step assert istep == restart_step restart_fields = restart_data["fields"] old_order = restart_data["order"] if old_order != order: old_discr = EagerDGDiscretization(actx, local_mesh, order=old_order, mpi_communicator=comm) from meshmode.discretization.connection import make_same_mesh_connection connection = make_same_mesh_connection(actx, discr.discr_from_dd("vol"), old_discr.discr_from_dd("vol")) fields = connection(restart_fields) else: fields = restart_fields if logmgr: logmgr_add_cl_device_info(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) logmgr.add_watches(["step.max", "t_step.max", "t_log.max"]) try: logmgr.add_watches(["memory_usage_python.max", "memory_usage_gpu.max"]) except KeyError: pass if use_profiling: logmgr.add_watches(["multiply_time.max"]) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) vis = make_visualizer(discr) def rhs(t, w): return wave_operator(discr, c=wave_speed, w=w) compiled_rhs = actx.compile(rhs) while t < t_final: if logmgr: logmgr.tick_before() # restart must happen at beginning of step if istep % 100 == 0 and ( # Do not overwrite the restart file that we just read. istep != restart_step): from mirgecom.restart import write_restart_file write_restart_file( actx, restart_data={ "local_mesh": local_mesh, "order": order, "fields": fields, "t": t, "step": istep, "nel_1d": nel_1d, "num_parts": num_parts}, filename=snapshot_pattern.format(step=istep, rank=rank), comm=comm ) if istep % 10 == 0: print(istep, t, discr.norm(fields[0])) vis.write_parallel_vtk_file( comm, "fld-wave-mpi-%03d-%04d.vtu" % (rank, istep), [ ("u", fields[0]), ("v", fields[1:]), ], overwrite=True ) fields = thaw(freeze(fields, actx), actx) fields = rk4_step(fields, t, dt, compiled_rhs) t += dt istep += 1 if logmgr: set_dt(logmgr, dt) logmgr.tick_after() final_soln = discr.norm(fields[0]) assert np.abs(final_soln - 0.04409852463947439) < 1e-14
def refine_for_global_qbx(lpot_source, wrangler, group_factory, kernel_length_scale=None, force_stage2_uniform_refinement_rounds=None, scaled_max_curvature_threshold=None, debug=None, maxiter=None, visualize=None, expansion_disturbance_tolerance=None, refiner=None): """ Entry point for calling the refiner. :arg lpot_source: An instance of :class:`QBXLayerPotentialSource`. :arg wrangler: An instance of :class:`RefinerWrangler`. :arg group_factory: An instance of :class:`meshmode.mesh.discretization.ElementGroupFactory`. Used for discretizing the coarse refined mesh. :arg kernel_length_scale: The kernel length scale, or *None* if not applicable. All panels are refined to below this size. :arg maxiter: The maximum number of refiner iterations. :returns: A tuple ``(lpot_source, *conn*)`` where ``lpot_source`` is the refined layer potential source, and ``conn`` is a :class:`meshmode.discretization.connection.DiscretizationConnection` going from the original mesh to the refined mesh. """ if maxiter is None: maxiter = 10 if debug is None: # FIXME: Set debug=False by default once everything works. debug = True if expansion_disturbance_tolerance is None: expansion_disturbance_tolerance = 0.025 if force_stage2_uniform_refinement_rounds is None: force_stage2_uniform_refinement_rounds = 0 # TODO: Stop doing redundant checks by avoiding panels which no longer need # refinement. from meshmode.mesh.refinement import RefinerWithoutAdjacency from meshmode.discretization.connection import ( ChainedDiscretizationConnection, make_same_mesh_connection) if refiner is not None: assert refiner.get_current_mesh() == lpot_source.density_discr.mesh else: # We may be handed a mesh that's already non-conforming, we don't rely # on adjacency, and the no-adjacency refiner is faster. refiner = RefinerWithoutAdjacency(lpot_source.density_discr.mesh) connections = [] # {{{ first stage refinement def visualize_refinement(niter, stage_nr, stage_name, flags): if not visualize: return if stage_nr == 1: discr = lpot_source.density_discr elif stage_nr == 2: discr = lpot_source.stage2_density_discr else: raise ValueError("unexpected stage number") flags = flags.get() logger.info("for stage %s: splitting %d/%d stage-%d elements", stage_name, np.sum(flags), discr.mesh.nelements, stage_nr) from meshmode.discretization.visualization import make_visualizer vis = make_visualizer(wrangler.queue, discr, 3) assert len(flags) == discr.mesh.nelements flags = flags.astype(np.bool) nodes_flags = np.zeros(discr.nnodes) for grp in discr.groups: meg = grp.mesh_el_group grp.view(nodes_flags)[flags[meg.element_nr_base:meg.nelements + meg.element_nr_base]] = 1 nodes_flags = cl.array.to_device(wrangler.queue, nodes_flags) vis_data = [ ("refine_flags", nodes_flags), ] if 0: from pytential import sym, bind bdry_normals = bind(discr, sym.normal(discr.ambient_dim))( wrangler.queue).as_vector(dtype=object) vis_data.append(("bdry_normals", bdry_normals), ) vis.write_vtk_file("refinement-%s-%03d.vtu" % (stage_name, niter), vis_data) def warn_max_iterations(): from warnings import warn warn( "QBX layer potential source refiner did not terminate " "after %d iterations (the maximum). " "You may pass 'visualize=True' to with_refinement() " "to see what area of the geometry is causing trouble. " "If the issue is disturbance of expansion disks, you may " "pass a slightly increased value (currently: %g) for " "_expansion_disturbance_tolerance in with_refinement(). " "As a last resort, " "you may use Python's warning filtering mechanism to " "not treat this warning as an error. " "The criteria triggering refinement in each iteration " "were: %s. " % (len(violated_criteria), expansion_disturbance_tolerance, ", ".join("%d: %s" % (i + 1, vc_text) for i, vc_text in enumerate(violated_criteria))), RefinerNotConvergedWarning) violated_criteria = [] iter_violated_criteria = ["start"] niter = 0 while iter_violated_criteria: iter_violated_criteria = [] niter += 1 if niter > maxiter: warn_max_iterations() break refine_flags = make_empty_refine_flags(wrangler.queue, lpot_source) if kernel_length_scale is not None: with ProcessLogger( logger, "checking kernel length scale to panel size ratio"): from pytential import bind, sym quad_resolution = bind( lpot_source, sym._quad_resolution(lpot_source.ambient_dim, dofdesc=sym.GRANULARITY_ELEMENT))( wrangler.queue) violates_kernel_length_scale = \ wrangler.check_element_prop_threshold( element_property=quad_resolution, threshold=kernel_length_scale, refine_flags=refine_flags, debug=debug) if violates_kernel_length_scale: iter_violated_criteria.append("kernel length scale") visualize_refinement(niter, 1, "kernel-length-scale", refine_flags) if scaled_max_curvature_threshold is not None: with ProcessLogger(logger, "checking scaled max curvature threshold"): from pytential import sym, bind scaled_max_curv = bind( lpot_source, sym.ElementwiseMax( sym._scaled_max_curvature(lpot_source.ambient_dim), dofdesc=sym.GRANULARITY_ELEMENT))(wrangler.queue) violates_scaled_max_curv = \ wrangler.check_element_prop_threshold( element_property=scaled_max_curv, threshold=scaled_max_curvature_threshold, refine_flags=refine_flags, debug=debug) if violates_scaled_max_curv: iter_violated_criteria.append("curvature") visualize_refinement(niter, 1, "curvature", refine_flags) if not iter_violated_criteria: # Only start building trees once the simple length-based criteria # are happy. # Build tree and auxiliary data. # FIXME: The tree should not have to be rebuilt at each iteration. tree = wrangler.build_tree(lpot_source) peer_lists = wrangler.find_peer_lists(tree) has_disturbed_expansions = \ wrangler.check_expansion_disks_undisturbed_by_sources( lpot_source, tree, peer_lists, expansion_disturbance_tolerance, refine_flags, debug) if has_disturbed_expansions: iter_violated_criteria.append("disturbed expansions") visualize_refinement(niter, 1, "disturbed-expansions", refine_flags) del tree del peer_lists if iter_violated_criteria: violated_criteria.append(" and ".join(iter_violated_criteria)) conn = wrangler.refine(lpot_source.density_discr, refiner, refine_flags, group_factory, debug) connections.append(conn) lpot_source = lpot_source.copy(density_discr=conn.to_discr) del refine_flags # }}} # {{{ second stage refinement iter_violated_criteria = ["start"] niter = 0 fine_connections = [] stage2_density_discr = lpot_source.density_discr while iter_violated_criteria: iter_violated_criteria = [] niter += 1 if niter > maxiter: warn_max_iterations() break # Build tree and auxiliary data. # FIXME: The tree should not have to be rebuilt at each iteration. tree = wrangler.build_tree(lpot_source, use_stage2_discr=True) peer_lists = wrangler.find_peer_lists(tree) refine_flags = make_empty_refine_flags(wrangler.queue, lpot_source, use_stage2_discr=True) has_insufficient_quad_res = \ wrangler.check_sufficient_source_quadrature_resolution( lpot_source, tree, peer_lists, refine_flags, debug) if has_insufficient_quad_res: iter_violated_criteria.append("insufficient quadrature resolution") visualize_refinement(niter, 2, "quad-resolution", refine_flags) if iter_violated_criteria: violated_criteria.append(" and ".join(iter_violated_criteria)) conn = wrangler.refine(stage2_density_discr, refiner, refine_flags, group_factory, debug) stage2_density_discr = conn.to_discr fine_connections.append(conn) lpot_source = lpot_source.copy( to_refined_connection=ChainedDiscretizationConnection( fine_connections)) del tree del refine_flags del peer_lists for round in range(force_stage2_uniform_refinement_rounds): conn = wrangler.refine( stage2_density_discr, refiner, np.ones(stage2_density_discr.mesh.nelements, dtype=np.bool), group_factory, debug) stage2_density_discr = conn.to_discr fine_connections.append(conn) lpot_source = lpot_source.copy( to_refined_connection=ChainedDiscretizationConnection( fine_connections)) # }}} lpot_source = lpot_source.copy(debug=debug, _refined_for_global_qbx=True) if len(connections) == 0: # FIXME: This is inefficient connection = make_same_mesh_connection(lpot_source.density_discr, lpot_source.density_discr) else: connection = ChainedDiscretizationConnection(connections) return lpot_source, connection
def flatten_chained_connection(queue, connection): """Collapse a connection into a direct connection. If the given connection is already a :class:`~meshmode.discretization.connection.DirectDiscretizationConnection` nothing is done. However, if the connection is a :class:`~meshmode.discretization.connection.ChainedDiscretizationConnection`, a new direct connection is constructed that transports from :attr:`connection.from_discr` to :attr:`connection.to_discr`. The new direct connection will have a number of groups and batches that is, at worse, the product of all the connections in the chain. For example, if we consider a connection between a discretization and a two-level refinement, both levels will have :math:`n` groups and :math:`m + 1` batches per group, where :math:`m` is the number of subdivisions of an element (exact number depends on implementation details in :func:`~meshmode.discretization.connection.make_refinement_connection`). However, a direct connection from level :math:`0` to level :math:`2` will have at worst :math:`n^2` groups and each group will have :math:`(m + 1)^2` batches. .. warning:: If a large number of connections is chained, the number of groups and batches can become very large. :arg queue: An instance of :class:`pyopencl.CommandQueue`. :arg connection: An instance of :class:`~meshmode.discretization.connection.DiscretizationConnection`. :return: An instance of :class:`~meshmode.discretization.connection.DirectDiscretizationConnection`. """ from meshmode.discretization.connection import ( DirectDiscretizationConnection, DiscretizationConnectionElementGroup, make_same_mesh_connection) if not hasattr(connection, 'connections'): return connection if not connection.connections: return make_same_mesh_connection(connection.to_discr, connection.from_discr) # recursively build direct connections connections = connection.connections direct_connections = [] for conn in connections: direct_connections.append(flatten_chained_connection(queue, conn)) # merge all the direct connections from_conn = direct_connections[0] for to_conn in direct_connections[1:]: el_table = _build_element_lookup_table(queue, from_conn) grp_to_grp, batch_info = _build_new_group_table(from_conn, to_conn) # distribute the indices to new groups and batches from_bins = [[np.empty(0, dtype=np.int) for _ in g] for g in batch_info] to_bins = [[np.empty(0, dtype=np.int) for _ in g] for g in batch_info] for (igrp, ibatch), (_, from_batch) in _iterbatches(from_conn.groups): from_to_element_indices = from_batch.to_element_indices.get(queue) for (jgrp, jbatch), (_, to_batch) in _iterbatches(to_conn.groups): igrp_new, ibatch_new = grp_to_grp[igrp, ibatch, jgrp, jbatch] jfrom = to_batch.from_element_indices.get(queue) jto = to_batch.to_element_indices.get(queue) mask = np.isin(jfrom, from_to_element_indices) from_bins[igrp_new][ibatch_new] = \ np.hstack([from_bins[igrp_new][ibatch_new], el_table[igrp][jfrom[mask]]]) to_bins[igrp_new][ibatch_new] = \ np.hstack([to_bins[igrp_new][ibatch_new], jto[mask]]) # build new groups groups = [] for igrp, (from_bin, to_bin) in enumerate(zip(from_bins, to_bins)): groups.append( DiscretizationConnectionElementGroup( list( _build_batches(queue, from_bin, to_bin, batch_info[igrp])))) from_conn = DirectDiscretizationConnection( from_discr=from_conn.from_discr, to_discr=to_conn.to_discr, groups=groups, is_surjective=connection.is_surjective) return from_conn
def main(snapshot_pattern="wave-eager-{step:04d}-{rank:04d}.pkl", restart_step=None): """Drive the example.""" cl_ctx = cl.create_some_context() queue = cl.CommandQueue(cl_ctx) actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool( cl_tools.ImmediateAllocator(queue))) from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() num_parts = comm.Get_size() if restart_step is None: from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis mesh_dist = MPIMeshDistributor(comm) dim = 2 nel_1d = 16 if mesh_dist.is_mananger_rank(): from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh(a=(-0.5, ) * dim, b=(0.5, ) * dim, nelements_per_axis=(nel_1d, ) * dim) print("%d elements" % mesh.nelements) part_per_element = get_partition_by_pymetis(mesh, num_parts) local_mesh = mesh_dist.send_mesh_parts(mesh, part_per_element, num_parts) del mesh else: local_mesh = mesh_dist.receive_mesh_part() fields = None else: from mirgecom.restart import read_restart_data restart_data = read_restart_data( actx, snapshot_pattern.format(step=restart_step, rank=rank)) local_mesh = restart_data["local_mesh"] nel_1d = restart_data["nel_1d"] assert comm.Get_size() == restart_data["num_parts"] order = 3 discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) if local_mesh.dim == 2: # no deep meaning here, just a fudge factor dt = 0.7 / (nel_1d * order**2) elif dim == 3: # no deep meaning here, just a fudge factor dt = 0.4 / (nel_1d * order**2) else: raise ValueError("don't have a stable time step guesstimate") t_final = 3 if restart_step is None: t = 0 istep = 0 fields = flat_obj_array(bump(actx, discr), [discr.zeros(actx) for i in range(discr.dim)]) else: t = restart_data["t"] istep = restart_step assert istep == restart_step restart_fields = restart_data["fields"] old_order = restart_data["order"] if old_order != order: old_discr = EagerDGDiscretization(actx, local_mesh, order=old_order, mpi_communicator=comm) from meshmode.discretization.connection import make_same_mesh_connection connection = make_same_mesh_connection( actx, discr.discr_from_dd("vol"), old_discr.discr_from_dd("vol")) fields = connection(restart_fields) else: fields = restart_fields vis = make_visualizer(discr) def rhs(t, w): return wave_operator(discr, c=1, w=w) while t < t_final: # restart must happen at beginning of step if istep % 100 == 0 and ( # Do not overwrite the restart file that we just read. istep != restart_step): from mirgecom.restart import write_restart_file write_restart_file(actx, restart_data={ "local_mesh": local_mesh, "order": order, "fields": fields, "t": t, "step": istep, "nel_1d": nel_1d, "num_parts": num_parts }, filename=snapshot_pattern.format(step=istep, rank=rank), comm=comm) if istep % 10 == 0: print(istep, t, discr.norm(fields[0])) vis.write_parallel_vtk_file( comm, "fld-wave-eager-mpi-%03d-%04d.vtu" % (rank, istep), [ ("u", fields[0]), ("v", fields[1:]), ]) fields = rk4_step(fields, t, dt, rhs) t += dt istep += 1
def main(): import logging logging.basicConfig(level=logging.INFO) ctx = cl.create_some_context() queue = cl.CommandQueue(ctx) if 1: ext = 0.5 mesh = generate_regular_rect_mesh(a=(-ext / 2, -ext / 2), b=(ext / 2, ext / 2), n=(int(ext / h), int(ext / h))) else: mesh = generate_gmsh(FileSource("circle.step"), 2, order=mesh_order, force_ambient_dim=2, other_options=[ "-string", "Mesh.CharacteristicLengthMax = %g;" % h ]) logger.info("%d elements" % mesh.nelements) # {{{ discretizations and connections vol_discr = Discretization( ctx, mesh, InterpolatoryQuadratureSimplexGroupFactory(vol_quad_order)) ovsmp_vol_discr = Discretization( ctx, mesh, InterpolatoryQuadratureSimplexGroupFactory(vol_ovsmp_quad_order)) from meshmode.mesh import BTAG_ALL from meshmode.discretization.connection import (make_face_restriction, make_same_mesh_connection) bdry_connection = make_face_restriction( vol_discr, InterpolatoryQuadratureSimplexGroupFactory(bdry_quad_order), BTAG_ALL) bdry_discr = bdry_connection.to_discr vol_to_ovsmp_vol = make_same_mesh_connection(ovsmp_vol_discr, vol_discr) # }}} # {{{ visualizers vol_vis = make_visualizer(queue, vol_discr, 20) bdry_vis = make_visualizer(queue, bdry_discr, 20) # }}} vol_x = vol_discr.nodes().with_queue(queue) ovsmp_vol_x = ovsmp_vol_discr.nodes().with_queue(queue) rhs = rhs_func(vol_x[0], vol_x[1]) poisson_true_sol = sol_func(vol_x[0], vol_x[1]) vol_vis.write_vtk_file("volume.vtu", [("f", rhs)]) bdry_normals = bind(bdry_discr, p.normal( mesh.ambient_dim))(queue).as_vector(dtype=object) bdry_vis.write_vtk_file("boundary.vtu", [("normals", bdry_normals)]) bdry_nodes = bdry_discr.nodes().with_queue(queue) bdry_f = rhs_func(bdry_nodes[0], bdry_nodes[1]) bdry_f_2 = bdry_connection(queue, rhs) bdry_vis.write_vtk_file("y.vtu", [("f", bdry_f_2)]) if 0: vol_vis.show_scalar_in_mayavi(rhs, do_show=False) bdry_vis.show_scalar_in_mayavi(bdry_f - bdry_f_2, line_width=10, do_show=False) import mayavi.mlab as mlab mlab.colorbar() mlab.show() # {{{ compute volume potential from sumpy.qbx import LayerPotential from sumpy.expansion.local import LineTaylorLocalExpansion def get_kernel(): from sumpy.symbolic import pymbolic_real_norm_2 from pymbolic.primitives import make_sym_vector from pymbolic import var d = make_sym_vector("d", 3) r = pymbolic_real_norm_2(d[:-1]) # r3d = pymbolic_real_norm_2(d) #expr = var("log")(r3d) log = var("log") sqrt = var("sqrt") a = d[-1] expr = log(r) expr = log(sqrt(r**2 + a**2)) expr = log(sqrt(r + a**2)) #expr = log(sqrt(r**2 + a**2))-a**2/2/(r**2+a**2) #expr = 2*log(sqrt(r**2 + a**2)) scaling = 1 / (2 * var("pi")) from sumpy.kernel import ExpressionKernel return ExpressionKernel(dim=3, expression=expr, global_scaling_const=scaling, is_complex_valued=False) laplace_2d_in_3d_kernel = get_kernel() layer_pot = LayerPotential( ctx, [LineTaylorLocalExpansion(laplace_2d_in_3d_kernel, order=0)]) targets = cl.array.zeros(queue, (3, ) + vol_x.shape[1:], vol_x.dtype) targets[:2] = vol_x center_dist = 0.125 * np.min( cl.clmath.sqrt( bind(vol_discr, p.area_element(mesh.ambient_dim, mesh.dim))(queue)).get()) centers = make_obj_array( [ci.copy().reshape(vol_discr.nnodes) for ci in targets]) centers[2][:] = center_dist print(center_dist) sources = cl.array.zeros(queue, (3, ) + ovsmp_vol_x.shape[1:], ovsmp_vol_x.dtype) sources[:2] = ovsmp_vol_x ovsmp_rhs = vol_to_ovsmp_vol(queue, rhs) ovsmp_vol_weights = bind( ovsmp_vol_discr, p.area_element(mesh.ambient_dim, mesh.dim) * p.QWeight())(queue) print("volume: %d source nodes, %d target nodes" % (ovsmp_vol_discr.nnodes, vol_discr.nnodes)) evt, (vol_pot, ) = layer_pot( queue, targets=targets.reshape(3, vol_discr.nnodes), centers=centers, sources=sources.reshape(3, ovsmp_vol_discr.nnodes), strengths=((ovsmp_vol_weights * ovsmp_rhs).reshape( ovsmp_vol_discr.nnodes), ), expansion_radii=np.zeros(vol_discr.nnodes), ) vol_pot_bdry = bdry_connection(queue, vol_pot) # }}} # {{{ solve bvp from sumpy.kernel import LaplaceKernel from pytential.symbolic.pde.scalar import DirichletOperator op = DirichletOperator(LaplaceKernel(2), -1, use_l2_weighting=True) sym_sigma = sym.var("sigma") op_sigma = op.operator(sym_sigma) from pytential.qbx import QBXLayerPotentialSource qbx = QBXLayerPotentialSource( bdry_discr, fine_order=bdry_ovsmp_quad_order, qbx_order=qbx_order, fmm_order=fmm_order, ) bound_op = bind(qbx, op_sigma) poisson_bc = poisson_bc_func(bdry_nodes[0], bdry_nodes[1]) bvp_bc = poisson_bc - vol_pot_bdry bdry_f = rhs_func(bdry_nodes[0], bdry_nodes[1]) bvp_rhs = bind(bdry_discr, op.prepare_rhs(sym.var("bc")))(queue, bc=bvp_bc) from pytential.solve import gmres gmres_result = gmres(bound_op.scipy_op(queue, "sigma", dtype=np.float64), bvp_rhs, tol=1e-14, progress=True, hard_failure=False) sigma = gmres_result.solution print("gmres state:", gmres_result.state) # }}} bvp_sol = bind((qbx, vol_discr), op.representation(sym_sigma))(queue, sigma=sigma) poisson_sol = bvp_sol + vol_pot poisson_err = poisson_sol - poisson_true_sol rel_err = (norm(vol_discr, queue, poisson_err) / norm(vol_discr, queue, poisson_true_sol)) bdry_vis.write_vtk_file("poisson-boundary.vtu", [ ("vol_pot_bdry", vol_pot_bdry), ("sigma", sigma), ]) vol_vis.write_vtk_file("poisson-volume.vtu", [ ("bvp_sol", bvp_sol), ("poisson_sol", poisson_sol), ("poisson_true_sol", poisson_true_sol), ("poisson_err", poisson_err), ("vol_pot", vol_pot), ("rhs", rhs), ]) print("h = %s" % h) print("mesh_order = %s" % mesh_order) print("vol_quad_order = %s" % vol_quad_order) print("vol_ovsmp_quad_order = %s" % vol_ovsmp_quad_order) print("bdry_quad_order = %s" % bdry_quad_order) print("bdry_ovsmp_quad_order = %s" % bdry_ovsmp_quad_order) print("qbx_order = %s" % qbx_order) #print("vol_qbx_order = %s" % vol_qbx_order) print("fmm_order = %s" % fmm_order) print() print("rel err: %g" % rel_err)
def refine_for_global_qbx(lpot_source, wrangler, group_factory, kernel_length_scale=None, force_stage2_uniform_refinement_rounds=None, scaled_max_curvature_threshold=None, debug=None, maxiter=None, visualize=None, expansion_disturbance_tolerance=None, refiner=None): """ Entry point for calling the refiner. :arg lpot_source: An instance of :class:`QBXLayerPotentialSource`. :arg wrangler: An instance of :class:`RefinerWrangler`. :arg group_factory: An instance of :class:`meshmode.mesh.discretization.ElementGroupFactory`. Used for discretizing the coarse refined mesh. :arg kernel_length_scale: The kernel length scale, or *None* if not applicable. All panels are refined to below this size. :arg maxiter: The maximum number of refiner iterations. :returns: A tuple ``(lpot_source, *conn*)`` where ``lpot_source`` is the refined layer potential source, and ``conn`` is a :class:`meshmode.discretization.connection.DiscretizationConnection` going from the original mesh to the refined mesh. """ if maxiter is None: maxiter = 10 if debug is None: # FIXME: Set debug=False by default once everything works. debug = True if expansion_disturbance_tolerance is None: expansion_disturbance_tolerance = 0.025 if force_stage2_uniform_refinement_rounds is None: force_stage2_uniform_refinement_rounds = 0 # TODO: Stop doing redundant checks by avoiding panels which no longer need # refinement. from meshmode.mesh.refinement import RefinerWithoutAdjacency from meshmode.discretization.connection import ( ChainedDiscretizationConnection, make_same_mesh_connection) if refiner is not None: assert refiner.get_current_mesh() == lpot_source.density_discr.mesh else: # We may be handed a mesh that's already non-conforming, we don't rely # on adjacency, and the no-adjacency refiner is faster. refiner = RefinerWithoutAdjacency(lpot_source.density_discr.mesh) connections = [] # {{{ first stage refinement def visualize_refinement(niter, stage_nr, stage_name, flags): if not visualize: return if stage_nr == 1: discr = lpot_source.density_discr elif stage_nr == 2: discr = lpot_source.stage2_density_discr else: raise ValueError("unexpected stage number") flags = flags.get() logger.info("for stage %s: splitting %d/%d stage-%d elements", stage_name, np.sum(flags), discr.mesh.nelements, stage_nr) from meshmode.discretization.visualization import make_visualizer vis = make_visualizer(wrangler.queue, discr, 3) assert len(flags) == discr.mesh.nelements flags = flags.astype(np.bool) nodes_flags = np.zeros(discr.nnodes) for grp in discr.groups: meg = grp.mesh_el_group grp.view(nodes_flags)[ flags[meg.element_nr_base:meg.nelements+meg.element_nr_base]] = 1 nodes_flags = cl.array.to_device(wrangler.queue, nodes_flags) vis_data = [ ("refine_flags", nodes_flags), ] if 0: from pytential import sym, bind bdry_normals = bind(discr, sym.normal(discr.ambient_dim))( wrangler.queue).as_vector(dtype=object) vis_data.append(("bdry_normals", bdry_normals),) vis.write_vtk_file("refinement-%s-%03d.vtu" % (stage_name, niter), vis_data) def warn_max_iterations(): from warnings import warn warn( "QBX layer potential source refiner did not terminate " "after %d iterations (the maximum). " "You may pass 'visualize=True' to with_refinement() " "to see what area of the geometry is causing trouble. " "If the issue is disturbance of expansion disks, you may " "pass a slightly increased value (currently: %g) for " "_expansion_disturbance_tolerance in with_refinement(). " "As a last resort, " "you may use Python's warning filtering mechanism to " "not treat this warning as an error. " "The criteria triggering refinement in each iteration " "were: %s. " % ( len(violated_criteria), expansion_disturbance_tolerance, ", ".join( "%d: %s" % (i+1, vc_text) for i, vc_text in enumerate(violated_criteria))), RefinerNotConvergedWarning) violated_criteria = [] iter_violated_criteria = ["start"] niter = 0 while iter_violated_criteria: iter_violated_criteria = [] niter += 1 if niter > maxiter: warn_max_iterations() break refine_flags = make_empty_refine_flags(wrangler.queue, lpot_source) if kernel_length_scale is not None: with ProcessLogger(logger, "checking kernel length scale to panel size ratio"): violates_kernel_length_scale = \ wrangler.check_element_prop_threshold( element_property=( lpot_source._coarsest_quad_resolution( "npanels")), threshold=kernel_length_scale, refine_flags=refine_flags, debug=debug) if violates_kernel_length_scale: iter_violated_criteria.append("kernel length scale") visualize_refinement( niter, 1, "kernel-length-scale", refine_flags) if scaled_max_curvature_threshold is not None: with ProcessLogger(logger, "checking scaled max curvature threshold"): from pytential.qbx.utils import to_last_dim_length from pytential import sym, bind scaled_max_curv = to_last_dim_length( lpot_source.density_discr, bind(lpot_source, sym.ElementwiseMax( sym._scaled_max_curvature( lpot_source.density_discr.ambient_dim))) (wrangler.queue), "npanels") violates_scaled_max_curv = \ wrangler.check_element_prop_threshold( element_property=scaled_max_curv, threshold=scaled_max_curvature_threshold, refine_flags=refine_flags, debug=debug) if violates_scaled_max_curv: iter_violated_criteria.append("curvature") visualize_refinement(niter, 1, "curvature", refine_flags) if not iter_violated_criteria: # Only start building trees once the simple length-based criteria # are happy. # Build tree and auxiliary data. # FIXME: The tree should not have to be rebuilt at each iteration. tree = wrangler.build_tree(lpot_source) peer_lists = wrangler.find_peer_lists(tree) has_disturbed_expansions = \ wrangler.check_expansion_disks_undisturbed_by_sources( lpot_source, tree, peer_lists, expansion_disturbance_tolerance, refine_flags, debug) if has_disturbed_expansions: iter_violated_criteria.append("disturbed expansions") visualize_refinement(niter, 1, "disturbed-expansions", refine_flags) del tree del peer_lists if iter_violated_criteria: violated_criteria.append(" and ".join(iter_violated_criteria)) conn = wrangler.refine( lpot_source.density_discr, refiner, refine_flags, group_factory, debug) connections.append(conn) lpot_source = lpot_source.copy(density_discr=conn.to_discr) del refine_flags # }}} # {{{ second stage refinement iter_violated_criteria = ["start"] niter = 0 fine_connections = [] stage2_density_discr = lpot_source.density_discr while iter_violated_criteria: iter_violated_criteria = [] niter += 1 if niter > maxiter: warn_max_iterations() break # Build tree and auxiliary data. # FIXME: The tree should not have to be rebuilt at each iteration. tree = wrangler.build_tree(lpot_source, use_stage2_discr=True) peer_lists = wrangler.find_peer_lists(tree) refine_flags = make_empty_refine_flags( wrangler.queue, lpot_source, use_stage2_discr=True) has_insufficient_quad_res = \ wrangler.check_sufficient_source_quadrature_resolution( lpot_source, tree, peer_lists, refine_flags, debug) if has_insufficient_quad_res: iter_violated_criteria.append("insufficient quadrature resolution") visualize_refinement(niter, 2, "quad-resolution", refine_flags) if iter_violated_criteria: violated_criteria.append(" and ".join(iter_violated_criteria)) conn = wrangler.refine( stage2_density_discr, refiner, refine_flags, group_factory, debug) stage2_density_discr = conn.to_discr fine_connections.append(conn) lpot_source = lpot_source.copy( to_refined_connection=ChainedDiscretizationConnection( fine_connections)) del tree del refine_flags del peer_lists for round in range(force_stage2_uniform_refinement_rounds): conn = wrangler.refine( stage2_density_discr, refiner, np.ones(stage2_density_discr.mesh.nelements, dtype=np.bool), group_factory, debug) stage2_density_discr = conn.to_discr fine_connections.append(conn) lpot_source = lpot_source.copy( to_refined_connection=ChainedDiscretizationConnection( fine_connections)) # }}} lpot_source = lpot_source.copy(debug=debug, _refined_for_global_qbx=True) if len(connections) == 0: # FIXME: This is inefficient connection = make_same_mesh_connection( lpot_source.density_discr, lpot_source.density_discr) else: connection = ChainedDiscretizationConnection(connections) return lpot_source, connection
def flatten_chained_connection(queue, connection): """Collapse a connection into a direct connection. If the given connection is already a :class:`~meshmode.discretization.connection.DirectDiscretizationConnection` nothing is done. However, if the connection is a :class:`~meshmode.discretization.connection.ChainedDiscretizationConnection`, a new direct connection is constructed that transports from :attr:`connection.from_discr` to :attr:`connection.to_discr`. The new direct connection will have a number of groups and batches that is, at worse, the product of all the connections in the chain. For example, if we consider a connection between a discretization and a two-level refinement, both levels will have :math:`n` groups and :math:`m + 1` batches per group, where :math:`m` is the number of subdivisions of an element (exact number depends on implementation details in :func:`~meshmode.discretization.connection.make_refinement_connection`). However, a direct connection from level :math:`0` to level :math:`2` will have at worst :math:`n^2` groups and each group will have :math:`(m + 1)^2` batches. .. warning:: If a large number of connections is chained, the number of groups and batches can become very large. :arg queue: An instance of :class:`pyopencl.CommandQueue`. :arg connection: An instance of :class:`~meshmode.discretization.connection.DiscretizationConnection`. :return: An instance of :class:`~meshmode.discretization.connection.DirectDiscretizationConnection`. """ from meshmode.discretization.connection import ( DirectDiscretizationConnection, DiscretizationConnectionElementGroup, make_same_mesh_connection) if not hasattr(connection, 'connections'): return connection if not connection.connections: return make_same_mesh_connection(connection.to_discr, connection.from_discr) # recursively build direct connections connections = connection.connections direct_connections = [] for conn in connections: direct_connections.append(flatten_chained_connection(queue, conn)) # merge all the direct connections from_conn = direct_connections[0] for to_conn in direct_connections[1:]: el_table = _build_element_lookup_table(queue, from_conn) grp_to_grp, batch_info = _build_new_group_table(from_conn, to_conn) # distribute the indices to new groups and batches from_bins = [[np.empty(0, dtype=np.int) for _ in g] for g in batch_info] to_bins = [[np.empty(0, dtype=np.int) for _ in g] for g in batch_info] for (igrp, ibatch), (_, from_batch) in _iterbatches(from_conn.groups): from_to_element_indices = from_batch.to_element_indices.get(queue) for (jgrp, jbatch), (_, to_batch) in _iterbatches(to_conn.groups): igrp_new, ibatch_new = grp_to_grp[igrp, ibatch, jgrp, jbatch] jfrom = to_batch.from_element_indices.get(queue) jto = to_batch.to_element_indices.get(queue) mask = np.isin(jfrom, from_to_element_indices) from_bins[igrp_new][ibatch_new] = \ np.hstack([from_bins[igrp_new][ibatch_new], el_table[igrp][jfrom[mask]]]) to_bins[igrp_new][ibatch_new] = \ np.hstack([to_bins[igrp_new][ibatch_new], jto[mask]]) # build new groups groups = [] for igrp, (from_bin, to_bin) in enumerate(zip(from_bins, to_bins)): groups.append(DiscretizationConnectionElementGroup( list(_build_batches(queue, from_bin, to_bin, batch_info[igrp])))) from_conn = DirectDiscretizationConnection( from_discr=from_conn.from_discr, to_discr=to_conn.to_discr, groups=groups, is_surjective=connection.is_surjective) return from_conn