def _init_reg_banks(self):
self._regs, self._regidx = [], list(range(self.nreg))
self._idxcurr = 0
# Create a proxylist of matrix-banks for each storage register
for i in self._regidx:
self._regs.append(
proxylist([self.backend.matrix_bank(em, i)
for em in self.system.ele_banks])
)
def _load_int_inters(self, rallocs, mesh, elemap, **kwargs):
key = 'con_p{0}'.format(rallocs.prank)
lhs, rhs = mesh[key].astype('U4,i4,i1,i1').tolist()
int_inters = self.intinterscls(self.backend, lhs, rhs, elemap,
self.cfg, **kwargs)
# Although we only have a single internal interfaces instance
# we wrap it in a proxylist for consistency
return proxylist([int_inters])
def _get_kernels(self, name, nargs, **kwargs):
# Transpose from [nregs][neletypes] to [neletypes][nregs]
transregs = zip(*self._regs)
# Generate an kernel for each element type
kerns = proxylist([])
for tr in transregs:
kerns.append(self.backend.kernel(name, *tr[:nargs], **kwargs))
return kerns
def _get_axnpby_kerns_dgfs(self, indices, **kwargs):
# Transpose from [nregs][neletypes] to [neletypes][nregs]
transregs = zip(*self._regs)
# Generate an kernel for each element type
kerns = proxylist([])
for tr in transregs:
args = list([tr[indx] for indx in indices])
kerns.append(self.backend.kernel('axnpby',
*args, **kwargs))
return kerns
def _get_copy_from_reg_kerns(self, inp, out, **kwargs):
# Transpose from [nregs][neletypes] to [neletypes][nregs]
transregs = zip(*self._regs)
fsoln = self.system.eles_scal_upts_inb_full
# Generate an kernel for each element type
kerns = proxylist([])
for tr, ftr in zip(transregs, fsoln):
args = list([tr[inp], ftr[out]])
kerns.append(self.backend.kernel('copy_from_reg',*args,**kwargs))
return kerns
def _load_mpi_inters(self, rallocs, mesh, elemap, **kwargs):
lhsprank = rallocs.prank
mpi_inters = proxylist([])
for rhsprank in rallocs.prankconn[lhsprank]:
rhsmrank = rallocs.pmrankmap[rhsprank]
interarr = mesh['con_p%dp%d' % (lhsprank, rhsprank)]
interarr = interarr.astype('U4,i4,i1,i1').tolist()
mpiiface = self.mpiinterscls(self.backend, interarr, rhsmrank,
rallocs, elemap, self.cfg, **kwargs)
mpi_inters.append(mpiiface)
return mpi_inters
def _load_eles(self, rallocs, mesh, initsoln, nreg, nonce, **kwargs):
basismap = {b.name: b for b in subclasses(BaseShape, just_leaf=True)}
# Look for and load each element type from the mesh
elemap = OrderedDict()
for f in mesh:
m = re.match('spt_(.+?)_p%d$' % rallocs.prank, f)
if m:
# Element type
t = m.group(1)
elemap[t] = self.elementscls(basismap[t], mesh[f], self.cfg,
**kwargs)
# Construct a proxylist to simplify collective operations
eles = proxylist(elemap.values())
# Set the initial conditions either from a frfss file or from
# explicit expressions in the config file
if initsoln and (not ('vm' in kwargs)):
# Load the config and stats files from the solution
solncfg = Inifile(initsoln['config'])
solnsts = Inifile(initsoln['stats'])
# Get the names of the conserved variables (fields)
solnfields = solnsts.get('data', 'fields', '')
currfields = ','.join(eles[0].convarmap[eles[0].ndims])
# Ensure they match up
if solnfields and solnfields != currfields:
raise RuntimeError('Invalid solution for system')
# Process the solution
for k, ele in elemap.items():
soln = initsoln['soln_%s_p%d' % (k, rallocs.prank)]
ele.set_ics_from_soln(soln, solncfg)
else:
eles.set_ics_from_cfg()
# Allocate these elements on the backend
eles.set_backend(self.backend, nreg, nonce)
return eles, elemap
def get_nregs_moms(self, nreg_moms):
if self.nreg_moms != None and self.nreg_moms != nreg_moms:
raise RuntimeError("Some issue")
if self.nreg_moms == nreg_moms: return list(range(nreg_moms))
# Moment storage
nalph = self.sm.nalph
eles_scal_upts_inb_moms = proxylist(self.ele_banks)
# loop over the sub-domains in the full mixed domain
for t, (nupts, nvars, neles) in enumerate(self.ele_shapes):
eles_scal_upts_inb_moms[t] = self.backend.matrix_bank([
self.backend.matrix((nupts * neles, nalph))
for i in range(nreg_moms)
])
self.eles_scal_upts_inb_moms = eles_scal_upts_inb_moms
self.nreg_moms = nreg_moms
self.nalph = nalph
return list(range(nreg_moms))
def _load_bc_inters(self, rallocs, mesh, elemap, **kwargs):
bccls = self.bbcinterscls
bcmap = {b.type: b for b in subclasses(bccls, just_leaf=True)}
bc_inters = proxylist([])
for f in mesh:
m = re.match('bcon_(.+?)_p%d$' % rallocs.prank, f)
if m:
# Get the region name
rgn = m.group(1)
# Determine the config file section
cfgsect = 'soln-bcs-%s' % rgn
# Get the interface
interarr = mesh[f].astype('U4,i4,i1,i1').tolist()
# Instantiate
bcclass = bcmap[self.cfg.get(cfgsect, 'type')]
bciface = bcclass(self.backend, interarr, elemap, cfgsect,
self.cfg, **kwargs)
bc_inters.append(bciface)
return bc_inters
def __init__(self, backend, rallocs, mesh, initsoln, nreg, cfg):
if(not backend.name=='cuda'):
raise ValueError("CUDA backend supported!")
# load the velocity mesh
self.vm = self.velocitymeshcls(backend, cfg, self._nspcs)
cv = self.vm.cv()
vsize = self.vm.vsize()
# need to define the expressions
# the prefix "f_" should be same as in elementcls distvar
# size of distvar should be equal to NvBatchSize
for ivar in range(self.vm.NvBatchSize()):
cfg.set('soln-ics', 'f_' + str(ivar), '0.')
# now, we can initialize things
super().__init__(backend, rallocs, mesh, initsoln, nreg, cfg,
vm=self.vm)
print('Finished initializing the BaseSystem')
# define the time-step
minjac = 100.0
for t, ele in self.ele_map.items():
djac = ele.djac_at_np('upts')
minjac = np.min([minjac, np.min(djac)])
advmax = self.vm.L()
unitCFLdt = np.array([np.sqrt(minjac)/advmax/self.ndims])
gunitCFLdt = np.zeros(1)
# MPI info
comm, rank, root = get_comm_rank_root()
# Reduce and, if we are the root rank, output
if rank != root:
comm.Reduce(unitCFLdt, gunitCFLdt, op=get_mpi('min'), root=root)
else:
comm.Reduce(unitCFLdt, gunitCFLdt, op=get_mpi('min'), root=root)
print("Time-step for unit CFL:", gunitCFLdt)
print("The actual time-step will depend on DG order CFL")
# load the scattering model
smn = cfg.get('scattering-model', 'type')
scatteringcls = subclass_where(DGFSBiScatteringModel,
scattering_model=smn)
self.sm = scatteringcls(backend, self.cfg, self.vm)
# Allocate and bank the storage required by the time integrator
#eles_scal_upts_full = proxylist(self.ele_banks)
eles_scal_upts_inb_full = proxylist(self.ele_banks)
spcs_eles_scal_upts_full = [list(self.ele_banks)
for spcs in range(self._nspcs)]
if initsoln:
#raise ValueError("Not implemented")
# Load the config and stats files from the solution
solncfg = Inifile(initsoln['config'])
solnsts = Inifile(initsoln['stats'])
# Get the names of the conserved variables (fields)
solnfields = solnsts.get('data', 'fields', '')
# see dgfsdistwriterbi.py plugin
currfields = []
fields = ['f_'+str(i) for i in range(vsize)]
lf = len(fields)
for p in range(self._nspcs):
currfields.extend(fields)
for ivar in range(-1,-lf-1,-1):
currfields[ivar] += ':'+str(p+1)
currfields = ','.join(currfields)
# Ensure they match up
if solnfields and solnfields != currfields:
raise RuntimeError('Invalid solution for system')
# Ensure the solnfields are not empty
if not solnfields:
raise RuntimeError('Invalid solution for system')
nreg0 = nreg//self._nspcs
assert nreg==nreg0*self._nspcs, "Should be multiple of nspcs"
# Process the solution
for t, (k, ele) in enumerate(self.ele_map.items()):
soln = initsoln['soln_%s_p%d' % (k, rallocs.prank)]
#ele.set_ics_from_soln(soln, solncfg)
# Recreate the existing solution basis
solnb = ele.basis.__class__(None, solncfg)
# Form the interpolation operator
interp = solnb.ubasis.nodal_basis_at(ele.basis.upts)
# Apply and reshape
data = np.dot(interp, soln.reshape(solnb.nupts, -1))
data = data.reshape(ele.nupts, self._nspcs*vsize, ele.neles)
for p in range(self._nspcs):
spcs_eles_scal_upts_full[p][t] = data[:,
p*vsize:(p+1)*vsize, :]
else:
# load the initial condition model
icn = cfg.get('soln-ics', 'type')
initcondcls = subclass_where(DGFSBiInitCondition, model=icn)
ic = initcondcls(backend, cfg, self.vm, 'soln-ics')
#initvals = ic.get_init_vals()
nreg0 = nreg//self._nspcs
assert nreg==nreg0*self._nspcs, "Should be multiple of nspcs"
# loop over the sub-domains in the full mixed domain
for p in range(self._nspcs):
for t, ele in enumerate(self.ele_map.values()):
spcs_eles_scal_upts_full[p][t] = np.empty(
(ele.nupts, vsize, ele.neles))
ic.apply_init_vals(p, spcs_eles_scal_upts_full[p][t], ele)
# Convert from primitive to conservative form if needed
nreg0 = nreg//self._nspcs
assert nreg==nreg0*self._nspcs, "Should be multiple of nspcs"
for t in range(len(eles_scal_upts_inb_full)):
scal_upts_full = []
for p in range(self._nspcs):
if p==0:
scal_upts_full = [
backend.matrix(spcs_eles_scal_upts_full[p][t].shape,
spcs_eles_scal_upts_full[p][t], tags={'align'})
for i in range(nreg0)]
else:
scal_upts_full.extend([
backend.matrix(spcs_eles_scal_upts_full[p][t].shape,
spcs_eles_scal_upts_full[p][t], tags={'align'})
for i in range(nreg0)])
eles_scal_upts_inb_full[t] = backend.matrix_bank(scal_upts_full)
#eles_scal_upts_outb_full[t] = backend.matrix_bank(scal_upts_full)
self.eles_scal_upts_inb_full = eles_scal_upts_inb_full
del spcs_eles_scal_upts_full
def __init__(self, kernels):
self._kernels = proxylist(kernels)
def __init__(self, backend, systemcls, rallocs, mesh, initsoln, cfg):
self.backend = backend
self.rallocs = rallocs
self.isrestart = initsoln is not None
self.cfg = cfg
self.prevcfgs = {f: initsoln[f] for f in initsoln or []
if f.startswith('config-')}
# Ensure the system is compatible with our formulation
if self.formulation not in systemcls.elementscls.formulations:
raise RuntimeError(
'System {0} does not support time stepping formulation {1}'
.format(systemcls.name, self.formulation)
)
# Start time
self.tstart = cfg.getfloat('solver-time-integrator', 'tstart', 0.0)
self.tend = cfg.getfloat('solver-time-integrator', 'tend')
# Current time; defaults to tstart unless restarting
if self.isrestart:
stats = Inifile(initsoln['stats'])
self.tcurr = stats.getfloat('solver-time-integrator', 'tcurr')
else:
self.tcurr = self.tstart
# List of target times to advance to
self.tlist = deque([self.tend])
# Accepted and rejected step counters
self.nacptsteps = 0
self.nrjctsteps = 0
self.nacptchain = 0
# Current and minimum time steps
self._dt = cfg.getfloat('solver-time-integrator', 'dt')
self.dtmin = cfg.getfloat('solver-time-integrator', 'dt-min', 1e-12)
# Determine the amount of temp storage required by this method
self.nreg = self._stepper_nregs
# Construct the relevant mesh partition
self._init_system(systemcls, backend, rallocs, mesh, initsoln)
# Storage for register banks and current index
self._init_reg_banks()
# Extract the UUID of the mesh (to be saved with solutions)
self.mesh_uuid = mesh['mesh_uuid']
# Get a queue for subclasses to use
self._queue = backend.queue()
# Global degree of freedom count
self._gndofs = self._get_gndofs()
# Solution cache
self._curr_soln = None
# Add kernel cache
self._axnpby_kerns = {}
# Record the starting wall clock time
self._wstart = time.time()
# Event handlers for advance_to
self.completed_step_handlers = proxylist(self._get_plugins())
# Delete the memory-intensive elements map from the system
del self.system.ele_map