def _init_reg_banks(self): self._regs, self._regidx = [], list(range(self.nreg)) self._idxcurr = 0 # Create a proxylist of matrix-banks for each storage register for i in self._regidx: self._regs.append( proxylist([self.backend.matrix_bank(em, i) for em in self.system.ele_banks]) )
def _load_int_inters(self, rallocs, mesh, elemap, **kwargs): key = 'con_p{0}'.format(rallocs.prank) lhs, rhs = mesh[key].astype('U4,i4,i1,i1').tolist() int_inters = self.intinterscls(self.backend, lhs, rhs, elemap, self.cfg, **kwargs) # Although we only have a single internal interfaces instance # we wrap it in a proxylist for consistency return proxylist([int_inters])
def _get_kernels(self, name, nargs, **kwargs): # Transpose from [nregs][neletypes] to [neletypes][nregs] transregs = zip(*self._regs) # Generate an kernel for each element type kerns = proxylist([]) for tr in transregs: kerns.append(self.backend.kernel(name, *tr[:nargs], **kwargs)) return kerns
def _get_axnpby_kerns_dgfs(self, indices, **kwargs): # Transpose from [nregs][neletypes] to [neletypes][nregs] transregs = zip(*self._regs) # Generate an kernel for each element type kerns = proxylist([]) for tr in transregs: args = list([tr[indx] for indx in indices]) kerns.append(self.backend.kernel('axnpby', *args, **kwargs)) return kerns
def _get_copy_from_reg_kerns(self, inp, out, **kwargs): # Transpose from [nregs][neletypes] to [neletypes][nregs] transregs = zip(*self._regs) fsoln = self.system.eles_scal_upts_inb_full # Generate an kernel for each element type kerns = proxylist([]) for tr, ftr in zip(transregs, fsoln): args = list([tr[inp], ftr[out]]) kerns.append(self.backend.kernel('copy_from_reg',*args,**kwargs)) return kerns
def _load_mpi_inters(self, rallocs, mesh, elemap, **kwargs): lhsprank = rallocs.prank mpi_inters = proxylist([]) for rhsprank in rallocs.prankconn[lhsprank]: rhsmrank = rallocs.pmrankmap[rhsprank] interarr = mesh['con_p%dp%d' % (lhsprank, rhsprank)] interarr = interarr.astype('U4,i4,i1,i1').tolist() mpiiface = self.mpiinterscls(self.backend, interarr, rhsmrank, rallocs, elemap, self.cfg, **kwargs) mpi_inters.append(mpiiface) return mpi_inters
def _load_eles(self, rallocs, mesh, initsoln, nreg, nonce, **kwargs): basismap = {b.name: b for b in subclasses(BaseShape, just_leaf=True)} # Look for and load each element type from the mesh elemap = OrderedDict() for f in mesh: m = re.match('spt_(.+?)_p%d$' % rallocs.prank, f) if m: # Element type t = m.group(1) elemap[t] = self.elementscls(basismap[t], mesh[f], self.cfg, **kwargs) # Construct a proxylist to simplify collective operations eles = proxylist(elemap.values()) # Set the initial conditions either from a frfss file or from # explicit expressions in the config file if initsoln and (not ('vm' in kwargs)): # Load the config and stats files from the solution solncfg = Inifile(initsoln['config']) solnsts = Inifile(initsoln['stats']) # Get the names of the conserved variables (fields) solnfields = solnsts.get('data', 'fields', '') currfields = ','.join(eles[0].convarmap[eles[0].ndims]) # Ensure they match up if solnfields and solnfields != currfields: raise RuntimeError('Invalid solution for system') # Process the solution for k, ele in elemap.items(): soln = initsoln['soln_%s_p%d' % (k, rallocs.prank)] ele.set_ics_from_soln(soln, solncfg) else: eles.set_ics_from_cfg() # Allocate these elements on the backend eles.set_backend(self.backend, nreg, nonce) return eles, elemap
def get_nregs_moms(self, nreg_moms): if self.nreg_moms != None and self.nreg_moms != nreg_moms: raise RuntimeError("Some issue") if self.nreg_moms == nreg_moms: return list(range(nreg_moms)) # Moment storage nalph = self.sm.nalph eles_scal_upts_inb_moms = proxylist(self.ele_banks) # loop over the sub-domains in the full mixed domain for t, (nupts, nvars, neles) in enumerate(self.ele_shapes): eles_scal_upts_inb_moms[t] = self.backend.matrix_bank([ self.backend.matrix((nupts * neles, nalph)) for i in range(nreg_moms) ]) self.eles_scal_upts_inb_moms = eles_scal_upts_inb_moms self.nreg_moms = nreg_moms self.nalph = nalph return list(range(nreg_moms))
def _load_bc_inters(self, rallocs, mesh, elemap, **kwargs): bccls = self.bbcinterscls bcmap = {b.type: b for b in subclasses(bccls, just_leaf=True)} bc_inters = proxylist([]) for f in mesh: m = re.match('bcon_(.+?)_p%d$' % rallocs.prank, f) if m: # Get the region name rgn = m.group(1) # Determine the config file section cfgsect = 'soln-bcs-%s' % rgn # Get the interface interarr = mesh[f].astype('U4,i4,i1,i1').tolist() # Instantiate bcclass = bcmap[self.cfg.get(cfgsect, 'type')] bciface = bcclass(self.backend, interarr, elemap, cfgsect, self.cfg, **kwargs) bc_inters.append(bciface) return bc_inters
def __init__(self, backend, rallocs, mesh, initsoln, nreg, cfg): if(not backend.name=='cuda'): raise ValueError("CUDA backend supported!") # load the velocity mesh self.vm = self.velocitymeshcls(backend, cfg, self._nspcs) cv = self.vm.cv() vsize = self.vm.vsize() # need to define the expressions # the prefix "f_" should be same as in elementcls distvar # size of distvar should be equal to NvBatchSize for ivar in range(self.vm.NvBatchSize()): cfg.set('soln-ics', 'f_' + str(ivar), '0.') # now, we can initialize things super().__init__(backend, rallocs, mesh, initsoln, nreg, cfg, vm=self.vm) print('Finished initializing the BaseSystem') # define the time-step minjac = 100.0 for t, ele in self.ele_map.items(): djac = ele.djac_at_np('upts') minjac = np.min([minjac, np.min(djac)]) advmax = self.vm.L() unitCFLdt = np.array([np.sqrt(minjac)/advmax/self.ndims]) gunitCFLdt = np.zeros(1) # MPI info comm, rank, root = get_comm_rank_root() # Reduce and, if we are the root rank, output if rank != root: comm.Reduce(unitCFLdt, gunitCFLdt, op=get_mpi('min'), root=root) else: comm.Reduce(unitCFLdt, gunitCFLdt, op=get_mpi('min'), root=root) print("Time-step for unit CFL:", gunitCFLdt) print("The actual time-step will depend on DG order CFL") # load the scattering model smn = cfg.get('scattering-model', 'type') scatteringcls = subclass_where(DGFSBiScatteringModel, scattering_model=smn) self.sm = scatteringcls(backend, self.cfg, self.vm) # Allocate and bank the storage required by the time integrator #eles_scal_upts_full = proxylist(self.ele_banks) eles_scal_upts_inb_full = proxylist(self.ele_banks) spcs_eles_scal_upts_full = [list(self.ele_banks) for spcs in range(self._nspcs)] if initsoln: #raise ValueError("Not implemented") # Load the config and stats files from the solution solncfg = Inifile(initsoln['config']) solnsts = Inifile(initsoln['stats']) # Get the names of the conserved variables (fields) solnfields = solnsts.get('data', 'fields', '') # see dgfsdistwriterbi.py plugin currfields = [] fields = ['f_'+str(i) for i in range(vsize)] lf = len(fields) for p in range(self._nspcs): currfields.extend(fields) for ivar in range(-1,-lf-1,-1): currfields[ivar] += ':'+str(p+1) currfields = ','.join(currfields) # Ensure they match up if solnfields and solnfields != currfields: raise RuntimeError('Invalid solution for system') # Ensure the solnfields are not empty if not solnfields: raise RuntimeError('Invalid solution for system') nreg0 = nreg//self._nspcs assert nreg==nreg0*self._nspcs, "Should be multiple of nspcs" # Process the solution for t, (k, ele) in enumerate(self.ele_map.items()): soln = initsoln['soln_%s_p%d' % (k, rallocs.prank)] #ele.set_ics_from_soln(soln, solncfg) # Recreate the existing solution basis solnb = ele.basis.__class__(None, solncfg) # Form the interpolation operator interp = solnb.ubasis.nodal_basis_at(ele.basis.upts) # Apply and reshape data = np.dot(interp, soln.reshape(solnb.nupts, -1)) data = data.reshape(ele.nupts, self._nspcs*vsize, ele.neles) for p in range(self._nspcs): spcs_eles_scal_upts_full[p][t] = data[:, p*vsize:(p+1)*vsize, :] else: # load the initial condition model icn = cfg.get('soln-ics', 'type') initcondcls = subclass_where(DGFSBiInitCondition, model=icn) ic = initcondcls(backend, cfg, self.vm, 'soln-ics') #initvals = ic.get_init_vals() nreg0 = nreg//self._nspcs assert nreg==nreg0*self._nspcs, "Should be multiple of nspcs" # loop over the sub-domains in the full mixed domain for p in range(self._nspcs): for t, ele in enumerate(self.ele_map.values()): spcs_eles_scal_upts_full[p][t] = np.empty( (ele.nupts, vsize, ele.neles)) ic.apply_init_vals(p, spcs_eles_scal_upts_full[p][t], ele) # Convert from primitive to conservative form if needed nreg0 = nreg//self._nspcs assert nreg==nreg0*self._nspcs, "Should be multiple of nspcs" for t in range(len(eles_scal_upts_inb_full)): scal_upts_full = [] for p in range(self._nspcs): if p==0: scal_upts_full = [ backend.matrix(spcs_eles_scal_upts_full[p][t].shape, spcs_eles_scal_upts_full[p][t], tags={'align'}) for i in range(nreg0)] else: scal_upts_full.extend([ backend.matrix(spcs_eles_scal_upts_full[p][t].shape, spcs_eles_scal_upts_full[p][t], tags={'align'}) for i in range(nreg0)]) eles_scal_upts_inb_full[t] = backend.matrix_bank(scal_upts_full) #eles_scal_upts_outb_full[t] = backend.matrix_bank(scal_upts_full) self.eles_scal_upts_inb_full = eles_scal_upts_inb_full del spcs_eles_scal_upts_full
def __init__(self, kernels): self._kernels = proxylist(kernels)
def __init__(self, backend, systemcls, rallocs, mesh, initsoln, cfg): self.backend = backend self.rallocs = rallocs self.isrestart = initsoln is not None self.cfg = cfg self.prevcfgs = {f: initsoln[f] for f in initsoln or [] if f.startswith('config-')} # Ensure the system is compatible with our formulation if self.formulation not in systemcls.elementscls.formulations: raise RuntimeError( 'System {0} does not support time stepping formulation {1}' .format(systemcls.name, self.formulation) ) # Start time self.tstart = cfg.getfloat('solver-time-integrator', 'tstart', 0.0) self.tend = cfg.getfloat('solver-time-integrator', 'tend') # Current time; defaults to tstart unless restarting if self.isrestart: stats = Inifile(initsoln['stats']) self.tcurr = stats.getfloat('solver-time-integrator', 'tcurr') else: self.tcurr = self.tstart # List of target times to advance to self.tlist = deque([self.tend]) # Accepted and rejected step counters self.nacptsteps = 0 self.nrjctsteps = 0 self.nacptchain = 0 # Current and minimum time steps self._dt = cfg.getfloat('solver-time-integrator', 'dt') self.dtmin = cfg.getfloat('solver-time-integrator', 'dt-min', 1e-12) # Determine the amount of temp storage required by this method self.nreg = self._stepper_nregs # Construct the relevant mesh partition self._init_system(systemcls, backend, rallocs, mesh, initsoln) # Storage for register banks and current index self._init_reg_banks() # Extract the UUID of the mesh (to be saved with solutions) self.mesh_uuid = mesh['mesh_uuid'] # Get a queue for subclasses to use self._queue = backend.queue() # Global degree of freedom count self._gndofs = self._get_gndofs() # Solution cache self._curr_soln = None # Add kernel cache self._axnpby_kerns = {} # Record the starting wall clock time self._wstart = time.time() # Event handlers for advance_to self.completed_step_handlers = proxylist(self._get_plugins()) # Delete the memory-intensive elements map from the system del self.system.ele_map