def fill(self, content, fields, selector): # Here we get a copy of the file, which we skip through and read the # bits we want. oct_handler = self.oct_handler all_fields = self.domain.ds.index.fluid_field_list fields = [f for ft, f in fields] tr = {} cell_count = selector.count_oct_cells(self.oct_handler, self.domain_id) levels, cell_inds, file_inds = self.oct_handler.file_index_octs( selector, self.domain_id, cell_count) for field in fields: tr[field] = np.zeros(cell_count, 'float64') for level, offset in enumerate(self.domain.hydro_offset): if offset == -1: continue content.seek(offset) nc = self.domain.level_count[level] temp = {} for field in all_fields: temp[field] = np.empty((nc, 8), dtype="float64") for i in range(8): for field in all_fields: if field not in fields: fpu.skip(content) else: temp[field][:, i] = fpu.read_vector(content, 'd') # cell 1 oct_handler.fill_level(level, levels, cell_inds, file_inds, tr, temp) return tr
def _count_art_octs(self, f, offset, MinLev, MaxLevelNow): level_oct_offsets = [0, ] level_child_offsets = [0, ] f.seek(offset) nchild, ntot = 8, 0 Level = np.zeros(MaxLevelNow+1 - MinLev, dtype='int64') iNOLL = np.zeros(MaxLevelNow+1 - MinLev, dtype='int64') iHOLL = np.zeros(MaxLevelNow+1 - MinLev, dtype='int64') for Lev in range(MinLev + 1, MaxLevelNow+1): level_oct_offsets.append(f.tell()) # Get the info for this level, skip the rest # print "Reading oct tree data for level", Lev # print 'offset:',f.tell() Level[Lev], iNOLL[Lev], iHOLL[Lev] = fpu.read_vector(f, 'i', '>') # print 'Level %i : '%Lev, iNOLL # print 'offset after level record:',f.tell() nLevel = iNOLL[Lev] ntot = ntot + nLevel # Skip all the oct hierarchy data ns = fpu.peek_record_size(f, endian='>') size = struct.calcsize('>i') + ns + struct.calcsize('>i') f.seek(f.tell()+size * nLevel) level_child_offsets.append(f.tell()) # Skip the child vars data ns = fpu.peek_record_size(f, endian='>') size = struct.calcsize('>i') + ns + struct.calcsize('>i') f.seek(f.tell()+size * nLevel*nchild) # find nhydrovars nhydrovars = 8+2 f.seek(offset) return nhydrovars, iNOLL, level_oct_offsets, level_child_offsets
def fill(self, content, fields, selector, file_handler): # Here we get a copy of the file, which we skip through and read the # bits we want. oct_handler = self.oct_handler all_fields = [f for ft, f in file_handler.field_list] fields = [f for ft, f in fields] tr = {} cell_count = selector.count_oct_cells(self.oct_handler, self.domain_id) levels, cell_inds, file_inds = self.oct_handler.file_index_octs( selector, self.domain_id, cell_count) # Initializing data container for field in fields: tr[field] = np.zeros(cell_count, 'float64') # Loop over levels for level, offset in enumerate(file_handler.offset): if offset == -1: continue content.seek(offset) nc = file_handler.level_count[level] tmp = {} # Initalize temporary data container for io for field in all_fields: tmp[field] = np.empty((nc, 8), dtype="float64") for i in range(8): # Read the selected fields for field in all_fields: if field not in fields: fpu.skip(content) else: tmp[field][:, i] = fpu.read_vector(content, 'd') # i-th cell oct_handler.fill_level(level, levels, cell_inds, file_inds, tr, tmp) return tr
def fill(self, content, fields, selector): # Here we get a copy of the file, which we skip through and read the # bits we want. oct_handler = self.oct_handler all_fields = self.domain.ds.index.fluid_field_list fields = [f for ft, f in fields] tr = {} cell_count = selector.count_oct_cells(self.oct_handler, self.domain_id) levels, cell_inds, file_inds = self.oct_handler.file_index_octs( selector, self.domain_id, cell_count) for field in fields: tr[field] = np.zeros(cell_count, 'float64') for level, offset in enumerate(self.domain.hydro_offset): if offset == -1: continue content.seek(offset) nc = self.domain.level_count[level] temp = {} for field in all_fields: temp[field] = np.empty((nc, 8), dtype="float64") for i in range(8): for field in all_fields: if field not in fields: fpu.skip(content) else: temp[field][:,i] = fpu.read_vector(content, 'd') # cell 1 oct_handler.fill_level(level, levels, cell_inds, file_inds, tr, temp) return tr
def _read_particle_subset(self, subset, fields): f = open(subset.domain.part_fn, "rb") foffsets = subset.domain.particle_field_offsets tr = {} # We do *all* conversion into boxlen here. # This means that no other conversions need to be applied to convert # positions into the same domain as the octs themselves. for field in sorted(fields, key=lambda a: foffsets[a]): f.seek(foffsets[field]) dt = subset.domain.particle_field_types[field] tr[field] = fpu.read_vector(f, dt) if field[1].startswith("particle_position"): np.divide(tr[field], subset.domain.ds["boxlen"], tr[field]) cosmo = subset.domain.ds.cosmological_simulation if cosmo == 1 and field[1] == "particle_age": tf = subset.domain.ds.t_frw dtau = subset.domain.ds.dtau tauf = subset.domain.ds.tau_frw tsim = subset.domain.ds.time_simu h100 = subset.domain.ds.hubble_constant nOver2 = subset.domain.ds.n_frw / 2 t_scale = 1. / (h100 * 100 * cm_per_km / cm_per_mpc) / subset.domain.ds['unit_t'] ages = tr[field] tr[field] = get_ramses_ages(tf, tauf, dtau, tsim, t_scale, ages, nOver2, len(ages)) return tr
def _read_particle_header(self): if not os.path.exists(self.part_fn): self.local_particle_count = 0 self.particle_field_offsets = {} return f = open(self.part_fn, "rb") f.seek(0, os.SEEK_END) flen = f.tell() f.seek(0) hvals = {} attrs = ( ('ncpu', 1, 'I'), ('ndim', 1, 'I'), ('npart', 1, 'I') ) hvals.update(fpu.read_attrs(f, attrs)) fpu.read_vector(f, 'I') attrs = ( ('nstar_tot', 1, 'I'), ('mstar_tot', 1, 'd'), ('mstar_lost', 1, 'd'), ('nsink', 1, 'I') ) hvals.update(fpu.read_attrs(f, attrs)) self.particle_header = hvals self.local_particle_count = hvals['npart'] particle_fields = [ ("particle_position_x", "d"), ("particle_position_y", "d"), ("particle_position_z", "d"), ("particle_velocity_x", "d"), ("particle_velocity_y", "d"), ("particle_velocity_z", "d"), ("particle_mass", "d"), ("particle_identifier", "I"), ("particle_refinement_level", "I")] if hvals["nstar_tot"] > 0: particle_fields += [("particle_age", "d"), ("particle_metallicity", "d")] field_offsets = {} _pfields = {} for field, vtype in particle_fields: if f.tell() >= flen: break field_offsets["io", field] = f.tell() _pfields["io", field] = vtype fpu.skip(f, 1) self.particle_field_offsets = field_offsets self.particle_field_types = _pfields self.particle_types = self.particle_types_raw = ("io",)
def _read_particle_header(self): if not os.path.exists(self.part_fn): self.local_particle_count = 0 self.particle_field_offsets = {} return f = open(self.part_fn, "rb") f.seek(0, os.SEEK_END) flen = f.tell() f.seek(0) hvals = {} attrs = ( ('ncpu', 1, 'I'), ('ndim', 1, 'I'), ('npart', 1, 'I') ) hvals.update(fpu.read_attrs(f, attrs)) fpu.read_vector(f, 'I') attrs = ( ('nstar_tot', 1, 'I'), ('mstar_tot', 1, 'd'), ('mstar_lost', 1, 'd'), ('nsink', 1, 'I') ) hvals.update(fpu.read_attrs(f, attrs)) self.particle_header = hvals self.local_particle_count = hvals['npart'] particle_fields = [ ("particle_position_x", "d"), ("particle_position_y", "d"), ("particle_position_z", "d"), ("particle_velocity_x", "d"), ("particle_velocity_y", "d"), ("particle_velocity_z", "d"), ("particle_mass", "d"), ("particle_identifier", "I"), ("particle_refinement_level", "I")] if hvals["nstar_tot"] > 0: particle_fields += [("particle_age", "d"), ("particle_metallicity", "d")] field_offsets = {} _pfields = {} for field, vtype in particle_fields: if f.tell() >= flen: break field_offsets["io", field] = f.tell() _pfields["io", field] = vtype fpu.skip(f, 1) self.particle_field_offsets = field_offsets self.particle_field_types = _pfields self.particle_types = self.particle_types_raw = ("io",)
def create_cooling_fields(self): num = os.path.basename( self.ds.parameter_filename).split(".")[0].split("_")[1] filename = "%s/cooling_%05i.out" % (os.path.dirname( self.ds.parameter_filename), int(num)) if not os.path.exists(filename): return def _create_field(name, interp_object): def _func(field, data): shape = data["temperature"].shape d = { 'lognH': np.log10(_X * data["density"] / mh).ravel(), 'logT': np.log10(data["temperature"]).ravel() } rv = 10**interp_object(d).reshape(shape) # Return array in unit 'per volume' consistently with line below return data.ds.arr(rv, 'code_length**-3') self.add_field(name=name, sampling_type="cell", function=_func, units="code_length**-3") avals = {} tvals = {} with open(filename, "rb") as f: n1, n2 = fpu.read_vector(f, 'i') n = n1 * n2 for ax in _cool_axes: avals[ax] = fpu.read_vector(f, 'd') for tname in _cool_arrs: var = fpu.read_vector(f, 'd') if var.size == n1 * n2: tvals[tname] = var.reshape((n1, n2), order='F') else: var = var.reshape((n1, n2, var.size // (n1 * n2)), order='F') for i in range(var.shape[-1]): tvals[_cool_species[i]] = var[:, :, i] for n in tvals: interp = BilinearFieldInterpolator(tvals[n], (avals["lognH"], avals["logT"]), ["lognH", "logT"], truncate=True) _create_field(("gas", n), interp)
def _read_hydro_header(self): # If no hydro file is found, return if not self._is_hydro(): return if self.nvar > 0: return self.nvar # Read the number of hydro variables f = open(self.hydro_fn, "rb") fpu.skip(f, 1) self.nvar = fpu.read_vector(f, "i")[0]
def _read_hydro_header(self): # If no hydro file is found, return if not self._is_hydro(): return if self.nvar > 0: return self.nvar # Read the number of hydro variables f = open(self.hydro_fn, "rb") fpu.skip(f, 1) self.nvar = fpu.read_vector(f, "i")[0]
def read_star_field(file, field=None): data = {} with open(file, "rb") as fh: for dtype, variables in star_struct: found = (isinstance(variables, tuple) and field in variables) or field == variables if found: data[field] = read_vector(fh, dtype[1], dtype[0]) else: skip(fh, endian=">") return data.pop(field)
def _read_particle_subset(self, subset, fields): f = open(subset.domain.part_fn, "rb") foffsets = subset.domain.particle_field_offsets tr = {} # We do *all* conversion into boxlen here. # This means that no other conversions need to be applied to convert # positions into the same domain as the octs themselves. for field in sorted(fields, key = lambda a: foffsets[a]): f.seek(foffsets[field]) dt = subset.domain.particle_field_types[field] tr[field] = fpu.read_vector(f, dt) if field[1].startswith("particle_position"): np.divide(tr[field], subset.domain.ds["boxlen"], tr[field]) return tr
def _read_amr_header(self): hvals = {} f = open(self.amr_fn, "rb") for header in ramses_header(hvals): hvals.update(fpu.read_attrs(f, header)) # That's the header, now we skip a few. hvals['numbl'] = np.array(hvals['numbl']).reshape( (hvals['nlevelmax'], hvals['ncpu'])) fpu.skip(f) if hvals['nboundary'] > 0: fpu.skip(f, 2) self.ngridbound = fpu.read_vector(f, 'i').astype("int64") else: self.ngridbound = np.zeros(hvals['nlevelmax'], dtype='int64') free_mem = fpu.read_attrs(f, (('free_mem', 5, 'i'), ) ) ordering = fpu.read_vector(f, 'c') fpu.skip(f, 4) # Now we're at the tree itself # Now we iterate over each level and each CPU. self.amr_header = hvals self.amr_offset = f.tell() self.local_oct_count = hvals['numbl'][self.ds.min_level:, self.domain_id - 1].sum() self.total_oct_count = hvals['numbl'][self.ds.min_level:,:].sum(axis=0)
def _read_particle_subset(self, subset, fields): f = open(subset.domain.part_fn, "rb") foffsets = subset.domain.particle_field_offsets tr = {} # We do *all* conversion into boxlen here. # This means that no other conversions need to be applied to convert # positions into the same domain as the octs themselves. for field in sorted(fields, key=lambda a: foffsets[a]): f.seek(foffsets[field]) dt = subset.domain.particle_field_types[field] tr[field] = fpu.read_vector(f, dt) if field[1].startswith("particle_position"): np.divide(tr[field], subset.domain.ds["boxlen"], tr[field]) return tr
def _read_amr_header(self): hvals = {} f = open(self.amr_fn, "rb") for header in ramses_header(hvals): hvals.update(fpu.read_attrs(f, header)) # That's the header, now we skip a few. hvals['numbl'] = np.array(hvals['numbl']).reshape( (hvals['nlevelmax'], hvals['ncpu'])) fpu.skip(f) if hvals['nboundary'] > 0: fpu.skip(f, 2) self.ngridbound = fpu.read_vector(f, 'i').astype("int64") else: self.ngridbound = np.zeros(hvals['nlevelmax'], dtype='int64') free_mem = fpu.read_attrs(f, (('free_mem', 5, 'i'), ) ) # NOQA ordering = fpu.read_vector(f, 'c') # NOQA fpu.skip(f, 4) # Now we're at the tree itself # Now we iterate over each level and each CPU. self.amr_header = hvals self.amr_offset = f.tell() self.local_oct_count = hvals['numbl'][self.ds.min_level:, self.domain_id - 1].sum() self.total_oct_count = hvals['numbl'][self.ds.min_level:,:].sum(axis=0)
def create_cooling_fields(self): num = os.path.basename(self.ds.parameter_filename).split("." )[0].split("_")[1] filename = "%s/cooling_%05i.out" % ( os.path.dirname(self.ds.parameter_filename), int(num)) if not os.path.exists(filename): return def _create_field(name, interp_object): def _func(field, data): shape = data["temperature"].shape d = {'lognH': np.log10(_X*data["density"]/mh).ravel(), 'logT' : np.log10(data["temperature"]).ravel()} rv = 10**interp_object(d).reshape(shape) return rv self.add_field(name = name, function=_func, units = "code_length**-3") avals = {} tvals = {} with open(filename, "rb") as f: n1, n2 = fpu.read_vector(f, 'i') n = n1 * n2 for ax in _cool_axes: avals[ax] = fpu.read_vector(f, 'd') for tname in _cool_arrs: var = fpu.read_vector(f, 'd') if var.size == n1*n2: tvals[tname] = var.reshape((n1, n2), order='F') else: var = var.reshape((n1, n2, var.size / (n1*n2)), order='F') for i in range(var.shape[-1]): tvals[_cool_species[i]] = var[:,:,i] for n in tvals: interp = BilinearFieldInterpolator(tvals[n], (avals["lognH"], avals["logT"]), ["lognH", "logT"], truncate = True) _create_field(("gas", n), interp)
def _ramses_particle_file_handler(fname, foffsets, data_types, subset, fields, count): '''General file handler, called by _read_particle_subset Parameters ---------- fname : string filename to read from foffsets: dict Offsets in file of the fields data_types: dict Data type of the fields subset: ``RAMSESDomainSubset`` A RAMSES domain subset object fields: list of tuple The fields to read count: integer The number of elements to count ''' tr = {} with open(fname, "rb") as f: # We do *all* conversion into boxlen here. # This means that no other conversions need to be applied to convert # positions into the same domain as the octs themselves. for field in sorted(fields, key=lambda a: foffsets[a]): if count == 0: tr[field] = np.empty(0, dtype=data_types[field]) continue f.seek(foffsets[field]) dt = data_types[field] tr[field] = fpu.read_vector(f, dt) if field[1].startswith("particle_position"): np.divide(tr[field], subset.domain.ds["boxlen"], tr[field]) cosmo = subset.domain.ds.cosmological_simulation if cosmo == 1 and field[1] == "particle_age": tf = subset.domain.ds.t_frw dtau = subset.domain.ds.dtau tauf = subset.domain.ds.tau_frw tsim = subset.domain.ds.time_simu h100 = subset.domain.ds.hubble_constant nOver2 = subset.domain.ds.n_frw / 2 t_scale = 1. / (h100 * 100 * cm_per_km / cm_per_mpc) / subset.domain.ds['unit_t'] ages = tr[field] tr[field] = get_ramses_ages(tf, tauf, dtau, tsim, t_scale, ages, nOver2, len(ages)) return tr
def _count_art_octs(self, f, offset, MinLev, MaxLevelNow): level_oct_offsets = [0, ] level_child_offsets = [0, ] f.seek(offset) nchild, ntot = 8, 0 Level = np.zeros(MaxLevelNow+1 - MinLev, dtype='int64') iNOLL = np.zeros(MaxLevelNow+1 - MinLev, dtype='int64') iHOLL = np.zeros(MaxLevelNow+1 - MinLev, dtype='int64') for Lev in range(MinLev + 1, MaxLevelNow+1): level_oct_offsets.append(f.tell()) # Get the info for this level, skip the rest # print "Reading oct tree data for level", Lev # print 'offset:',f.tell() Level[Lev], iNOLL[Lev], iHOLL[Lev] = fpu.read_vector(f, 'i', '>') # print 'Level %i : '%Lev, iNOLL # print 'offset after level record:',f.tell() iOct = iHOLL[Lev] - 1 nLevel = iNOLL[Lev] nLevCells = nLevel * nchild ntot = ntot + nLevel # Skip all the oct hierarchy data ns = fpu.peek_record_size(f, endian='>') size = struct.calcsize('>i') + ns + struct.calcsize('>i') f.seek(f.tell()+size * nLevel) level_child_offsets.append(f.tell()) # Skip the child vars data ns = fpu.peek_record_size(f, endian='>') size = struct.calcsize('>i') + ns + struct.calcsize('>i') f.seek(f.tell()+size * nLevel*nchild) # find nhydrovars nhydrovars = 8+2 f.seek(offset) return nhydrovars, iNOLL, level_oct_offsets, level_child_offsets
def _parse_parameter_file(self): """ Get the various simulation parameters & constants. """ self.dimensionality = 3 self.refine_by = 2 self.periodicity = (True, True, True) self.cosmological_simulation = True self.parameters = {} self.unique_identifier = \ int(os.stat(self.parameter_filename)[stat.ST_CTIME]) self.parameters.update(constants) self.parameters['Time'] = 1.0 # read the amr header with open(self._file_amr, 'rb') as f: amr_header_vals = fpu.read_attrs(f, amr_header_struct, '>') for to_skip in ['tl', 'dtl', 'tlold', 'dtlold', 'iSO']: skipped = fpu.skip(f, endian='>') (self.ncell) = fpu.read_vector(f, 'i', '>')[0] # Try to figure out the root grid dimensions est = int(np.rint(self.ncell**(1.0 / 3.0))) # Note here: this is the number of *cells* on the root grid. # This is not the same as the number of Octs. # domain dimensions is the number of root *cells* self.domain_dimensions = np.ones(3, dtype='int64') * est self.root_grid_mask_offset = f.tell() self.root_nocts = self.domain_dimensions.prod() / 8 self.root_ncells = self.root_nocts * 8 mylog.debug( "Estimating %i cells on a root grid side," + "%i root octs", est, self.root_nocts) self.root_iOctCh = fpu.read_vector(f, 'i', '>')[:self.root_ncells] self.root_iOctCh = self.root_iOctCh.reshape(self.domain_dimensions, order='F') self.root_grid_offset = f.tell() self.root_nhvar = fpu.skip(f, endian='>') self.root_nvar = fpu.skip(f, endian='>') # make sure that the number of root variables is a multiple of # rootcells assert self.root_nhvar % self.root_ncells == 0 assert self.root_nvar % self.root_ncells == 0 self.nhydro_variables = ((self.root_nhvar + self.root_nvar) / self.root_ncells) self.iOctFree, self.nOct = fpu.read_vector(f, 'i', '>') self.child_grid_offset = f.tell() self.parameters.update(amr_header_vals) amr_header_vals = None # estimate the root level float_center, fl, iocts, nocts, root_level = _read_art_level_info( f, [0, self.child_grid_offset], 1, coarse_grid=self.domain_dimensions[0]) del float_center, fl, iocts, nocts self.root_level = root_level mylog.info("Using root level of %02i", self.root_level) # read the particle header self.particle_types = [] self.particle_types_raw = () if not self.skip_particles and self._file_particle_header: with open(self._file_particle_header, "rb") as fh: particle_header_vals = fpu.read_attrs(fh, particle_header_struct, '>') fh.seek(seek_extras) n = particle_header_vals['Nspecies'] wspecies = np.fromfile(fh, dtype='>f', count=10) lspecies = np.fromfile(fh, dtype='>i', count=10) self.parameters['wspecies'] = wspecies[:n] self.parameters['lspecies'] = lspecies[:n] for specie in range(n): self.particle_types.append("specie%i" % specie) self.particle_types_raw = tuple(self.particle_types) ls_nonzero = np.diff(lspecies)[:n - 1] ls_nonzero = np.append(lspecies[0], ls_nonzero) self.star_type = len(ls_nonzero) mylog.info("Discovered %i species of particles", len(ls_nonzero)) mylog.info("Particle populations: " + '%9i ' * len(ls_nonzero), *ls_nonzero) for k, v in particle_header_vals.items(): if k in self.parameters.keys(): if not self.parameters[k] == v: mylog.info("Inconsistent parameter %s %1.1e %1.1e", k, v, self.parameters[k]) else: self.parameters[k] = v self.parameters_particles = particle_header_vals self.parameters.update(particle_header_vals) self.parameters['ng'] = self.parameters['Ngridc'] self.parameters['ncell0'] = self.parameters['ng']**3 # setup standard simulation params yt expects to see self.current_redshift = self.parameters["aexpn"]**-1.0 - 1.0 self.omega_lambda = self.parameters['Oml0'] self.omega_matter = self.parameters['Om0'] self.hubble_constant = self.parameters['hubble'] self.min_level = self.parameters['min_level'] self.max_level = self.parameters['max_level'] if self.limit_level is not None: self.max_level = min(self.limit_level, self.parameters['max_level']) if self.force_max_level is not None: self.max_level = self.force_max_level self.hubble_time = 1.0 / (self.hubble_constant * 100 / 3.08568025e19) self.current_time = self.quan(b2t(self.parameters['t']), 'Gyr') self.gamma = self.parameters["gamma"] mylog.info("Max level is %02i", self.max_level)
def _parse_parameter_file(self): """ Get the various simulation parameters & constants. """ self.dimensionality = 3 self.refine_by = 2 self.periodicity = (True, True, True) self.cosmological_simulation = True self.parameters = {} self.unique_identifier = \ int(os.stat(self.parameter_filename)[stat.ST_CTIME]) self.parameters.update(constants) self.parameters['Time'] = 1.0 # read the amr header with open(self._file_amr, 'rb') as f: amr_header_vals = fpu.read_attrs(f, amr_header_struct, '>') for to_skip in ['tl', 'dtl', 'tlold', 'dtlold', 'iSO']: skipped = fpu.skip(f, endian='>') (self.ncell) = fpu.read_vector(f, 'i', '>')[0] # Try to figure out the root grid dimensions est = int(np.rint(self.ncell**(1.0/3.0))) # Note here: this is the number of *cells* on the root grid. # This is not the same as the number of Octs. # domain dimensions is the number of root *cells* self.domain_dimensions = np.ones(3, dtype='int64')*est self.root_grid_mask_offset = f.tell() self.root_nocts = self.domain_dimensions.prod()/8 self.root_ncells = self.root_nocts*8 mylog.debug("Estimating %i cells on a root grid side," + "%i root octs", est, self.root_nocts) self.root_iOctCh = fpu.read_vector(f, 'i', '>')[:self.root_ncells] self.root_iOctCh = self.root_iOctCh.reshape(self.domain_dimensions, order='F') self.root_grid_offset = f.tell() self.root_nhvar = fpu.skip(f, endian='>') self.root_nvar = fpu.skip(f, endian='>') # make sure that the number of root variables is a multiple of # rootcells assert self.root_nhvar % self.root_ncells == 0 assert self.root_nvar % self.root_ncells == 0 self.nhydro_variables = ((self.root_nhvar+self.root_nvar) / self.root_ncells) self.iOctFree, self.nOct = fpu.read_vector(f, 'i', '>') self.child_grid_offset = f.tell() self.parameters.update(amr_header_vals) amr_header_vals = None # estimate the root level float_center, fl, iocts, nocts, root_level = _read_art_level_info( f, [0, self.child_grid_offset], 1, coarse_grid=self.domain_dimensions[0]) del float_center, fl, iocts, nocts self.root_level = root_level mylog.info("Using root level of %02i", self.root_level) # read the particle header self.particle_types = [] self.particle_types_raw = () if not self.skip_particles and self._file_particle_header: with open(self._file_particle_header, "rb") as fh: particle_header_vals = fpu.read_attrs( fh, particle_header_struct, '>') fh.seek(seek_extras) n = particle_header_vals['Nspecies'] wspecies = np.fromfile(fh, dtype='>f', count=10) lspecies = np.fromfile(fh, dtype='>i', count=10) self.parameters['wspecies'] = wspecies[:n] self.parameters['lspecies'] = lspecies[:n] for specie in range(n): self.particle_types.append("specie%i" % specie) self.particle_types_raw = tuple( self.particle_types) ls_nonzero = np.diff(lspecies)[:n-1] ls_nonzero = np.append(lspecies[0], ls_nonzero) self.star_type = len(ls_nonzero) mylog.info("Discovered %i species of particles", len(ls_nonzero)) mylog.info("Particle populations: "+'%9i '*len(ls_nonzero), *ls_nonzero) for k, v in particle_header_vals.items(): if k in self.parameters.keys(): if not self.parameters[k] == v: mylog.info( "Inconsistent parameter %s %1.1e %1.1e", k, v, self.parameters[k]) else: self.parameters[k] = v self.parameters_particles = particle_header_vals self.parameters.update(particle_header_vals) self.parameters['ng'] = self.parameters['Ngridc'] self.parameters['ncell0'] = self.parameters['ng']**3 # setup standard simulation params yt expects to see self.current_redshift = self.parameters["aexpn"]**-1.0 - 1.0 self.omega_lambda = self.parameters['Oml0'] self.omega_matter = self.parameters['Om0'] self.hubble_constant = self.parameters['hubble'] self.min_level = self.parameters['min_level'] self.max_level = self.parameters['max_level'] if self.limit_level is not None: self.max_level = min( self.limit_level, self.parameters['max_level']) if self.force_max_level is not None: self.max_level = self.force_max_level self.hubble_time = 1.0/(self.hubble_constant*100/3.08568025e19) self.current_time = self.quan(b2t(self.parameters['t']), 'Gyr') self.gamma = self.parameters["gamma"] mylog.info("Max level is %02i", self.max_level)
def _read_art_level_info(f, level_oct_offsets, level, coarse_grid=128, ncell0=None, root_level=None): pos = f.tell() f.seek(level_oct_offsets[level]) # Get the info for this level, skip the rest junk, nLevel, iOct = read_vector(f, "i", ">") # fortran indices start at 1 # Skip all the oct index data le = np.zeros((nLevel, 3), dtype="int64") fl = np.ones((nLevel, 6), dtype="int64") iocts = np.zeros(nLevel + 1, dtype="int64") idxa, idxb = 0, 0 chunk = int(1e6) # this is ~111MB for 15 dimensional 64 bit arrays left = nLevel while left > 0: this_chunk = min(chunk, left) idxb = idxa + this_chunk data = np.fromfile(f, dtype=">i", count=this_chunk * 15) data = data.reshape(this_chunk, 15) left -= this_chunk le[idxa:idxb, :] = data[:, 1:4] fl[idxa:idxb, 1] = np.arange(idxa, idxb) # pad byte is last, LL2, then ioct right before it iocts[idxa:idxb] = data[:, -3] idxa = idxa + this_chunk del data # emulate fortran code # do ic1 = 1 , nLevel # read(19) (iOctPs(i,iOct),i=1,3),(iOctNb(i,iOct),i=1,6), # & iOctPr(iOct), iOctLv(iOct), iOctLL1(iOct), # & iOctLL2(iOct) # iOct = iOctLL1(iOct) # ioct always represents the index of the next variable # not the current, so shift forward one index # the last index isn't used iocts[1:] = iocts[:-1] # shift iocts = iocts[:nLevel] # chop off the last, unused, index iocts[0] = iOct # starting value # now correct iocts for fortran indices start @ 1 iocts = iocts - 1 assert np.unique(iocts).shape[0] == nLevel # left edges are expressed as if they were on # level 15, so no matter what level max(le)=2**15 # correct to the yt convention # le = le/2**(root_level-1-level)-1 # try to find the root_level first def cfc(root_level, level, le): d_x = 1.0 / (2.0**(root_level - level + 1)) fc = (d_x * le) - 2**(level - 1) return fc if root_level is None: root_level = np.floor(np.log2(le.max() * 1.0 / coarse_grid)) root_level = root_level.astype("int64") for _ in range(10): fc = cfc(root_level, level, le) go = np.diff(np.unique(fc)).min() < 1.1 if go: break root_level += 1 else: fc = cfc(root_level, level, le) unitary_center = fc / (coarse_grid * 2.0**(level - 1)) assert np.all(unitary_center < 1.0) # again emulate the fortran code # This is all for calculating child oct locations # iC_ = iC + nbshift # iO = ishft ( iC_ , - ndim ) # id = ishft ( 1, MaxLevel - iOctLv(iO) ) # j = iC_ + 1 - ishft( iO , ndim ) # Posx = d_x * (iOctPs(1,iO) + sign ( id , idelta(j,1) )) # Posy = d_x * (iOctPs(2,iO) + sign ( id , idelta(j,2) )) # Posz = d_x * (iOctPs(3,iO) + sign ( id , idelta(j,3) )) # idelta = [[-1, 1, -1, 1, -1, 1, -1, 1], # [-1, -1, 1, 1, -1, -1, 1, 1], # [-1, -1, -1, -1, 1, 1, 1, 1]] # idelta = np.array(idelta) # if ncell0 is None: # ncell0 = coarse_grid**3 # nchild = 8 # ndim = 3 # nshift = nchild -1 # nbshift = nshift - ncell0 # iC = iocts #+ nbshift # iO = iC >> ndim #possibly >> # id = 1 << (root_level - level) # j = iC + 1 - ( iO << 3) # delta = np.abs(id)*idelta[:,j-1] # try without the -1 # le = le/2**(root_level+1-level) # now read the hvars and vars arrays # we are looking for iOctCh # we record if iOctCh is >0, in which it is subdivided # iOctCh = np.zeros((nLevel+1,8),dtype='bool') f.seek(pos) return unitary_center, fl, iocts, nLevel, root_level
def _parse_parameter_file(self): """ Get the various simulation parameters & constants. """ self.domain_left_edge = np.zeros(3, dtype="float") self.domain_right_edge = np.zeros(3, dtype="float") + 1.0 self.dimensionality = 3 self.refine_by = 2 self.periodicity = (True, True, True) self.cosmological_simulation = True self.parameters = {} self.parameters.update(constants) self.parameters["Time"] = 1.0 # read the amr header with open(self._file_amr, "rb") as f: amr_header_vals = fpu.read_attrs(f, amr_header_struct, ">") n_to_skip = len(("tl", "dtl", "tlold", "dtlold", "iSO")) fpu.skip(f, n_to_skip, endian=">") (self.ncell) = fpu.read_vector(f, "i", ">")[0] # Try to figure out the root grid dimensions est = int(np.rint(self.ncell**(1.0 / 3.0))) # Note here: this is the number of *cells* on the root grid. # This is not the same as the number of Octs. # domain dimensions is the number of root *cells* self.domain_dimensions = np.ones(3, dtype="int64") * est self.root_grid_mask_offset = f.tell() self.root_nocts = self.domain_dimensions.prod() // 8 self.root_ncells = self.root_nocts * 8 mylog.debug( "Estimating %i cells on a root grid side, %i root octs", est, self.root_nocts, ) self.root_iOctCh = fpu.read_vector(f, "i", ">")[:self.root_ncells] self.root_iOctCh = self.root_iOctCh.reshape(self.domain_dimensions, order="F") self.root_grid_offset = f.tell() self.root_nhvar = fpu.skip(f, endian=">") self.root_nvar = fpu.skip(f, endian=">") # make sure that the number of root variables is a multiple of # rootcells assert self.root_nhvar % self.root_ncells == 0 assert self.root_nvar % self.root_ncells == 0 self.nhydro_variables = (self.root_nhvar + self.root_nvar) / self.root_ncells self.iOctFree, self.nOct = fpu.read_vector(f, "i", ">") self.child_grid_offset = f.tell() # lextra needs to be loaded as a string, but it's actually # array values. So pop it off here, and then re-insert. lextra = amr_header_vals.pop("lextra") amr_header_vals["lextra"] = np.fromstring(lextra, ">f4") self.parameters.update(amr_header_vals) amr_header_vals = None # estimate the root level float_center, fl, iocts, nocts, root_level = _read_art_level_info( f, [0, self.child_grid_offset], 1, coarse_grid=self.domain_dimensions[0]) del float_center, fl, iocts, nocts self.root_level = root_level mylog.info("Using root level of %02i", self.root_level) # read the particle header self.particle_types = [] self.particle_types_raw = () if not self.skip_particles and self._file_particle_header: with open(self._file_particle_header, "rb") as fh: particle_header_vals = fpu.read_attrs(fh, particle_header_struct, ">") fh.seek(seek_extras) n = particle_header_vals["Nspecies"] wspecies = np.fromfile(fh, dtype=">f", count=10) lspecies = np.fromfile(fh, dtype=">i", count=10) # extras needs to be loaded as a string, but it's actually # array values. So pop it off here, and then re-insert. extras = particle_header_vals.pop("extras") particle_header_vals["extras"] = np.fromstring(extras, ">f4") self.parameters["wspecies"] = wspecies[:n] self.parameters["lspecies"] = lspecies[:n] for specie in range(n): self.particle_types.append("specie%i" % specie) self.particle_types_raw = tuple(self.particle_types) ls_nonzero = np.diff(lspecies)[:n - 1] ls_nonzero = np.append(lspecies[0], ls_nonzero) self.star_type = len(ls_nonzero) mylog.info("Discovered %i species of particles", len(ls_nonzero)) info_str = "Particle populations: " + "%9i " * len(ls_nonzero) mylog.info(info_str, *ls_nonzero) self._particle_type_counts = dict( zip(self.particle_types_raw, ls_nonzero)) for k, v in particle_header_vals.items(): if k in self.parameters.keys(): if not self.parameters[k] == v: mylog.info( "Inconsistent parameter %s %1.1e %1.1e", k, v, self.parameters[k], ) else: self.parameters[k] = v self.parameters_particles = particle_header_vals self.parameters.update(particle_header_vals) self.parameters["ng"] = self.parameters["Ngridc"] self.parameters["ncell0"] = self.parameters["ng"]**3 # setup standard simulation params yt expects to see self.current_redshift = self.parameters["aexpn"]**-1.0 - 1.0 self.omega_lambda = self.parameters["Oml0"] self.omega_matter = self.parameters["Om0"] self.hubble_constant = self.parameters["hubble"] self.min_level = self.parameters["min_level"] self.max_level = self.parameters["max_level"] if self.limit_level is not None: self.max_level = min(self.limit_level, self.parameters["max_level"]) if self.force_max_level is not None: self.max_level = self.force_max_level self.hubble_time = 1.0 / (self.hubble_constant * 100 / 3.08568025e19) self.current_time = self.quan(b2t(self.parameters["t"]), "Gyr") self.gamma = self.parameters["gamma"] mylog.info("Max level is %02i", self.max_level)
def _read_amr(self): """Open the oct file, read in octs level-by-level. For each oct, only the position, index, level and domain are needed - its position in the octree is found automatically. The most important is finding all the information to feed oct_handler.add """ self.oct_handler = RAMSESOctreeContainer(self.ds.domain_dimensions/2, self.ds.domain_left_edge, self.ds.domain_right_edge) root_nodes = self.amr_header['numbl'][self.ds.min_level,:].sum() self.oct_handler.allocate_domains(self.total_oct_count, root_nodes) fb = open(self.amr_fn, "rb") fb.seek(self.amr_offset) f = BytesIO() f.write(fb.read()) f.seek(0) mylog.debug("Reading domain AMR % 4i (%0.3e, %0.3e)", self.domain_id, self.total_oct_count.sum(), self.ngridbound.sum()) def _ng(c, l): if c < self.amr_header['ncpu']: ng = self.amr_header['numbl'][l, c] else: ng = self.ngridbound[c - self.amr_header['ncpu'] + self.amr_header['nboundary']*l] return ng min_level = self.ds.min_level # yt max level is not the same as the RAMSES one. # yt max level is the maximum number of additional refinement levels # so for a uni grid run with no refinement, it would be 0. # So we initially assume that. max_level = 0 nx, ny, nz = (((i-1.0)/2.0) for i in self.amr_header['nx']) for level in range(self.amr_header['nlevelmax']): # Easier if do this 1-indexed for cpu in range(self.amr_header['nboundary'] + self.amr_header['ncpu']): #ng is the number of octs on this level on this domain ng = _ng(cpu, level) if ng == 0: continue ind = fpu.read_vector(f, "I").astype("int64") fpu.skip(f, 2) pos = np.empty((ng, 3), dtype='float64') pos[:,0] = fpu.read_vector(f, "d") - nx pos[:,1] = fpu.read_vector(f, "d") - ny pos[:,2] = fpu.read_vector(f, "d") - nz #pos *= self.ds.domain_width #pos += self.dataset.domain_left_edge fpu.skip(f, 31) #parents = fpu.read_vector(f, "I") #fpu.skip(f, 6) #children = np.empty((ng, 8), dtype='int64') #for i in range(8): # children[:,i] = fpu.read_vector(f, "I") #cpu_map = np.empty((ng, 8), dtype="int64") #for i in range(8): # cpu_map[:,i] = fpu.read_vector(f, "I") #rmap = np.empty((ng, 8), dtype="int64") #for i in range(8): # rmap[:,i] = fpu.read_vector(f, "I") # We don't want duplicate grids. # Note that we're adding *grids*, not individual cells. if level >= min_level: assert(pos.shape[0] == ng) n = self.oct_handler.add(cpu + 1, level - min_level, pos, count_boundary = 1) self._error_check(cpu, level, pos, n, ng, (nx, ny, nz)) if n > 0: max_level = max(level - min_level, max_level) self.max_level = max_level self.oct_handler.finalize()
def _read_amr(self): """Open the oct file, read in octs level-by-level. For each oct, only the position, index, level and domain are needed - its position in the octree is found automatically. The most important is finding all the information to feed oct_handler.add """ self.oct_handler = RAMSESOctreeContainer(self.ds.domain_dimensions / 2, self.ds.domain_left_edge, self.ds.domain_right_edge) root_nodes = self.amr_header['numbl'][self.ds.min_level, :].sum() self.oct_handler.allocate_domains(self.total_oct_count, root_nodes) mylog.debug("Reading domain AMR % 4i (%0.3e, %0.3e)", self.domain_id, self.total_oct_count.sum(), self.ngridbound.sum()) f = self.amr_file f.seek(self.amr_offset) def _ng(c, l): if c < self.amr_header['ncpu']: ng = self.amr_header['numbl'][l, c] else: ng = self.ngridbound[c - self.amr_header['ncpu'] + self.amr_header['nboundary'] * l] return ng min_level = self.ds.min_level # yt max level is not the same as the RAMSES one. # yt max level is the maximum number of additional refinement levels # so for a uni grid run with no refinement, it would be 0. # So we initially assume that. max_level = 0 nx, ny, nz = (((i - 1.0) / 2.0) for i in self.amr_header['nx']) for level in range(self.amr_header['nlevelmax']): # Easier if do this 1-indexed for cpu in range(self.amr_header['nboundary'] + self.amr_header['ncpu']): #ng is the number of octs on this level on this domain ng = _ng(cpu, level) if ng == 0: continue ind = fpu.read_vector(f, "I").astype("int64") # NOQA fpu.skip(f, 2) pos = np.empty((ng, 3), dtype='float64') pos[:, 0] = fpu.read_vector(f, "d") - nx pos[:, 1] = fpu.read_vector(f, "d") - ny pos[:, 2] = fpu.read_vector(f, "d") - nz #pos *= self.ds.domain_width #pos += self.dataset.domain_left_edge fpu.skip(f, 31) #parents = fpu.read_vector(f, "I") #fpu.skip(f, 6) #children = np.empty((ng, 8), dtype='int64') #for i in range(8): # children[:,i] = fpu.read_vector(f, "I") #cpu_map = np.empty((ng, 8), dtype="int64") #for i in range(8): # cpu_map[:,i] = fpu.read_vector(f, "I") #rmap = np.empty((ng, 8), dtype="int64") #for i in range(8): # rmap[:,i] = fpu.read_vector(f, "I") # We don't want duplicate grids. # Note that we're adding *grids*, not individual cells. if level >= min_level: assert (pos.shape[0] == ng) n = self.oct_handler.add(cpu + 1, level - min_level, pos, count_boundary=1) self._error_check(cpu, level, pos, n, ng, (nx, ny, nz)) if n > 0: max_level = max(level - min_level, max_level) self.max_level = max_level self.oct_handler.finalize() # Close AMR file f.close()
def _read_particle_header(self): if not os.path.exists(self.part_fn): self.local_particle_count = 0 self.particle_field_offsets = {} return f = open(self.part_fn, "rb") f.seek(0, os.SEEK_END) flen = f.tell() f.seek(0) hvals = {} attrs = (('ncpu', 1, 'I'), ('ndim', 1, 'I'), ('npart', 1, 'I')) hvals.update(fpu.read_attrs(f, attrs)) fpu.read_vector(f, 'I') attrs = (('nstar_tot', 1, 'I'), ('mstar_tot', 1, 'd'), ('mstar_lost', 1, 'd'), ('nsink', 1, 'I')) hvals.update(fpu.read_attrs(f, attrs)) self.particle_header = hvals self.local_particle_count = hvals['npart'] # Try reading particle file descriptor if self._has_part_descriptor: particle_fields = (_read_part_file_descriptor( self._part_file_descriptor)) else: particle_fields = [("particle_position_x", "d"), ("particle_position_y", "d"), ("particle_position_z", "d"), ("particle_velocity_x", "d"), ("particle_velocity_y", "d"), ("particle_velocity_z", "d"), ("particle_mass", "d"), ("particle_identifier", "i"), ("particle_refinement_level", "I")] if self.ds._extra_particle_fields is not None: particle_fields += self.ds._extra_particle_fields ptype = 'io' field_offsets = {} _pfields = {} # Read offsets for field, vtype in particle_fields: if f.tell() >= flen: break field_offsets[ptype, field] = f.tell() _pfields[ptype, field] = vtype fpu.skip(f, 1) iextra = 0 while f.tell() < flen: iextra += 1 field, vtype = ('particle_extra_field_%i' % iextra, 'd') particle_fields.append((field, vtype)) field_offsets[ptype, field] = f.tell() _pfields[ptype, field] = vtype fpu.skip(f, 1) if iextra > 0 and not self.ds._warn_extra_fields: self.ds._warn_extra_fields = True w = ("Detected %s extra particle fields assuming kind " "`double`. Consider using the `extra_particle_fields` " "keyword argument if you have unexpected behavior.") mylog.warning(w % iextra) self.particle_field_offsets = field_offsets self.particle_field_types = _pfields # Register the particle type self._add_ptype(ptype)