def _initialize_index(self, data_file, regions): if self.index_ptype == "all": ptypes = self.ds.particle_types_raw pcount = sum(data_file.total_particles.values()) else: ptypes = [self.index_ptype] pcount = data_file.total_particles[self.index_ptype] morton = np.empty(pcount, dtype='uint64') if pcount == 0: return morton mylog.debug("Initializing index % 5i (% 7i particles)", data_file.file_id, pcount) ind = 0 with h5py.File(data_file.filename, "r") as f: if not f.keys(): return None dx = np.finfo(f["Group"]["GroupPos"].dtype).eps dx = 2.0 * self.ds.quan(dx, "code_length") for ptype in ptypes: if data_file.total_particles[ptype] == 0: continue pos = data_file._get_particle_positions(ptype, f=f) pos = self.ds.arr(pos, "code_length") if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \ np.any(pos.max(axis=0) > self.ds.domain_right_edge): raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), self.ds.domain_left_edge, self.ds.domain_right_edge) regions.add_data_file(pos, data_file.file_id) morton[ind:ind + pos.shape[0]] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], self.ds.domain_left_edge, self.ds.domain_right_edge) ind += pos.shape[0] return morton
def _initialize_index(self, data_file, regions): pcount = data_file.ds.parameters["nhalos"] + data_file.ds.parameters[ "nsubs"] morton = np.empty(pcount, dtype="uint64") mylog.debug("Initializing index % 5i (% 7i particles)", data_file.file_id, pcount) if pcount == 0: return morton ind = 0 pos = self._get_particle_positions() if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or np.any( pos.max(axis=0) > self.ds.domain_right_edge): raise YTDomainOverflow( pos.min(axis=0), pos.max(axis=0), self.ds.domain_left_edge, self.ds.domain_right_edge, ) regions.add_data_file(pos, data_file.file_id) morton[ind:ind + pos.shape[0]] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], data_file.ds.domain_left_edge, data_file.ds.domain_right_edge, ) return morton
def _initialize_index(self, data_file, regions): halos = data_file.read_data(usecols=['ID', 'Xc', 'Yc', 'Zc']) pcount = len(halos['ID']) morton = np.empty(pcount, dtype='uint64') mylog.debug('Initializing index % 5i (% 7i particles)', data_file.file_id, pcount) if pcount == 0: return morton ind = 0 pos = np.empty((pcount, 3), dtype='float64') pos = data_file.ds.arr(pos, 'code_length') dx = np.finfo(halos['Xc'].dtype).eps dx = 2.0 * self.ds.quan(dx, 'code_length') pos[:, 0] = halos['Xc'] pos[:, 1] = halos['Yc'] pos[:, 2] = halos['Zc'] dle = self.ds.domain_left_edge dre = self.ds.domain_right_edge # These are 32 bit numbers, so we give a little lee-way. # Otherwise, for big sets of particles, we often will bump into the # domain edges. This helps alleviate that. np.clip(pos, dle + dx, dre - dx, pos) if np.any(pos.min(axis=0) < dle) or np.any(pos.max(axis=0) > dre): raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle, dre) regions.add_data_file(pos, data_file.file_id) morton[ind:ind + pos.shape[0]] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], dle, dre) return morton
def _initialize_index(self, data_file, regions): dle = self.ds.domain_left_edge.in_units("code_length").d dre = self.ds.domain_right_edge.in_units("code_length").d pcount = 0 for dd in self.ds.midx.iter_bbox_data(dle, dre, ["x"]): pcount += dd["x"].size morton = np.empty(pcount, dtype="uint64") ind = 0 chunk_id = 0 for dd in self.ds.midx.iter_bbox_data(dle, dre, ["x", "y", "z"]): npart = dd["x"].size pos = np.empty((npart, 3), dtype=dd["x"].dtype) pos[:, 0] = dd["x"] pos[:, 1] = dd["y"] pos[:, 2] = dd["z"] if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or np.any( pos.max(axis=0) > self.ds.domain_right_edge): raise YTDomainOverflow( pos.min(axis=0), pos.max(axis=0), self.ds.domain_left_edge, self.ds.domain_right_edge, ) regions.add_data_file(pos, chunk_id) morton[ind:ind + npart] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], data_file.ds.domain_left_edge, data_file.ds.domain_right_edge, ) ind += npart return morton
def _initialize_index(self, data_file, regions): pcount = data_file.header["num_halos"] morton = np.empty(pcount, dtype="uint64") mylog.debug("Initializing index % 5i (% 7i particles)", data_file.file_id, pcount) if pcount == 0: return morton ind = 0 ptype = "halos" with open(data_file.filename, "rb") as f: pos = data_file._get_particle_positions(ptype, f=f) pos = data_file.ds.arr(pos, "code_length") if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or np.any( pos.max(axis=0) > self.ds.domain_right_edge): raise YTDomainOverflow( pos.min(axis=0), pos.max(axis=0), self.ds.domain_left_edge, self.ds.domain_right_edge, ) regions.add_data_file(pos, data_file.file_id) morton[ind:ind + pos.shape[0]] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], data_file.ds.domain_left_edge, data_file.ds.domain_right_edge, ) return morton
def _initialize_index(self, data_file, regions): pcount = sum(data_file.total_particles.values()) morton = np.empty(pcount, dtype='uint64') if pcount == 0: return morton mylog.debug("Initializing index % 5i (% 7i particles)", data_file.file_id, pcount) ind = 0 with h5py.File(data_file.filename, mode="r") as f: if not f.keys(): return None dx = np.finfo(f["FOF"]['CenterOfMass'].dtype).eps dx = 2.0 * self.ds.quan(dx, "code_length") for ptype in data_file.ds.particle_types_raw: if data_file.total_particles[ptype] == 0: continue pos = f[ptype]["CenterOfMass"][()].astype("float64") pos = np.resize(pos, (data_file.total_particles[ptype], 3)) pos = data_file.ds.arr(pos, "code_length") # These are 32 bit numbers, so we give a little lee-way. # Otherwise, for big sets of particles, we often will bump into the # domain edges. This helps alleviate that. np.clip(pos, self.ds.domain_left_edge + dx, self.ds.domain_right_edge - dx, pos) if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \ np.any(pos.max(axis=0) > self.ds.domain_right_edge): raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), self.ds.domain_left_edge, self.ds.domain_right_edge) regions.add_data_file(pos, data_file.file_id) morton[ind:ind + pos.shape[0]] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], data_file.ds.domain_left_edge, data_file.ds.domain_right_edge) ind += pos.shape[0] return morton
def _initialize_index(self, data_file, regions): pcount = data_file.header["num_halos"] morton = np.empty(pcount, dtype='uint64') mylog.debug("Initializing index % 5i (% 7i particles)", data_file.file_id, pcount) ind = 0 if pcount == 0: return None with h5py.File(data_file.filename, "r") as f: if not f.keys(): return None pos = np.empty((pcount, 3), dtype="float64") units = parse_h5_attr(f["particle_position_x"], "units") dx = np.finfo(f['particle_position_x'].dtype).eps dx = 2.0 * self.ds.quan(dx, units).to("code_length") pos[:, 0] = f["particle_position_x"].value pos[:, 1] = f["particle_position_y"].value pos[:, 2] = f["particle_position_z"].value pos = data_file.ds.arr(pos, units).to("code_length") dle = self.ds.domain_left_edge.to("code_length") dre = self.ds.domain_right_edge.to("code_length") # These are 32 bit numbers, so we give a little lee-way. # Otherwise, for big sets of particles, we often will bump into the # domain edges. This helps alleviate that. np.clip(pos, dle + dx, dre - dx, pos) if np.any(pos.min(axis=0) < dle) or \ np.any(pos.max(axis=0) > dre): raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle, dre) regions.add_data_file(pos, data_file.file_id) morton[ind:ind + pos.shape[0]] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], dle, dre) return morton
def _yield_coordinates(self, data_file, needed_ptype=None): # self.fields[g.id][fname] is the pattern here for ptype in self.ds.particle_types_raw: if needed_ptype is not None and needed_ptype is not ptype: continue try: pos = np.column_stack( [ self.fields[data_file.filename][ (ptype, f"particle_position_{ax}") ] for ax in "xyz" ] ) except KeyError: pos = self.fields[data_file.filename][ptype, "particle_position"] if np.any(pos.min(axis=0) < data_file.ds.domain_left_edge) or np.any( pos.max(axis=0) > data_file.ds.domain_right_edge ): raise YTDomainOverflow( pos.min(axis=0), pos.max(axis=0), data_file.ds.domain_left_edge, data_file.ds.domain_right_edge, ) yield ptype, pos
def _initialize_index(self, data_file, regions): halos = data_file.read_data(usecols=['ID']) pcount = len(halos['ID']) morton = np.empty(pcount, dtype='uint64') mylog.debug('Initializing index % 5i (% 7i particles)', data_file.file_id, pcount) if pcount == 0: return morton ind = 0 pos = data_file._get_particle_positions('halos') pos = data_file.ds.arr(pos, 'code_length') dle = self.ds.domain_left_edge dre = self.ds.domain_right_edge if np.any(pos.min(axis=0) < dle) or np.any(pos.max(axis=0) > dre): raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle, dre) regions.add_data_file(pos, data_file.file_id) morton[ind:ind + pos.shape[0]] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], dle, dre) return morton
def _initialize_index(self, data_file, regions): all_count = self._count_particles(data_file) pcount = sum(all_count.values()) morton = np.empty(pcount, dtype="uint64") mylog.debug("Initializing index % 5i (% 7i particles)", data_file.file_id, pcount) ind = 0 with h5py.File(data_file.filename, mode="r") as f: for ptype in all_count: if ptype not in f or all_count[ptype] == 0: continue pos = np.empty((all_count[ptype], 3), dtype="float64") units = _get_position_array_units(ptype, f, "x") if ptype == "grid": dx = f["grid"]["dx"][()].min() dx = self.ds.quan(dx, parse_h5_attr(f["grid"]["dx"], "units")).to("code_length") else: dx = 2.0 * np.finfo( f[ptype]["particle_position_x"].dtype).eps dx = self.ds.quan(dx, units).to("code_length") pos[:, 0] = _get_position_array(ptype, f, "x") pos[:, 1] = _get_position_array(ptype, f, "y") pos[:, 2] = _get_position_array(ptype, f, "z") pos = self.ds.arr(pos, units).to("code_length") dle = self.ds.domain_left_edge.to("code_length") dre = self.ds.domain_right_edge.to("code_length") # These are 32 bit numbers, so we give a little lee-way. # Otherwise, for big sets of particles, we often will bump into the # domain edges. This helps alleviate that. np.clip(pos, dle + dx, dre - dx, pos) if np.any(pos.min(axis=0) < dle) or np.any( pos.max(axis=0) > dre): raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle, dre) regions.add_data_file(pos, data_file.file_id) morton[ind:ind + pos.shape[0]] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], dle, dre) ind += pos.shape[0] return morton
def _initialize_index(self, data_file, regions): # self.fields[g.id][fname] is the pattern here morton = [] for ptype in self.ds.particle_types_raw: try: pos = np.column_stack(self.fields[data_file.filename][ (ptype, "particle_position_%s" % ax)] for ax in 'xyz') except KeyError: pos = self.fields[data_file.filename][ptype, "particle_position"] if np.any(pos.min(axis=0) < data_file.ds.domain_left_edge) or \ np.any(pos.max(axis=0) > data_file.ds.domain_right_edge): raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), data_file.ds.domain_left_edge, data_file.ds.domain_right_edge) regions.add_data_file(pos, data_file.file_id) morton.append(compute_morton( pos[:,0], pos[:,1], pos[:,2], data_file.ds.domain_left_edge, data_file.ds.domain_right_edge)) return np.concatenate(morton)
def _initialize_index(self, data_file, regions): all_count = self._count_particles(data_file) pcount = sum(all_count.values()) morton = np.empty(pcount, dtype='uint64') mylog.debug("Initializing index % 5i (% 7i particles)", data_file.file_id, pcount) ind = 0 with h5py.File(data_file.filename, "r") as f: for ptype in all_count: if ptype not in f or all_count[ptype] == 0: continue pos = np.empty((all_count[ptype], 3), dtype="float64") pos = data_file.ds.arr(pos, "code_length") if ptype == "grid": dx = f["grid"]["pdx"].value.min() else: raise NotImplementedError dx = self.ds.quan(dx, "code_length") pos[:, 0] = _get_position_array(ptype, f, "px") pos[:, 1] = _get_position_array(ptype, f, "py") pos[:,2] = np.zeros(all_count[ptype], dtype="float64") + \ self.ds.domain_left_edge[2].in_cgs().d # These are 32 bit numbers, so we give a little lee-way. # Otherwise, for big sets of particles, we often will bump into the # domain edges. This helps alleviate that. np.clip(pos, self.ds.domain_left_edge + dx, self.ds.domain_right_edge - dx, pos) if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \ np.any(pos.max(axis=0) > self.ds.domain_right_edge): raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), self.ds.domain_left_edge, self.ds.domain_right_edge) regions.add_data_file(pos, data_file.file_id) morton[ind:ind + pos.shape[0]] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], data_file.ds.domain_left_edge, data_file.ds.domain_right_edge) ind += pos.shape[0] return morton
def _initialize_index(self, data_file, regions): pcount = data_file.header["num_halos"] morton = np.empty(pcount, dtype='uint64') mylog.debug("Initializing index % 5i (% 7i particles)", data_file.file_id, pcount) if pcount == 0: return morton ind = 0 with open(data_file.filename, "rb") as f: f.seek(data_file._position_offset, os.SEEK_SET) halos = np.fromfile(f, dtype=self._halo_dt, count = pcount) pos = np.empty((halos.size, 3), dtype="float64") # These positions are in Mpc, *not* "code" units pos = data_file.ds.arr(pos, "code_length") eps = np.finfo(halos['particle_position_x'].dtype).eps # Make sure eps is not larger than the domain itself eps = eps*self.ds.domain_right_edge dx = np.abs(eps).max() pos[:,0] = halos["particle_position_x"] pos[:,1] = halos["particle_position_y"] pos[:,2] = halos["particle_position_z"] # These are 32 bit numbers, so we give a little lee-way. # Otherwise, for big sets of particles, we often will bump into the # domain edges. This helps alleviate that. np.clip(pos, self.ds.domain_left_edge + dx, self.ds.domain_right_edge - dx, pos) del halos if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \ np.any(pos.max(axis=0) > self.ds.domain_right_edge): raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), self.ds.domain_left_edge, self.ds.domain_right_edge) regions.add_data_file(pos, data_file.file_id) morton[ind:ind+pos.shape[0]] = compute_morton( pos[:,0], pos[:,1], pos[:,2], data_file.ds.domain_left_edge, data_file.ds.domain_right_edge) return morton
def _initialize_index(self, data_file, regions): pcount = data_file.header["num_halos"] morton = np.empty(pcount, dtype='uint64') mylog.debug("Initializing index % 5i (% 7i particles)", data_file.file_id, pcount) ind = 0 if pcount == 0: return None ptype = 'halos' with h5py.File(data_file.filename, "r") as f: if not f.keys(): return None units = parse_h5_attr(f["particle_position_x"], "units") pos = data_file._get_particle_positions(ptype, f=f) pos = data_file.ds.arr(pos, units).to("code_length") dle = self.ds.domain_left_edge.to("code_length") dre = self.ds.domain_right_edge.to("code_length") if np.any(pos.min(axis=0) < dle) or \ np.any(pos.max(axis=0) > dre): raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle, dre) regions.add_data_file(pos, data_file.file_id) morton[ind:ind + pos.shape[0]] = compute_morton( pos[:, 0], pos[:, 1], pos[:, 2], dle, dre) return morton