def _initialize_index(self, data_file, regions):
if self.index_ptype == "all":
ptypes = self.ds.particle_types_raw
pcount = sum(data_file.total_particles.values())
else:
ptypes = [self.index_ptype]
pcount = data_file.total_particles[self.index_ptype]
morton = np.empty(pcount, dtype='uint64')
if pcount == 0: return morton
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
with h5py.File(data_file.filename, "r") as f:
if not f.keys(): return None
dx = np.finfo(f["Group"]["GroupPos"].dtype).eps
dx = 2.0 * self.ds.quan(dx, "code_length")
for ptype in ptypes:
if data_file.total_particles[ptype] == 0:
continue
pos = data_file._get_particle_positions(ptype, f=f)
pos = self.ds.arr(pos, "code_length")
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \
np.any(pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], self.ds.domain_left_edge,
self.ds.domain_right_edge)
ind += pos.shape[0]
return morton
def _initialize_index(self, data_file, regions):
pcount = data_file.ds.parameters["nhalos"] + data_file.ds.parameters[
"nsubs"]
morton = np.empty(pcount, dtype="uint64")
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
if pcount == 0:
return morton
ind = 0
pos = self._get_particle_positions()
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or np.any(
pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(
pos.min(axis=0),
pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge,
)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0],
pos[:, 1],
pos[:, 2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge,
)
return morton
def _initialize_index(self, data_file, regions):
halos = data_file.read_data(usecols=['ID', 'Xc', 'Yc', 'Zc'])
pcount = len(halos['ID'])
morton = np.empty(pcount, dtype='uint64')
mylog.debug('Initializing index % 5i (% 7i particles)',
data_file.file_id, pcount)
if pcount == 0:
return morton
ind = 0
pos = np.empty((pcount, 3), dtype='float64')
pos = data_file.ds.arr(pos, 'code_length')
dx = np.finfo(halos['Xc'].dtype).eps
dx = 2.0 * self.ds.quan(dx, 'code_length')
pos[:, 0] = halos['Xc']
pos[:, 1] = halos['Yc']
pos[:, 2] = halos['Zc']
dle = self.ds.domain_left_edge
dre = self.ds.domain_right_edge
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, dle + dx, dre - dx, pos)
if np.any(pos.min(axis=0) < dle) or np.any(pos.max(axis=0) > dre):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle, dre)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], dle, dre)
return morton
def _initialize_index(self, data_file, regions):
dle = self.ds.domain_left_edge.in_units("code_length").d
dre = self.ds.domain_right_edge.in_units("code_length").d
pcount = 0
for dd in self.ds.midx.iter_bbox_data(dle, dre, ["x"]):
pcount += dd["x"].size
morton = np.empty(pcount, dtype="uint64")
ind = 0
chunk_id = 0
for dd in self.ds.midx.iter_bbox_data(dle, dre, ["x", "y", "z"]):
npart = dd["x"].size
pos = np.empty((npart, 3), dtype=dd["x"].dtype)
pos[:, 0] = dd["x"]
pos[:, 1] = dd["y"]
pos[:, 2] = dd["z"]
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or np.any(
pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(
pos.min(axis=0),
pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge,
)
regions.add_data_file(pos, chunk_id)
morton[ind:ind + npart] = compute_morton(
pos[:, 0],
pos[:, 1],
pos[:, 2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge,
)
ind += npart
return morton
def _initialize_index(self, data_file, regions):
pcount = data_file.header["num_halos"]
morton = np.empty(pcount, dtype="uint64")
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
if pcount == 0:
return morton
ind = 0
ptype = "halos"
with open(data_file.filename, "rb") as f:
pos = data_file._get_particle_positions(ptype, f=f)
pos = data_file.ds.arr(pos, "code_length")
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or np.any(
pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(
pos.min(axis=0),
pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge,
)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0],
pos[:, 1],
pos[:, 2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge,
)
return morton
def _initialize_index(self, data_file, regions):
pcount = sum(data_file.total_particles.values())
morton = np.empty(pcount, dtype='uint64')
if pcount == 0: return morton
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
with h5py.File(data_file.filename, mode="r") as f:
if not f.keys(): return None
dx = np.finfo(f["FOF"]['CenterOfMass'].dtype).eps
dx = 2.0 * self.ds.quan(dx, "code_length")
for ptype in data_file.ds.particle_types_raw:
if data_file.total_particles[ptype] == 0: continue
pos = f[ptype]["CenterOfMass"][()].astype("float64")
pos = np.resize(pos, (data_file.total_particles[ptype], 3))
pos = data_file.ds.arr(pos, "code_length")
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, self.ds.domain_left_edge + dx,
self.ds.domain_right_edge - dx, pos)
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \
np.any(pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge)
ind += pos.shape[0]
return morton
def _initialize_index(self, data_file, regions):
pcount = data_file.header["num_halos"]
morton = np.empty(pcount, dtype='uint64')
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
if pcount == 0: return None
with h5py.File(data_file.filename, "r") as f:
if not f.keys(): return None
pos = np.empty((pcount, 3), dtype="float64")
units = parse_h5_attr(f["particle_position_x"], "units")
dx = np.finfo(f['particle_position_x'].dtype).eps
dx = 2.0 * self.ds.quan(dx, units).to("code_length")
pos[:, 0] = f["particle_position_x"].value
pos[:, 1] = f["particle_position_y"].value
pos[:, 2] = f["particle_position_z"].value
pos = data_file.ds.arr(pos, units).to("code_length")
dle = self.ds.domain_left_edge.to("code_length")
dre = self.ds.domain_right_edge.to("code_length")
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, dle + dx, dre - dx, pos)
if np.any(pos.min(axis=0) < dle) or \
np.any(pos.max(axis=0) > dre):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle,
dre)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], dle, dre)
return morton
def _yield_coordinates(self, data_file, needed_ptype=None):
# self.fields[g.id][fname] is the pattern here
for ptype in self.ds.particle_types_raw:
if needed_ptype is not None and needed_ptype is not ptype:
continue
try:
pos = np.column_stack(
[
self.fields[data_file.filename][
(ptype, f"particle_position_{ax}")
]
for ax in "xyz"
]
)
except KeyError:
pos = self.fields[data_file.filename][ptype, "particle_position"]
if np.any(pos.min(axis=0) < data_file.ds.domain_left_edge) or np.any(
pos.max(axis=0) > data_file.ds.domain_right_edge
):
raise YTDomainOverflow(
pos.min(axis=0),
pos.max(axis=0),
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge,
)
yield ptype, pos
def _initialize_index(self, data_file, regions):
halos = data_file.read_data(usecols=['ID'])
pcount = len(halos['ID'])
morton = np.empty(pcount, dtype='uint64')
mylog.debug('Initializing index % 5i (% 7i particles)',
data_file.file_id, pcount)
if pcount == 0:
return morton
ind = 0
pos = data_file._get_particle_positions('halos')
pos = data_file.ds.arr(pos, 'code_length')
dle = self.ds.domain_left_edge
dre = self.ds.domain_right_edge
if np.any(pos.min(axis=0) < dle) or np.any(pos.max(axis=0) > dre):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle, dre)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], dle, dre)
return morton
def _initialize_index(self, data_file, regions):
all_count = self._count_particles(data_file)
pcount = sum(all_count.values())
morton = np.empty(pcount, dtype="uint64")
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
with h5py.File(data_file.filename, mode="r") as f:
for ptype in all_count:
if ptype not in f or all_count[ptype] == 0:
continue
pos = np.empty((all_count[ptype], 3), dtype="float64")
units = _get_position_array_units(ptype, f, "x")
if ptype == "grid":
dx = f["grid"]["dx"][()].min()
dx = self.ds.quan(dx,
parse_h5_attr(f["grid"]["dx"],
"units")).to("code_length")
else:
dx = 2.0 * np.finfo(
f[ptype]["particle_position_x"].dtype).eps
dx = self.ds.quan(dx, units).to("code_length")
pos[:, 0] = _get_position_array(ptype, f, "x")
pos[:, 1] = _get_position_array(ptype, f, "y")
pos[:, 2] = _get_position_array(ptype, f, "z")
pos = self.ds.arr(pos, units).to("code_length")
dle = self.ds.domain_left_edge.to("code_length")
dre = self.ds.domain_right_edge.to("code_length")
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, dle + dx, dre - dx, pos)
if np.any(pos.min(axis=0) < dle) or np.any(
pos.max(axis=0) > dre):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0),
dle, dre)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], dle, dre)
ind += pos.shape[0]
return morton
def _initialize_index(self, data_file, regions):
# self.fields[g.id][fname] is the pattern here
morton = []
for ptype in self.ds.particle_types_raw:
try:
pos = np.column_stack(self.fields[data_file.filename][
(ptype, "particle_position_%s" % ax)] for ax in 'xyz')
except KeyError:
pos = self.fields[data_file.filename][ptype, "particle_position"]
if np.any(pos.min(axis=0) < data_file.ds.domain_left_edge) or \
np.any(pos.max(axis=0) > data_file.ds.domain_right_edge):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0),
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge)
regions.add_data_file(pos, data_file.file_id)
morton.append(compute_morton(
pos[:,0], pos[:,1], pos[:,2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge))
return np.concatenate(morton)
def _initialize_index(self, data_file, regions):
all_count = self._count_particles(data_file)
pcount = sum(all_count.values())
morton = np.empty(pcount, dtype='uint64')
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
with h5py.File(data_file.filename, "r") as f:
for ptype in all_count:
if ptype not in f or all_count[ptype] == 0: continue
pos = np.empty((all_count[ptype], 3), dtype="float64")
pos = data_file.ds.arr(pos, "code_length")
if ptype == "grid":
dx = f["grid"]["pdx"].value.min()
else:
raise NotImplementedError
dx = self.ds.quan(dx, "code_length")
pos[:, 0] = _get_position_array(ptype, f, "px")
pos[:, 1] = _get_position_array(ptype, f, "py")
pos[:,2] = np.zeros(all_count[ptype], dtype="float64") + \
self.ds.domain_left_edge[2].in_cgs().d
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, self.ds.domain_left_edge + dx,
self.ds.domain_right_edge - dx, pos)
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \
np.any(pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge)
ind += pos.shape[0]
return morton
def _initialize_index(self, data_file, regions):
pcount = data_file.header["num_halos"]
morton = np.empty(pcount, dtype='uint64')
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
if pcount == 0: return morton
ind = 0
with open(data_file.filename, "rb") as f:
f.seek(data_file._position_offset, os.SEEK_SET)
halos = np.fromfile(f, dtype=self._halo_dt, count = pcount)
pos = np.empty((halos.size, 3), dtype="float64")
# These positions are in Mpc, *not* "code" units
pos = data_file.ds.arr(pos, "code_length")
eps = np.finfo(halos['particle_position_x'].dtype).eps
# Make sure eps is not larger than the domain itself
eps = eps*self.ds.domain_right_edge
dx = np.abs(eps).max()
pos[:,0] = halos["particle_position_x"]
pos[:,1] = halos["particle_position_y"]
pos[:,2] = halos["particle_position_z"]
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, self.ds.domain_left_edge + dx,
self.ds.domain_right_edge - dx, pos)
del halos
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \
np.any(pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(pos.min(axis=0),
pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind+pos.shape[0]] = compute_morton(
pos[:,0], pos[:,1], pos[:,2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge)
return morton
def _initialize_index(self, data_file, regions):
pcount = data_file.header["num_halos"]
morton = np.empty(pcount, dtype='uint64')
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
if pcount == 0: return None
ptype = 'halos'
with h5py.File(data_file.filename, "r") as f:
if not f.keys(): return None
units = parse_h5_attr(f["particle_position_x"], "units")
pos = data_file._get_particle_positions(ptype, f=f)
pos = data_file.ds.arr(pos, units).to("code_length")
dle = self.ds.domain_left_edge.to("code_length")
dre = self.ds.domain_right_edge.to("code_length")
if np.any(pos.min(axis=0) < dle) or \
np.any(pos.max(axis=0) > dre):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle,
dre)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], dle, dre)
return morton