def _read_particle_fields(self, chunks, ptf, selector):
# Now we have all the sizes, and we can allocate
chunks = list(chunks)
data_files = set([])
# Only support halo reading for now.
assert len(ptf) == 1
assert list(ptf.keys())[0] == "halos"
for chunk in chunks:
for obj in chunk.objs:
data_files.update(obj.data_files)
pn = "particle_position_%s"
for data_file in sorted(data_files,
key=lambda x: (x.filename, x.start)):
si, ei = data_file.start, data_file.end
with h5py.File(data_file.filename, mode="r") as f:
for ptype, field_list in sorted(ptf.items()):
units = parse_h5_attr(f[pn % "x"], "units")
pos = data_file._get_particle_positions(ptype, f=f)
x, y, z = (self.ds.arr(pos[:, i], units) for i in range(3))
mask = selector.select_points(x, y, z, 0.0)
del x, y, z
if mask is None:
continue
for field in field_list:
data = f[field][si:ei][mask].astype("float64")
yield (ptype, field), data
def _is_valid(self, *args, **kwargs):
if not args[0].endswith(".h5"): return False
with h5py.File(args[0], "r") as f:
data_type = parse_h5_attr(f, "data_type")
if data_type == "yt_profile":
return True
return False
def _identify_fields(self, data_file):
with h5py.File(data_file.filename, mode="r") as f:
fields = [("halos", field) for field in f
if not isinstance(f[field], h5py.Group)]
units = dict([(("halos", field), parse_h5_attr(f[field], "units"))
for field in f])
return fields, units
def _read_particle_fields(self, chunks, ptf, selector):
pn = "particle_position_%s"
chunks = list(chunks)
for chunk in chunks: # These should be organized by grid filename
f = None
for g in chunk.objs:
if g.filename is None:
continue
if f is None:
f = h5py.File(g.filename, mode="r")
if g.NumberOfParticles == 0:
continue
for ptype, field_list in sorted(ptf.items()):
units = parse_h5_attr(f[ptype][pn % "x"], "units")
x, y, z = (self.ds.arr(
f[ptype][pn % ax][()].astype("float64"), units)
for ax in "xyz")
mask = selector.select_points(x, y, z, 0.0)
if mask is None:
continue
for field in field_list:
data = np.asarray(f[ptype][field][()], "=f8")
yield (ptype, field), data[mask]
if f:
f.close()
def _is_valid(self, *args, **kwargs):
if not args[0].endswith(".h5"): return False
with h5py.File(args[0], mode="r") as f:
if "data_type" in f.attrs and \
parse_h5_attr(f, "data_type") == "halo_catalog":
return True
return False
def __init__(self, ds, io, filename, file_id, range):
with h5py.File(filename, mode="r") as f:
self.header = dict(
(field, parse_h5_attr(f, field)) for field in f.attrs.keys()
)
super(YTDataHDF5File, self).__init__(ds, io, filename, file_id, range)
def _is_valid(cls, filename, *args, **kwargs):
if not filename.endswith(".h5"):
return False
with h5py.File(filename, mode="r") as f:
data_type = parse_h5_attr(f, "data_type")
if data_type == "yt_profile":
return True
return False
def _is_valid(cls, filename, *args, **kwargs):
if not filename.endswith(".h5"):
return False
with h5py.File(filename, mode="r") as f:
if ("data_type" in f.attrs
and parse_h5_attr(f, "data_type") == "halo_catalog"):
return True
return False
def _is_valid(self, *args, **kwargs):
if not args[0].endswith(".h5"):
return False
with h5py.File(args[0], mode="r") as f:
data_type = parse_h5_attr(f, "data_type")
if data_type in ["yt_light_ray"]:
return True
return False
def __init__(self, ds, io, filename, file_id, range):
with h5py.File(filename, mode="r") as f:
self.header = {
field: parse_h5_attr(f, field)
for field in f.attrs.keys()
}
super().__init__(ds, io, filename, file_id, range)
def _with_parameter_file_open(self, f):
self.num_particles = dict(
[
(group, parse_h5_attr(f[group], "num_elements"))
for group in f
if group != self.default_fluid_type
]
)
def _parse_parameter_file(self):
self.refine_by = 2
with h5py.File(self.parameter_filename, mode="r") as f:
for key in f.attrs.keys():
v = parse_h5_attr(f, key)
if key == "con_args":
try:
v = eval(v)
except ValueError:
# support older ytdata outputs
v = v.astype('str')
self.parameters[key] = v
self._with_parameter_file_open(f)
# if saved, restore unit registry from the json string
if "unit_registry_json" in self.parameters:
self.unit_registry = UnitRegistry.from_json(
self.parameters["unit_registry_json"])
# reset self.arr and self.quan to use new unit_registry
self._arr = None
self._quan = None
for dim in [
"length", "mass", "pressure", "temperature", "time",
"velocity"
]:
cu = "code_" + dim
if cu not in self.unit_registry:
self.unit_registry.add(cu, 1.0, getattr(dimensions, dim))
if "code_magnetic" not in self.unit_registry:
self.unit_registry.add("code_magnetic", 1.0,
dimensions.magnetic_field)
# if saved, set unit system
if "unit_system_name" in self.parameters:
unit_system = self.parameters["unit_system_name"]
del self.parameters["unit_system_name"]
else:
unit_system = "cgs"
# reset unit system since we may have a new unit registry
self._assign_unit_system(unit_system)
# assign units to parameters that have associated unit string
del_pars = []
for par in self.parameters:
ustr = "%s_units" % par
if ustr in self.parameters:
if isinstance(self.parameters[par], np.ndarray):
to_u = self.arr
else:
to_u = self.quan
self.parameters[par] = to_u(self.parameters[par],
self.parameters[ustr])
del_pars.append(ustr)
for par in del_pars:
del self.parameters[par]
for attr in self._con_attrs:
setattr(self, attr, self.parameters.get(attr))
def _is_valid(self, *args, **kwargs):
if not args[0].endswith(".h5"): return False
with h5py.File(args[0], mode="r") as f:
data_type = parse_h5_attr(f, "data_type")
if data_type is None:
return False
if data_type == "yt_clump_tree":
return True
return False
def __init__(self, ds, io, filename, file_id, frange):
with h5py.File(filename, mode="r") as f:
self.header = dict(
(field, parse_h5_attr(f, field)) for field in f.attrs.keys())
pids = f.get("particles/ids")
self.total_ids = 0 if pids is None else pids.size
self.group_length_sum = self.total_ids
super(YTHaloCatalogFile, self).__init__(ds, io, filename, file_id,
frange)
def _yield_coordinates(self, data_file):
pn = "particle_position_%s"
with h5py.File(data_file.filename, 'r') as f:
units = parse_h5_attr(f[pn % "x"], "units")
x, y, z = (self.ds.arr(f[pn % ax].value.astype("float64"), units)
for ax in 'xyz')
pos = uvstack([x, y, z]).T
pos.convert_to_units('code_length')
yield 'halos', pos
def _yield_coordinates(self, data_file):
pn = "particle_position_%s"
with h5py.File(data_file.filename, mode="r") as f:
units = parse_h5_attr(f[pn % "x"], "units")
x, y, z = (self.ds.arr(f[pn % ax][()].astype("float64"), units)
for ax in "xyz")
pos = uvstack([x, y, z]).T
pos.convert_to_units("code_length")
yield "halos", pos
def _identify_fields(self, data_file):
fields = []
units = {}
with h5py.File(data_file.filename, "r") as f:
for ptype in f:
fields.extend([(ptype, str(field)) for field in f[ptype]])
units.update(dict([((ptype, str(field)),
parse_h5_attr(f[ptype][field], "units"))
for field in f[ptype]]))
return fields, units
def _detect_output_fields(self):
self.field_list = []
self.ds.field_units = self.ds.field_units or {}
with h5py.File(self.ds.parameter_filename, mode="r") as f:
for group in f:
for field in f[group]:
field_name = (str(group), str(field))
self.field_list.append(field_name)
self.ds.field_units[field_name] = parse_h5_attr(
f[group][field], "units")
def _identify_fields(self, data_file):
with h5py.File(data_file.filename, mode="r") as f:
scalar_fields = [("halos", field) for field in f
if not isinstance(f[field], h5py.Group)]
units = {("halos", field): parse_h5_attr(f[field], "units")
for field in f}
if "particles" in f:
id_fields = [("halos", field) for field in f["particles"]]
else:
id_fields = []
return scalar_fields + id_fields, scalar_fields, id_fields, units
def __init__(self, table_type, redshift=0.0, data_dir=None, use_metals=True):
filename = _get_data_file(table_type, data_dir=data_dir)
only_on_root(mylog.info, "Loading emissivity data from %s." % filename)
in_file = h5py.File(filename, "r")
if "info" in in_file.attrs:
only_on_root(mylog.info, parse_h5_attr(in_file, "info"))
if parse_h5_attr(in_file, "version") != data_version[table_type]:
raise ObsoleteDataException(table_type)
else:
only_on_root(mylog.info, "X-ray '%s' emissivity data version: %s." % \
(table_type, parse_h5_attr(in_file, "version")))
self.log_T = in_file["log_T"][:]
self.emissivity_primordial = in_file["emissivity_primordial"][:]
if "log_nH" in in_file:
self.log_nH = in_file["log_nH"][:]
if use_metals:
self.emissivity_metals = in_file["emissivity_metals"][:]
self.ebin = YTArray(in_file["E"], "keV")
in_file.close()
self.dE = np.diff(self.ebin)
self.emid = 0.5*(self.ebin[1:]+self.ebin[:-1]).to("erg")
self.redshift = redshift
def _read_particle_coords(self, chunks, ptf):
# This will read chunks and yield the results.
chunks = list(chunks)
data_files = set([])
# Only support halo reading for now.
assert (len(ptf) == 1)
assert (list(ptf.keys())[0] == "halos")
for chunk in chunks:
for obj in chunk.objs:
data_files.update(obj.data_files)
pn = "particle_position_%s"
for data_file in sorted(data_files):
with h5py.File(data_file.filename, "r") as f:
units = parse_h5_attr(f[pn % "x"], "units")
x, y, z = \
(self.ds.arr(f[pn % ax].value.astype("float64"), units)
for ax in "xyz")
yield "halos", (x, y, z)
def _read_particle_coords(self, chunks, ptf):
# This will read chunks and yield the results.
chunks = list(chunks)
data_files = set([])
# Only support halo reading for now.
assert (len(ptf) == 1)
assert (list(ptf.keys())[0] == "halos")
ptype = 'halos'
for chunk in chunks:
for obj in chunk.objs:
data_files.update(obj.data_files)
pn = "particle_position_%s"
for data_file in sorted(data_files):
with h5py.File(data_file.filename, "r") as f:
units = parse_h5_attr(f[pn % "x"], "units")
pos = data_file._get_particle_positions(ptype, f=f)
x, y, z = (self.ds.arr(pos[:, i], units) for i in range(3))
yield "halos", (x, y, z)
def _initialize_index(self, data_file, regions):
all_count = self._count_particles(data_file)
pcount = sum(all_count.values())
morton = np.empty(pcount, dtype="uint64")
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
with h5py.File(data_file.filename, mode="r") as f:
for ptype in all_count:
if ptype not in f or all_count[ptype] == 0:
continue
pos = np.empty((all_count[ptype], 3), dtype="float64")
units = _get_position_array_units(ptype, f, "x")
if ptype == "grid":
dx = f["grid"]["dx"][()].min()
dx = self.ds.quan(dx,
parse_h5_attr(f["grid"]["dx"],
"units")).to("code_length")
else:
dx = 2.0 * np.finfo(
f[ptype]["particle_position_x"].dtype).eps
dx = self.ds.quan(dx, units).to("code_length")
pos[:, 0] = _get_position_array(ptype, f, "x")
pos[:, 1] = _get_position_array(ptype, f, "y")
pos[:, 2] = _get_position_array(ptype, f, "z")
pos = self.ds.arr(pos, units).to("code_length")
dle = self.ds.domain_left_edge.to("code_length")
dre = self.ds.domain_right_edge.to("code_length")
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, dle + dx, dre - dx, pos)
if np.any(pos.min(axis=0) < dle) or np.any(
pos.max(axis=0) > dre):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0),
dle, dre)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], dle, dre)
ind += pos.shape[0]
return morton
def _read_particle_coords(self, chunks, ptf):
pn = "particle_position_%s"
chunks = list(chunks)
for chunk in chunks:
f = None
for g in chunk.objs:
if g.filename is None: continue
if f is None:
f = h5py.File(g.filename, "r")
if g.NumberOfParticles == 0:
continue
for ptype, field_list in sorted(ptf.items()):
units = parse_h5_attr(f[ptype][pn % "x"], "units")
x, y, z = \
(self.ds.arr(f[ptype][pn % ax][()].astype("float64"), units)
for ax in "xyz")
for field in field_list:
if np.asarray(f[ptype][field]).ndim > 1:
self._array_fields[field] = f[ptype][field].shape[1:]
yield ptype, (x, y, z)
if f: f.close()
def _initialize_index(self, data_file, regions):
pcount = data_file.header["num_halos"]
morton = np.empty(pcount, dtype='uint64')
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
if pcount == 0: return None
ptype = 'halos'
with h5py.File(data_file.filename, "r") as f:
if not f.keys(): return None
units = parse_h5_attr(f["particle_position_x"], "units")
pos = data_file._get_particle_positions(ptype, f=f)
pos = data_file.ds.arr(pos, units).to("code_length")
dle = self.ds.domain_left_edge.to("code_length")
dre = self.ds.domain_right_edge.to("code_length")
if np.any(pos.min(axis=0) < dle) or \
np.any(pos.max(axis=0) > dre):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle,
dre)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], dle, dre)
return morton
def _read_particle_fields(self, chunks, ptf, selector):
# Now we have all the sizes, and we can allocate
chunks = list(chunks)
data_files = set([])
# Only support halo reading for now.
assert (len(ptf) == 1)
assert (list(ptf.keys())[0] == "halos")
for chunk in chunks:
for obj in chunk.objs:
data_files.update(obj.data_files)
pn = "particle_position_%s"
for data_file in sorted(data_files):
with h5py.File(data_file.filename, "r") as f:
for ptype, field_list in sorted(ptf.items()):
units = parse_h5_attr(f[pn % "x"], "units")
x, y, z = \
(self.ds.arr(f[pn % ax].value.astype("float64"), units)
for ax in "xyz")
mask = selector.select_points(x, y, z, 0.0)
del x, y, z
if mask is None: continue
for field in field_list:
data = f[field][mask].astype("float64")
yield (ptype, field), data
def _parse_parameter_file(self):
self.refine_by = 2
with h5py.File(self.parameter_filename, mode="r") as f:
for key in f.attrs.keys():
v = parse_h5_attr(f, key)
if key == "con_args":
try:
v = eval(v)
except ValueError:
# support older ytdata outputs
v = v.astype("str")
except NameError:
# This is the most common error we expect, and it
# results from having the eval do a concatenated decoded
# set of the values.
v = [_.decode("utf8") for _ in v]
self.parameters[key] = v
self._with_parameter_file_open(f)
# if saved, restore unit registry from the json string
if "unit_registry_json" in self.parameters:
self.unit_registry = UnitRegistry.from_json(
self.parameters["unit_registry_json"])
# reset self.arr and self.quan to use new unit_registry
self._arr = None
self._quan = None
for dim in [
"length",
"mass",
"pressure",
"temperature",
"time",
"velocity",
]:
cu = "code_" + dim
if cu not in self.unit_registry:
self.unit_registry.add(cu, 1.0, getattr(dimensions, dim))
if "code_magnetic" not in self.unit_registry:
self.unit_registry.add("code_magnetic", 1.0,
dimensions.magnetic_field)
# if saved, set unit system
if "unit_system_name" in self.parameters:
unit_system = self.parameters["unit_system_name"]
del self.parameters["unit_system_name"]
else:
unit_system = "cgs"
# reset unit system since we may have a new unit registry
self._assign_unit_system(unit_system)
# assign units to parameters that have associated unit string
del_pars = []
for par in self.parameters:
ustr = f"{par}_units"
if ustr in self.parameters:
if isinstance(self.parameters[par], np.ndarray):
to_u = self.arr
else:
to_u = self.quan
self.parameters[par] = to_u(self.parameters[par],
self.parameters[ustr])
del_pars.append(ustr)
for par in del_pars:
del self.parameters[par]
for attr in self._con_attrs:
try:
sattr = _set_attrs.get(attr, attr)
setattr(self, sattr, self.parameters.get(attr))
except TypeError:
# some Dataset attributes are properties with setters
# which may not accept None as an input
pass
if self.geometry is None:
self.geometry = "cartesian"
def _identify_fields(self, data_file):
with h5py.File(data_file.filename, "r") as f:
fields = [("halos", field) for field in f]
units = dict([(("halos", field), parse_h5_attr(f[field], "units"))
for field in f])
return fields, units
def __init__(self, ds, io, filename, file_id):
with h5py.File(filename, "r") as f:
self.header = dict((field, parse_h5_attr(f, field)) \
for field in f.attrs.keys())
super(HaloCatalogHDF5File, self).__init__(ds, io, filename, file_id)
def _get_position_array_units(ptype, f, ax):
if ptype == "grid":
pos_name = ""
else:
pos_name = "particle_position_"
return parse_h5_attr(f[ptype][pos_name + ax], "units")