def _parse_parameter_file(self):
hvals = self._get_hvals()
self.dimensionality = 3
self.refine_by = 2
self.parameters["HydroMethod"] = "sph"
self.unique_identifier = \
int(os.stat(self.parameter_filename)[stat.ST_CTIME])
# Set standard values
# We may have an overridden bounding box.
if self.domain_left_edge is None:
self.domain_left_edge = np.zeros(3, "float64")
self.domain_right_edge = np.ones(3, "float64") * hvals["BoxSize"]
nz = 1 << self.over_refine_factor
self.domain_dimensions = np.ones(3, "int32") * nz
self.periodicity = (True, True, True)
self.cosmological_simulation = 1
self.current_redshift = hvals["Redshift"]
self.omega_lambda = hvals["OmegaLambda"]
self.omega_matter = hvals["Omega0"]
self.hubble_constant = hvals["HubbleParam"]
# According to the Gadget manual, OmegaLambda will be zero for
# non-cosmological datasets. However, it may be the case that
# individuals are running cosmological simulations *without* Lambda, in
# which case we may be doing something incorrect here.
# It may be possible to deduce whether ComovingIntegration is on
# somehow, but opinions on this vary.
if self.omega_lambda == 0.0:
only_on_root(mylog.info, "Omega Lambda is 0.0, so we are turning off Cosmology.")
self.hubble_constant = 1.0 # So that scaling comes out correct
self.cosmological_simulation = 0
self.current_redshift = 0.0
# This may not be correct.
self.current_time = hvals["Time"]
else:
# Now we calculate our time based on the cosmology, because in
# ComovingIntegration hvals["Time"] will in fact be the expansion
# factor, not the actual integration time, so we re-calculate
# global time from our Cosmology.
cosmo = Cosmology(self.hubble_constant,
self.omega_matter, self.omega_lambda)
self.current_time = cosmo.hubble_time(self.current_redshift)
only_on_root(mylog.info, "Calculating time from %0.3e to be %0.3e seconds",
hvals["Time"], self.current_time)
self.parameters = hvals
prefix = os.path.abspath(
os.path.join(os.path.dirname(self.parameter_filename),
os.path.basename(self.parameter_filename).split(".", 1)[0]))
if hvals["NumFiles"] > 1:
self.filename_template = "%s.%%(num)s%s" % (prefix, self._suffix)
else:
self.filename_template = self.parameter_filename
self.file_count = hvals["NumFiles"]
def _find_outputs(self):
"""
Search for directories matching the data dump keywords.
If found, get dataset times py opening the ds.
"""
# look for time outputs.
potential_time_outputs = glob.glob(
os.path.join(
self.parameters["GlobalDir"], f"{self.parameters['DataDumpDir']}*"
)
)
self.all_time_outputs = self._check_for_outputs(potential_time_outputs)
self.all_time_outputs.sort(key=lambda obj: obj["time"])
# look for redshift outputs.
potential_redshift_outputs = glob.glob(
os.path.join(
self.parameters["GlobalDir"], f"{self.parameters['RedshiftDumpDir']}*"
)
)
self.all_redshift_outputs = self._check_for_outputs(potential_redshift_outputs)
self.all_redshift_outputs.sort(key=lambda obj: obj["time"])
self.all_outputs = self.all_time_outputs + self.all_redshift_outputs
self.all_outputs.sort(key=lambda obj: obj["time"])
only_on_root(mylog.info, "Located %d total outputs.", len(self.all_outputs))
# manually set final time and redshift with last output
if self.all_outputs:
self.final_time = self.all_outputs[-1]["time"]
if self.cosmological_simulation:
self.final_redshift = self.all_outputs[-1]["redshift"]
def _check_for_outputs(self, potential_outputs):
r"""
Check a list of files to see if they are valid datasets.
"""
only_on_root(mylog.info, "Checking %d potential outputs.",
len(potential_outputs))
my_outputs = {}
llevel = mylog.level
# suppress logging as we load every dataset, unless set to debug
if llevel > 10 and llevel < 40:
mylog.setLevel(40)
for my_storage, output in parallel_objects(potential_outputs,
storage=my_outputs):
if os.path.exists(output):
try:
ds = load(output)
if ds is not None:
my_storage.result = {"filename": output,
"time": ds.current_time.in_units("s")}
if ds.cosmological_simulation:
my_storage.result["redshift"] = ds.current_redshift
except YTUnidentifiedDataType:
mylog.error("Failed to load %s", output)
mylog.setLevel(llevel)
my_outputs = [my_output for my_output in my_outputs.values() \
if my_output is not None]
return my_outputs
def _check_for_outputs(self, potential_outputs):
r"""
Check a list of files to see if they are valid datasets.
"""
only_on_root(mylog.info, "Checking %d potential outputs.",
len(potential_outputs))
my_outputs = {}
for my_storage, output in parallel_objects(potential_outputs,
storage=my_outputs):
try:
ds = load(output)
except (FileNotFoundError, YTUnidentifiedDataType):
mylog.error("Failed to load %s", output)
continue
my_storage.result = {
"filename": output,
"time": ds.current_time.in_units("s"),
}
if ds.cosmological_simulation:
my_storage.result["redshift"] = ds.current_redshift
my_outputs = [
my_output for my_output in my_outputs.values()
if my_output is not None
]
return my_outputs
def _set_code_unit_attributes(self):
"""
Sets the units from the SWIFT internal unit system.
Currently sets length, mass, time, and temperature.
SWIFT uses comoving coordinates without the usual h-factors.
"""
units = self._get_info_attributes("Units")
if self.cosmological_simulation == 1:
msg = "Assuming length units are in comoving centimetres"
only_on_root(mylog.info, msg)
self.length_unit = self.quan(
float(units["Unit length in cgs (U_L)"]), "cmcm")
else:
msg = "Assuming length units are in physical centimetres"
only_on_root(mylog.info, msg)
self.length_unit = self.quan(
float(units["Unit length in cgs (U_L)"]), "cm")
self.mass_unit = self.quan(float(units["Unit mass in cgs (U_M)"]), "g")
self.time_unit = self.quan(float(units["Unit time in cgs (U_t)"]), "s")
self.temperature_unit = self.quan(
float(units["Unit temperature in cgs (U_T)"]), "K")
return
def _check_for_outputs(self, potential_outputs):
r"""
Check a list of files to see if they are valid datasets.
"""
only_on_root(mylog.info, "Checking %d potential outputs.",
len(potential_outputs))
my_outputs = {}
for my_storage, output in parallel_objects(potential_outputs,
storage=my_outputs):
if os.path.exists(output):
try:
ds = load(output)
if ds is not None:
num_steps = ds.num_steps
my_storage.result = {
"filename": output,
"num_steps": num_steps
}
except YTOutputNotIdentified:
mylog.error("Failed to load %s", output)
my_outputs = [
my_output for my_output in my_outputs.values()
if my_output is not None
]
return my_outputs
def load_all_plugins(self, ftype="gas"):
loaded = []
for n in sorted(field_plugins):
loaded += self.load_plugin(n, ftype)
only_on_root(mylog.info, "Loaded %s (%s new fields)",
n, len(loaded))
self.find_dependencies(loaded)
def _check_for_outputs(self, potential_outputs):
r"""
Check a list of files to see if they are valid datasets.
"""
only_on_root(mylog.info, "Checking %d potential outputs.",
len(potential_outputs))
my_outputs = {}
for my_storage, output in parallel_objects(potential_outputs,
storage=my_outputs):
if os.path.exists(output):
try:
ds = load(output)
if ds is not None:
my_storage.result = {
"filename": output,
"time": ds.current_time.in_units("s")
}
if ds.cosmological_simulation:
my_storage.result["redshift"] = ds.current_redshift
except YTOutputNotIdentified:
mylog.error("Failed to load %s", output)
my_outputs = [my_output for my_output in my_outputs.values() \
if my_output is not None]
return my_outputs
def _initialize_particle_handler(self):
self._setup_data_io()
self._setup_filenames()
index_ptype = self.index_ptype
if index_ptype == "all":
self.total_particles = sum(
sum(d.total_particles.values()) for d in self.data_files)
else:
self.total_particles = sum(d.total_particles[index_ptype]
for d in self.data_files)
ds = self.dataset
self.oct_handler = ParticleOctreeContainer(
[1, 1, 1],
ds.domain_left_edge,
ds.domain_right_edge,
over_refine=ds.over_refine_factor)
self.oct_handler.n_ref = ds.n_ref
only_on_root(
mylog.info, "Allocating for %0.3e particles "
"(index particle type '%s')", self.total_particles, index_ptype)
# No more than 256^3 in the region finder.
N = min(len(self.data_files), 256)
self.regions = ParticleRegions(ds.domain_left_edge,
ds.domain_right_edge, [N, N, N],
len(self.data_files))
self._initialize_indices()
self.oct_handler.finalize()
self.max_level = self.oct_handler.max_level
self.dataset.max_level = self.max_level
tot = sum(self.oct_handler.recursively_count().values())
only_on_root(mylog.info, "Identified %0.3e octs", tot)
def _set_code_unit_attributes(self):
# Set a sane default for cosmological simulations.
if self._unit_base is None and self.cosmological_simulation == 1:
only_on_root(mylog.info,
"Assuming length units are in Mpc/h (comoving)")
self._unit_base = dict(length=(1.0, "Mpccm/h"))
# The other same defaults we will use from the standard Gadget
# defaults.
unit_base = self._unit_base or {}
if "length" in unit_base:
length_unit = unit_base["length"]
elif "UnitLength_in_cm" in unit_base:
if self.cosmological_simulation == 0:
length_unit = (unit_base["UnitLength_in_cm"], "cm")
else:
length_unit = (unit_base["UnitLength_in_cm"], "cmcm/h")
else:
raise RuntimeError
length_unit = _fix_unit_ordering(length_unit)
setdefaultattr(self, 'length_unit',
self.quan(length_unit[0], length_unit[1]))
if "velocity" in unit_base:
velocity_unit = unit_base["velocity"]
elif "UnitVelocity_in_cm_per_s" in unit_base:
velocity_unit = (unit_base["UnitVelocity_in_cm_per_s"], "cm/s")
else:
if self.cosmological_simulation == 0:
velocity_unit = (1e5, "cm/s")
else:
velocity_unit = (1e5, "cmcm/s")
velocity_unit = _fix_unit_ordering(velocity_unit)
setdefaultattr(self, 'velocity_unit',
self.quan(velocity_unit[0], velocity_unit[1]))
# We set hubble_constant = 1.0 for non-cosmology, so this is safe.
# Default to 1e10 Msun/h if mass is not specified.
if "mass" in unit_base:
mass_unit = unit_base["mass"]
elif "UnitMass_in_g" in unit_base:
if self.cosmological_simulation == 0:
mass_unit = (unit_base["UnitMass_in_g"], "g")
else:
mass_unit = (unit_base["UnitMass_in_g"], "g/h")
else:
# Sane default
mass_unit = (1.0, "1e10*Msun/h")
mass_unit = _fix_unit_ordering(mass_unit)
setdefaultattr(self, 'mass_unit', self.quan(mass_unit[0],
mass_unit[1]))
if "time" in unit_base:
time_unit = unit_base["time"]
elif "UnitTime_in_s" in unit_base:
time_unit = (unit_base["UnitTime_in_s"], "s")
else:
time_unit = (1., "s")
setdefaultattr(self, 'time_unit', self.quan(time_unit[0],
time_unit[1]))
def __init__(self, parameter_filename, simulation_type,
near_redshift, far_redshift,
observer_redshift=0.0,
use_minimum_datasets=True, deltaz_min=0.0,
minimum_coherent_box_fraction=0.0,
time_data=True, redshift_data=True,
find_outputs=False, set_parameters=None,
output_dir="LC", output_prefix="LightCone"):
self.near_redshift = near_redshift
self.far_redshift = far_redshift
self.observer_redshift = observer_redshift
self.use_minimum_datasets = use_minimum_datasets
self.deltaz_min = deltaz_min
self.minimum_coherent_box_fraction = minimum_coherent_box_fraction
if set_parameters is None:
self.set_parameters = {}
else:
self.set_parameters = set_parameters
self.output_dir = output_dir
self.output_prefix = output_prefix
# Create output directory.
if not os.path.exists(self.output_dir):
only_on_root(os.mkdir, self.output_dir)
# Calculate light cone solution.
CosmologySplice.__init__(self, parameter_filename, simulation_type,
find_outputs=find_outputs)
self.light_cone_solution = \
self.create_cosmology_splice(self.near_redshift, self.far_redshift,
minimal=self.use_minimum_datasets,
deltaz_min=self.deltaz_min,
time_data=time_data,
redshift_data=redshift_data)
def load_all_plugins(self, ftype="gas"):
loaded = []
for n in sorted(field_plugins):
loaded += self.load_plugin(n, ftype)
only_on_root(mylog.debug, "Loaded %s (%s new fields)", n,
len(loaded))
self.find_dependencies(loaded)
def load_all_plugins(self, ftype: Optional[str] = "gas"):
if ftype is None:
return
mylog.debug("Loading field plugins for field type: %s.", ftype)
loaded = []
for n in sorted(field_plugins):
loaded += self.load_plugin(n, ftype)
only_on_root(mylog.debug, "Loaded %s (%s new fields)", n,
len(loaded))
self.find_dependencies(loaded)
def _find_outputs(self):
"""
Search for directories matching the data dump keywords.
If found, get dataset times py opening the ds.
"""
potential_outputs = glob.glob(self._snapshot_format())
self.all_outputs = self._check_for_outputs(potential_outputs)
self.all_outputs.sort(key=lambda obj: obj["time"])
only_on_root(mylog.info, "Located %d total outputs.", len(self.all_outputs))
# manually set final time and redshift with last output
if self.all_outputs:
self.final_time = self.all_outputs[-1]["time"]
if self.cosmological_simulation:
self.final_redshift = self.all_outputs[-1]["redshift"]
def _check_for_outputs(self, potential_outputs):
"""
Check a list of files to see if they are valid datasets.
"""
only_on_root(
mylog.info, "Checking %d potential outputs.", len(potential_outputs)
)
my_outputs = {}
llevel = mylog.level
# suppress logging as we load every dataset, unless set to debug
if llevel > 10 and llevel < 40:
mylog.setLevel(40)
for my_storage, output in parallel_objects(
potential_outputs, storage=my_outputs
):
if self.parameters["DataDumpDir"] in output:
dir_key = self.parameters["DataDumpDir"]
output_key = self.parameters["DataDumpName"]
else:
dir_key = self.parameters["RedshiftDumpDir"]
output_key = self.parameters["RedshiftDumpName"]
index = output[output.find(dir_key) + len(dir_key) :]
filename = os.path.join(
self.parameters["GlobalDir"],
f"{dir_key}{index}",
f"{output_key}{index}",
)
try:
ds = load(filename)
except (FileNotFoundError, YTUnidentifiedDataType):
mylog.error("Failed to load %s", filename)
continue
my_storage.result = {
"filename": filename,
"time": ds.current_time.in_units("s"),
}
if ds.cosmological_simulation:
my_storage.result["redshift"] = ds.current_redshift
mylog.setLevel(llevel)
my_outputs = [
my_output for my_output in my_outputs.values() if my_output is not None
]
return my_outputs
def __init__(self,
parameter_filename,
simulation_type,
near_redshift,
far_redshift,
observer_redshift=0.0,
use_minimum_datasets=True,
deltaz_min=0.0,
minimum_coherent_box_fraction=0.0,
time_data=True,
redshift_data=True,
find_outputs=False,
set_parameters=None,
output_dir="LC",
output_prefix="LightCone"):
self.near_redshift = near_redshift
self.far_redshift = far_redshift
self.observer_redshift = observer_redshift
self.use_minimum_datasets = use_minimum_datasets
self.deltaz_min = deltaz_min
self.minimum_coherent_box_fraction = minimum_coherent_box_fraction
if set_parameters is None:
self.set_parameters = {}
else:
self.set_parameters = set_parameters
self.output_dir = output_dir
self.output_prefix = output_prefix
# Create output directory.
if not os.path.exists(self.output_dir):
only_on_root(os.mkdir, self.output_dir)
# Calculate light cone solution.
CosmologySplice.__init__(self,
parameter_filename,
simulation_type,
find_outputs=find_outputs)
self.light_cone_solution = \
self.create_cosmology_splice(self.near_redshift, self.far_redshift,
minimal=self.use_minimum_datasets,
deltaz_min=self.deltaz_min,
time_data=time_data,
redshift_data=redshift_data)
def _check_for_outputs(self, potential_outputs):
"""
Check a list of files to see if they are valid datasets.
"""
only_on_root(mylog.info, "Checking %d potential outputs.",
len(potential_outputs))
my_outputs = {}
llevel = mylog.level
# suppress logging as we load every dataset, unless set to debug
if llevel > 10 and llevel < 40:
mylog.setLevel(40)
for my_storage, output in parallel_objects(potential_outputs,
storage=my_outputs):
if self.parameters['DataDumpDir'] in output:
dir_key = self.parameters['DataDumpDir']
output_key = self.parameters['DataDumpName']
else:
dir_key = self.parameters['RedshiftDumpDir']
output_key = self.parameters['RedshiftDumpName']
index = output[output.find(dir_key) + len(dir_key):]
filename = os.path.join(self.parameters['GlobalDir'],
"%s%s" % (dir_key, index),
"%s%s" % (output_key, index))
if os.path.exists(filename):
try:
ds = load(filename)
if ds is not None:
my_storage.result = {
'filename': filename,
'time': ds.current_time.in_units("s")
}
if ds.cosmological_simulation:
my_storage.result['redshift'] = ds.current_redshift
except YTOutputNotIdentified:
mylog.error('Failed to load %s', filename)
mylog.setLevel(llevel)
my_outputs = [my_output for my_output in my_outputs.values() \
if my_output is not None]
return my_outputs
def __init__(self, filename=None):
default_filename = False
if filename is None:
filename = _get_data_file()
default_filename = True
if not os.path.exists(filename):
mylog.warning("File %s does not exist, will attempt to find it." % filename)
filename = _get_data_file(data_file=filename)
only_on_root(mylog.info, "Loading emissivity data from %s." % filename)
in_file = h5py.File(filename, "r")
if "info" in in_file.attrs:
only_on_root(mylog.info, in_file.attrs["info"])
if default_filename and \
in_file.attrs["version"] < xray_data_version:
raise ObsoleteDataException()
else:
only_on_root(mylog.info, "X-ray emissivity data version: %s." % \
in_file.attrs["version"])
for field in ["emissivity_primordial", "emissivity_metals",
"log_nH", "log_T", "log_E"]:
if field in in_file:
setattr(self, field, in_file[field][:])
in_file.close()
E_diff = np.diff(self.log_E)
self.E_bins = \
YTArray(np.power(10, np.concatenate([self.log_E[:-1] - 0.5 * E_diff,
[self.log_E[-1] - 0.5 * E_diff[-1],
self.log_E[-1] + 0.5 * E_diff[-1]]])),
"keV")
self.dnu = (np.diff(self.E_bins)/hcgs).in_units("Hz")
def __init__(self,
table_type,
redshift=0.0,
data_dir=None,
use_metals=True):
filename = _get_data_file(table_type, data_dir=data_dir)
only_on_root(mylog.info, "Loading emissivity data from %s", filename)
in_file = h5py.File(filename, mode="r")
if "info" in in_file.attrs:
only_on_root(mylog.info, parse_h5_attr(in_file, "info"))
if parse_h5_attr(in_file, "version") != data_version[table_type]:
raise ObsoleteDataException(table_type)
else:
only_on_root(
mylog.info,
"X-ray '%s' emissivity data version: %s." %
(table_type, parse_h5_attr(in_file, "version")),
)
self.log_T = in_file["log_T"][:]
self.emissivity_primordial = in_file["emissivity_primordial"][:]
if "log_nH" in in_file:
self.log_nH = in_file["log_nH"][:]
if use_metals:
self.emissivity_metals = in_file["emissivity_metals"][:]
self.ebin = YTArray(in_file["E"], "keV")
in_file.close()
self.dE = np.diff(self.ebin)
self.emid = 0.5 * (self.ebin[1:] + self.ebin[:-1]).to("erg")
self.redshift = redshift
def _check_for_outputs(self, potential_outputs):
r"""
Check a list of files to see if they are valid datasets.
"""
only_on_root(mylog.info, "Checking %d potential outputs.",
len(potential_outputs))
my_outputs = {}
for my_storage, output in parallel_objects(potential_outputs,
storage=my_outputs):
try:
ds = load(output)
except (FileNotFoundError, YTUnidentifiedDataType):
mylog.error("Failed to load %s", output)
continue
my_storage.result = {"filename": output, "num_steps": ds.num_steps}
my_outputs = [
my_output for my_output in my_outputs.values()
if my_output is not None
]
return my_outputs
def __init__(self, filename=None):
default_filename = False
if filename is None:
filename = _get_data_file()
default_filename = True
if not os.path.exists(filename):
mylog.warning("File %s does not exist, will attempt to find it." %
filename)
filename = _get_data_file(data_file=filename)
only_on_root(mylog.info, "Loading emissivity data from %s." % filename)
in_file = h5py.File(filename, "r")
if "info" in in_file.attrs:
only_on_root(mylog.info, in_file.attrs["info"])
if default_filename and \
in_file.attrs["version"] < xray_data_version:
raise ObsoleteDataException()
else:
only_on_root(mylog.info, "X-ray emissivity data version: %s." % \
in_file.attrs["version"])
for field in [
"emissivity_primordial", "emissivity_metals", "log_nH",
"log_T", "log_E"
]:
if field in in_file:
setattr(self, field, in_file[field][:])
in_file.close()
E_diff = np.diff(self.log_E)
self.E_bins = \
YTArray(np.power(10, np.concatenate([self.log_E[:-1] - 0.5 * E_diff,
[self.log_E[-1] - 0.5 * E_diff[-1],
self.log_E[-1] + 0.5 * E_diff[-1]]])),
"keV")
self.dnu = (np.diff(self.E_bins) / hcgs).in_units("Hz")
def __init__(self, filename, dataset_type="gadget_binary",
additional_fields=(),
unit_base=None,
index_order=None,
index_filename=None,
kdtree_filename=None,
kernel_name=None,
bounding_box = None,
header_spec = "default",
field_spec = "default",
ptype_spec = "default",
long_ids = False,
units_override=None,
mean_molecular_weight=None,
header_offset = 0,
unit_system="cgs",
use_dark_factor = False,
w_0 = -1.0,
w_a = 0.0):
if self._instantiated:
return
# Check if filename is a directory
if os.path.isdir(filename):
# Get the .0 snapshot file. We know there's only 1 and it's valid since we
# came through _is_valid in load()
for f in os.listdir(filename):
fname = os.path.join(filename, f)
if ('.0' in f) and ('.ewah' not in f) and os.path.isfile(fname):
filename = os.path.join(filename, f)
break
self._header = GadgetBinaryHeader(filename, header_spec)
header_size = self._header.size
if header_size != [256]:
only_on_root(
mylog.warn,
"Non-standard header size is detected! "
"Gadget-2 standard header is 256 bytes, but yours is %s. "
"Make sure a non-standard header is actually expected. "
"Otherwise something is wrong, "
"and you might want to check how the dataset is loaded. "
"Futher information about header specification can be found in "
"https://yt-project.org/docs/dev/examining/loading_data.html#header-specification.",
header_size
)
self._field_spec = self._setup_binary_spec(
field_spec, gadget_field_specs)
self._ptype_spec = self._setup_binary_spec(
ptype_spec, gadget_ptype_specs)
self.storage_filename = None
if long_ids:
self._id_dtype = 'u8'
else:
self._id_dtype = 'u4'
self.long_ids = long_ids
self.header_offset = header_offset
if unit_base is not None and "UnitLength_in_cm" in unit_base:
# We assume this is comoving, because in the absence of comoving
# integration the redshift will be zero.
unit_base['cmcm'] = 1.0 / unit_base["UnitLength_in_cm"]
self._unit_base = unit_base
if bounding_box is not None:
# This ensures that we know a bounding box has been applied
self._domain_override = True
bbox = np.array(bounding_box, dtype="float64")
if bbox.shape == (2, 3):
bbox = bbox.transpose()
self.domain_left_edge = bbox[:, 0]
self.domain_right_edge = bbox[:, 1]
else:
self.domain_left_edge = self.domain_right_edge = None
if units_override is not None:
raise RuntimeError("units_override is not supported for GadgetDataset. " +
"Use unit_base instead.")
# Set dark energy parameters before cosmology object is created
self.use_dark_factor = use_dark_factor
self.w_0 = w_0
self.w_a = w_a
super(GadgetDataset, self).__init__(
filename, dataset_type=dataset_type,
unit_system=unit_system,
index_order=index_order,
index_filename=index_filename,
kdtree_filename=kdtree_filename,
kernel_name=kernel_name)
if self.cosmological_simulation:
self.time_unit.convert_to_units('s/h')
self.length_unit.convert_to_units('kpccm/h')
self.mass_unit.convert_to_units('g/h')
else:
self.time_unit.convert_to_units('s')
self.length_unit.convert_to_units('kpc')
self.mass_unit.convert_to_units('Msun')
if mean_molecular_weight is None:
self.mu = default_mu
else:
self.mu = mean_molecular_weight
def _set_code_unit_attributes(self):
# If no units passed in by user, set a sane default (Gadget-2 users
# guide).
if self._unit_base is None:
if self.cosmological_simulation == 1:
only_on_root(
mylog.info, "Assuming length units are in kpc/h (comoving)")
self._unit_base = dict(length=(1.0, "kpccm/h"))
else:
only_on_root(
mylog.info, "Assuming length units are in kpc (physical)")
self._unit_base = dict(length=(1.0, "kpc"))
# If units passed in by user, decide what to do about
# co-moving and factors of h
unit_base = self._unit_base or {}
if "length" in unit_base:
length_unit = unit_base["length"]
elif "UnitLength_in_cm" in unit_base:
if self.cosmological_simulation == 0:
length_unit = (unit_base["UnitLength_in_cm"], "cm")
else:
length_unit = (unit_base["UnitLength_in_cm"], "cmcm/h")
else:
raise RuntimeError
length_unit = _fix_unit_ordering(length_unit)
self.length_unit = self.quan(length_unit[0], length_unit[1])
unit_base = self._unit_base or {}
if self.cosmological_simulation:
# see http://www.mpa-garching.mpg.de/gadget/gadget-list/0113.html
# for why we need to include a factor of square root of the
# scale factor
vel_units = "cm/s * sqrt(a)"
else:
vel_units = "cm/s"
if "velocity" in unit_base:
velocity_unit = unit_base["velocity"]
elif "UnitVelocity_in_cm_per_s" in unit_base:
velocity_unit = (unit_base["UnitVelocity_in_cm_per_s"], vel_units)
else:
velocity_unit = (1e5, vel_units)
velocity_unit = _fix_unit_ordering(velocity_unit)
self.velocity_unit = self.quan(velocity_unit[0], velocity_unit[1])
# We set hubble_constant = 1.0 for non-cosmology, so this is safe.
# Default to 1e10 Msun/h if mass is not specified.
if "mass" in unit_base:
mass_unit = unit_base["mass"]
elif "UnitMass_in_g" in unit_base:
if self.cosmological_simulation == 0:
mass_unit = (unit_base["UnitMass_in_g"], "g")
else:
mass_unit = (unit_base["UnitMass_in_g"], "g/h")
else:
# Sane default
mass_unit = (1e10, "Msun/h")
mass_unit = _fix_unit_ordering(mass_unit)
self.mass_unit = self.quan(mass_unit[0], mass_unit[1])
if self.cosmological_simulation:
# self.velocity_unit is the unit to rescale on-disk velocities, The
# actual internal velocity unit is really in comoving units
# since the time unit is derived from the internal velocity unit, we
# infer the internal velocity unit here and name it vel_unit
#
# see http://www.mpa-garching.mpg.de/gadget/gadget-list/0113.html
if 'velocity' in unit_base:
vel_unit = unit_base['velocity']
elif "UnitVelocity_in_cm_per_s" in unit_base:
vel_unit = (unit_base['UnitVelocity_in_cm_per_s'], 'cmcm/s')
else:
vel_unit = (1, 'kmcm/s')
vel_unit = self.quan(*vel_unit)
else:
vel_unit = self.velocity_unit
self.time_unit = self.length_unit / vel_unit
if "specific_energy" in unit_base:
specific_energy_unit = unit_base["specific_energy"]
elif "UnitEnergy_in_cgs" in unit_base and "UnitMass_in_g" in unit_base:
specific_energy_unit = \
unit_base["UnitEnergy_in_cgs"] / unit_base["UnitMass_in_g"]
specific_energy_unit = (specific_energy_unit, "(cm/s)**2")
else:
# Sane default
specific_energy_unit = (1, "(km/s) ** 2")
specific_energy_unit = _fix_unit_ordering(specific_energy_unit)
self.specific_energy_unit = self.quan(*specific_energy_unit)
def __init__(self,
filename,
dataset_type="gadget_binary",
additional_fields=(),
unit_base=None,
n_ref=64,
over_refine_factor=1,
kernel_name=None,
index_ptype="all",
bounding_box=None,
header_spec="default",
field_spec="default",
ptype_spec="default",
units_override=None,
unit_system="cgs",
use_dark_factor=False,
w_0=-1.0,
w_a=0.0):
if self._instantiated:
return
self._header = GadgetBinaryHeader(filename, header_spec)
header_size = self._header.size
if header_size != [256]:
only_on_root(
mylog.warn, "Non-standard header size is detected! "
"Gadget-2 standard header is 256 bytes, but yours is %s. "
"Make sure a non-standard header is actually expected. "
"Otherwise something is wrong, "
"and you might want to check how the dataset is loaded. "
"Futher information about header specification can be found in "
"https://yt-project.org/docs/dev/examining/loading_data.html#header-specification.",
header_size)
self._field_spec = self._setup_binary_spec(field_spec,
gadget_field_specs)
self._ptype_spec = self._setup_binary_spec(ptype_spec,
gadget_ptype_specs)
self.index_ptype = index_ptype
self.storage_filename = None
if unit_base is not None and "UnitLength_in_cm" in unit_base:
# We assume this is comoving, because in the absence of comoving
# integration the redshift will be zero.
unit_base['cmcm'] = 1.0 / unit_base["UnitLength_in_cm"]
self._unit_base = unit_base
if bounding_box is not None:
bbox = np.array(bounding_box, dtype="float64")
if bbox.shape == (2, 3):
bbox = bbox.transpose()
self.domain_left_edge = bbox[:, 0]
self.domain_right_edge = bbox[:, 1]
else:
self.domain_left_edge = self.domain_right_edge = None
if units_override is not None:
raise RuntimeError(
"units_override is not supported for GadgetDataset. " +
"Use unit_base instead.")
# Set dark energy parameters before cosmology object is created
self.use_dark_factor = use_dark_factor
self.w_0 = w_0
self.w_a = w_a
super(GadgetDataset,
self).__init__(filename,
dataset_type=dataset_type,
unit_system=unit_system,
n_ref=n_ref,
over_refine_factor=over_refine_factor,
kernel_name=kernel_name)
if self.cosmological_simulation:
self.time_unit.convert_to_units('s/h')
self.length_unit.convert_to_units('kpccm/h')
self.mass_unit.convert_to_units('g/h')
else:
self.time_unit.convert_to_units('s')
self.length_unit.convert_to_units('kpc')
self.mass_unit.convert_to_units('Msun')
def project_light_cone(self, field_of_view, image_resolution, field,
weight_field=None, photon_field=False,
save_stack=True, save_final_image=True,
save_slice_images=False,
cmap_name="algae",
njobs=1, dynamic=False):
r"""Create projections for light cone, then add them together.
Parameters
----------
field_of_view : YTQuantity or tuple of (float, str)
The field of view of the image and the units.
image_resolution : YTQuantity or tuple of (float, str)
The size of each image pixel and the units.
field : string
The projected field.
weight_field : string
the weight field of the projection. This has the same meaning as
in standard projections.
Default: None.
photon_field : bool
if True, the projection data for each slice is decremented by 4 Pi
R^2`, where R is the luminosity distance between the observer and
the slice redshift.
Default: False.
save_stack : bool
if True, the light cone data including each individual
slice is written to an hdf5 file.
Default: True.
save_final_image : bool
if True, save an image of the final light cone projection.
Default: True.
save_slice_images : bool
save images for each individual projection slice.
Default: False.
cmap_name : string
color map for images.
Default: "algae".
njobs : int
The number of parallel jobs over which the light cone projection
will be split. Choose -1 for one processor per individual
projection and 1 to have all processors work together on each
projection.
Default: 1.
dynamic : bool
If True, use dynamic load balancing to create the projections.
Default: False.
"""
if isinstance(field_of_view, tuple) and len(field_of_view) == 2:
field_of_view = self.simulation.quan(field_of_view[0],
field_of_view[1])
elif not isinstance(field_of_view, YTArray):
raise RuntimeError("field_of_view argument must be either a YTQauntity " +
"or a tuple of type (float, str).")
if isinstance(image_resolution, tuple) and len(image_resolution) == 2:
image_resolution = self.simulation.quan(image_resolution[0],
image_resolution[1])
elif not isinstance(image_resolution, YTArray):
raise RuntimeError("image_resolution argument must be either a YTQauntity " +
"or a tuple of type (float, str).")
# Calculate number of pixels on a side.
pixels = (field_of_view / image_resolution).in_units("")
# Clear projection stack.
projection_stack = []
projection_weight_stack = []
if "object" in self.light_cone_solution[-1]:
del self.light_cone_solution[-1]["object"]
# for q, output in enumerate(self.light_cone_solution):
all_storage = {}
for my_storage, output in parallel_objects(self.light_cone_solution,
storage=all_storage,
dynamic=dynamic):
output["object"] = load(output["filename"])
output["object"].parameters.update(self.set_parameters)
# Calculate fraction of box required for width corresponding to
# requested image size.
proper_box_size = self.simulation.box_size / \
(1.0 + output["redshift"])
output["box_width_fraction"] = (output["box_width_per_angle"] *
field_of_view).in_units("")
frb = _light_cone_projection(output, field, pixels,
weight_field=weight_field)
if photon_field:
# Decrement the flux by the luminosity distance.
# Assume field in frb is in erg/s/cm^2/Hz
dL = self.cosmology.luminosity_distance(self.observer_redshift,
output["redshift"])
proper_box_size = self.simulation.box_size / \
(1.0 + output["redshift"])
pixel_area = (proper_box_size.in_cgs() / pixels)**2 #in proper cm^2
factor = pixel_area / (4.0 * np.pi * dL.in_cgs()**2)
mylog.info("Distance to slice = %s" % dL)
frb[field] *= factor #in erg/s/cm^2/Hz on observer"s image plane.
if weight_field is None:
my_storage.result = {"field": frb[field]}
else:
my_storage.result = {"field": (frb[field] *
frb["weight_field"]),
"weight_field": frb["weight_field"]}
del output["object"]
# Combine results from each slice.
all_slices = list(all_storage.keys())
all_slices.sort()
for my_slice in all_slices:
if save_slice_images:
name = os.path.join(self.output_dir,
"%s_%04d_%04d" %
(self.output_prefix,
my_slice, len(self.light_cone_solution)))
if weight_field is None:
my_image = all_storage[my_slice]["field"]
else:
my_image = all_storage[my_slice]["field"] / \
all_storage[my_slice]["weight_field"]
only_on_root(write_image, np.log10(my_image),
"%s_%s.png" % (name, field), cmap_name=cmap_name)
projection_stack.append(all_storage[my_slice]["field"])
if weight_field is not None:
projection_weight_stack.append(all_storage[my_slice]["field"])
projection_stack = self.simulation.arr(projection_stack)
projection_weight_stack = self.simulation.arr(projection_weight_stack)
# Add up slices to make light cone projection.
if (weight_field is None):
light_cone_projection = projection_stack.sum(axis=0)
else:
light_cone_projection = \
projection_stack.sum(axis=0) / \
self.simulation.arr(projection_weight_stack).sum(axis=0)
filename = os.path.join(self.output_dir, self.output_prefix)
# Write image.
if save_final_image:
only_on_root(write_image, np.log10(light_cone_projection),
"%s_%s.png" % (filename, field), cmap_name=cmap_name)
# Write stack to hdf5 file.
if save_stack:
self._save_light_cone_stack(field, weight_field,
projection_stack, projection_weight_stack,
filename=filename,
attrs={"field_of_view": str(field_of_view),
"image_resolution": str(image_resolution)})
def project_light_cone(self,
field_of_view,
image_resolution,
field,
weight_field=None,
photon_field=False,
save_stack=True,
save_final_image=True,
save_slice_images=False,
cmap_name=None,
njobs=1,
dynamic=False):
r"""Create projections for light cone, then add them together.
Parameters
----------
field_of_view : YTQuantity or tuple of (float, str)
The field of view of the image and the units.
image_resolution : YTQuantity or tuple of (float, str)
The size of each image pixel and the units.
field : string
The projected field.
weight_field : string
the weight field of the projection. This has the same meaning as
in standard projections.
Default: None.
photon_field : bool
if True, the projection data for each slice is decremented by 4 Pi
R^2`, where R is the luminosity distance between the observer and
the slice redshift.
Default: False.
save_stack : bool
if True, the light cone data including each individual
slice is written to an hdf5 file.
Default: True.
save_final_image : bool
if True, save an image of the final light cone projection.
Default: True.
save_slice_images : bool
save images for each individual projection slice.
Default: False.
cmap_name : string
color map for images.
Default: your default colormap.
njobs : int
The number of parallel jobs over which the light cone projection
will be split. Choose -1 for one processor per individual
projection and 1 to have all processors work together on each
projection.
Default: 1.
dynamic : bool
If True, use dynamic load balancing to create the projections.
Default: False.
"""
if cmap_name is None:
cmap_name = ytcfg.get("yt", "default_colormap")
if isinstance(field_of_view, tuple) and len(field_of_view) == 2:
field_of_view = self.simulation.quan(field_of_view[0],
field_of_view[1])
elif not isinstance(field_of_view, YTArray):
raise RuntimeError(
"field_of_view argument must be either a YTQuantity " +
"or a tuple of type (float, str).")
if isinstance(image_resolution, tuple) and len(image_resolution) == 2:
image_resolution = self.simulation.quan(image_resolution[0],
image_resolution[1])
elif not isinstance(image_resolution, YTArray):
raise RuntimeError(
"image_resolution argument must be either a YTQuantity " +
"or a tuple of type (float, str).")
# Calculate number of pixels on a side.
pixels = int((field_of_view / image_resolution).in_units(""))
# Clear projection stack.
projection_stack = []
projection_weight_stack = []
if "object" in self.light_cone_solution[-1]:
del self.light_cone_solution[-1]["object"]
# for q, output in enumerate(self.light_cone_solution):
all_storage = {}
for my_storage, output in parallel_objects(self.light_cone_solution,
storage=all_storage,
dynamic=dynamic):
output["object"] = load(output["filename"])
output["object"].parameters.update(self.set_parameters)
# Calculate fraction of box required for width corresponding to
# requested image size.
proper_box_size = self.simulation.box_size / \
(1.0 + output["redshift"])
output["box_width_fraction"] = (output["box_width_per_angle"] *
field_of_view).in_units("")
frb = _light_cone_projection(output,
field,
pixels,
weight_field=weight_field)
if photon_field:
# Decrement the flux by the luminosity distance.
# Assume field in frb is in erg/s/cm^2/Hz
dL = self.cosmology.luminosity_distance(
self.observer_redshift, output["redshift"])
proper_box_size = self.simulation.box_size / \
(1.0 + output["redshift"])
pixel_area = (proper_box_size.in_cgs() /
pixels)**2 #in proper cm^2
factor = pixel_area / (4.0 * np.pi * dL.in_cgs()**2)
mylog.info("Distance to slice = %s" % dL)
frb[field] *= factor #in erg/s/cm^2/Hz on observer"s image plane.
if weight_field is None:
my_storage.result = {"field": frb[field]}
else:
my_storage.result = {
"field": (frb[field] * frb["weight_field"]),
"weight_field": frb["weight_field"]
}
del output["object"]
# Combine results from each slice.
all_slices = list(all_storage.keys())
all_slices.sort()
for my_slice in all_slices:
if save_slice_images:
name = os.path.join(
self.output_dir,
"%s_%04d_%04d" % (self.output_prefix, my_slice,
len(self.light_cone_solution)))
if weight_field is None:
my_image = all_storage[my_slice]["field"]
else:
my_image = all_storage[my_slice]["field"] / \
all_storage[my_slice]["weight_field"]
only_on_root(write_image,
np.log10(my_image),
"%s_%s.png" % (name, field),
cmap_name=cmap_name)
projection_stack.append(all_storage[my_slice]["field"])
if weight_field is not None:
projection_weight_stack.append(all_storage[my_slice]["field"])
projection_stack = self.simulation.arr(projection_stack)
projection_weight_stack = self.simulation.arr(projection_weight_stack)
# Add up slices to make light cone projection.
if (weight_field is None):
light_cone_projection = projection_stack.sum(axis=0)
else:
light_cone_projection = \
projection_stack.sum(axis=0) / \
self.simulation.arr(projection_weight_stack).sum(axis=0)
filename = os.path.join(self.output_dir, self.output_prefix)
# Write image.
if save_final_image:
only_on_root(write_image,
np.log10(light_cone_projection),
"%s_%s.png" % (filename, field),
cmap_name=cmap_name)
# Write stack to hdf5 file.
if save_stack:
self._save_light_cone_stack(field,
weight_field,
projection_stack,
projection_weight_stack,
filename=filename,
attrs={
"field_of_view":
str(field_of_view),
"image_resolution":
str(image_resolution)
})
def _initialize_index(self):
ds = self.dataset
only_on_root(
mylog.info,
"Allocating for %0.3e particles",
self.total_particles,
global_rootonly=True,
)
# if we have not yet set domain_left_edge and domain_right_edge then do
# an I/O pass over the particle coordinates to determine a bounding box
if self.ds.domain_left_edge is None:
min_ppos = np.empty(3, dtype="float64")
min_ppos[:] = np.nan
max_ppos = np.empty(3, dtype="float64")
max_ppos[:] = np.nan
only_on_root(
mylog.info,
"Bounding box cannot be inferred from metadata, reading "
"particle positions to infer bounding box",
)
for df in self.data_files:
for _, ppos in self.io._yield_coordinates(df):
min_ppos = np.nanmin(np.vstack([min_ppos, ppos]), axis=0)
max_ppos = np.nanmax(np.vstack([max_ppos, ppos]), axis=0)
only_on_root(
mylog.info,
"Load this dataset with bounding_box=[%s, %s] to avoid I/O "
"overhead from inferring bounding_box." % (min_ppos, max_ppos),
)
ds.domain_left_edge = ds.arr(1.05 * min_ppos, "code_length")
ds.domain_right_edge = ds.arr(1.05 * max_ppos, "code_length")
ds.domain_width = ds.domain_right_edge - ds.domain_left_edge
# use a trivial morton index for datasets containing a single chunk
if len(self.data_files) == 1:
order1 = 1
order2 = 1
else:
order1 = ds.index_order[0]
order2 = ds.index_order[1]
if order1 == 1 and order2 == 1:
dont_cache = True
else:
dont_cache = False
# If we have applied a bounding box then we can't cache the
# ParticleBitmap because it is doman dependent
if getattr(ds, "_domain_override", False):
dont_cache = True
if not hasattr(self.ds, "_file_hash"):
self.ds._file_hash = self._generate_hash()
self.regions = ParticleBitmap(
ds.domain_left_edge,
ds.domain_right_edge,
ds.periodicity,
self.ds._file_hash,
len(self.data_files),
index_order1=order1,
index_order2=order2,
)
# Load Morton index from file if provided
if getattr(ds, "index_filename", None) is None:
fname = ds.parameter_filename + ".index{}_{}.ewah".format(
self.regions.index_order1, self.regions.index_order2
)
else:
fname = ds.index_filename
dont_load = dont_cache and not hasattr(ds, "index_filename")
try:
if dont_load:
raise OSError
rflag = self.regions.load_bitmasks(fname)
rflag = self.regions.check_bitmasks()
self._initialize_frontend_specific()
if rflag == 0:
raise OSError
except (OSError, struct.error):
self.regions.reset_bitmasks()
self._initialize_coarse_index()
self._initialize_refined_index()
wdir = os.path.dirname(fname)
if not dont_cache and os.access(wdir, os.W_OK):
# Sometimes os mis-reports whether a directory is writable,
# So pass if writing the bitmask file fails.
try:
self.regions.save_bitmasks(fname)
except OSError:
pass
rflag = self.regions.check_bitmasks()
def _parse_parameter_file(self):
hvals = self._get_hvals()
self.dimensionality = 3
self.refine_by = 2
self.parameters["HydroMethod"] = "sph"
# Set standard values
# We may have an overridden bounding box.
if self.domain_left_edge is None and hvals["BoxSize"] != 0:
self.domain_left_edge = np.zeros(3, "float64")
self.domain_right_edge = np.ones(3, "float64") * hvals["BoxSize"]
self.domain_dimensions = np.ones(3, "int32")
self.periodicity = (True, True, True)
self.cosmological_simulation = 1
try:
self.current_redshift = hvals["Redshift"]
except KeyError:
# Probably not a cosmological dataset, we should just set
# z = 0 and let the user know
self.current_redshift = 0.0
only_on_root(mylog.info, "Redshift is not set in Header. Assuming z=0.")
try:
self.omega_lambda = hvals["OmegaLambda"]
self.omega_matter = hvals["Omega0"]
self.hubble_constant = hvals["HubbleParam"]
except KeyError:
# If these are not set it is definitely not a cosmological dataset.
self.omega_lambda = 0.0
self.omega_matter = 1.0 # Just in case somebody asks for it.
# Hubble is set below for Omega Lambda = 0.
# According to the Gadget manual, OmegaLambda will be zero for
# non-cosmological datasets. However, it may be the case that
# individuals are running cosmological simulations *without* Lambda, in
# which case we may be doing something incorrect here.
# It may be possible to deduce whether ComovingIntegration is on
# somehow, but opinions on this vary.
if self.omega_lambda == 0.0:
only_on_root(
mylog.info, "Omega Lambda is 0.0, so we are turning off Cosmology."
)
self.hubble_constant = 1.0 # So that scaling comes out correct
self.cosmological_simulation = 0
self.current_redshift = 0.0
# This may not be correct.
self.current_time = hvals["Time"]
else:
# Now we calculate our time based on the cosmology, because in
# ComovingIntegration hvals["Time"] will in fact be the expansion
# factor, not the actual integration time, so we re-calculate
# global time from our Cosmology.
cosmo = Cosmology(
hubble_constant=self.hubble_constant,
omega_matter=self.omega_matter,
omega_lambda=self.omega_lambda,
)
self.current_time = cosmo.lookback_time(self.current_redshift, 1e6)
only_on_root(
mylog.info,
"Calculating time from %0.3e to be %0.3e seconds",
hvals["Time"],
self.current_time,
)
self.parameters = hvals
prefix = os.path.abspath(
os.path.join(
os.path.dirname(self.parameter_filename),
os.path.basename(self.parameter_filename).split(".", 1)[0],
)
)
if hvals["NumFiles"] > 1:
self.filename_template = f"{prefix}.%(num)s{self._suffix}"
else:
self.filename_template = self.parameter_filename
self.file_count = hvals["NumFiles"]
