def _read_calibration_file(path):
"""
Read the correction from hdf5 calibration file
"""
mon = MonitoringContainer()
with tables.open_file(path) as f:
tel_ids = [
int(key[4:]) for key in f.root._v_children.keys()
if key.startswith('tel_')
]
for tel_id in tel_ids:
with HDF5TableReader(path) as h5_table:
base = f'/tel_{tel_id}'
# read the calibration data
table = base + '/calibration'
next(h5_table.read(table, mon.tel[tel_id].calibration))
# read pedestal data
table = base + '/pedestal'
next(h5_table.read(table, mon.tel[tel_id].pedestal))
# read flat-field data
table = base + '/flatfield'
next(h5_table.read(table, mon.tel[tel_id].flatfield))
# read the pixel_status container
table = base + '/pixel_status'
next(h5_table.read(table, mon.tel[tel_id].pixel_status))
return mon
def read_simu_info_hdf5(filename):
"""
Read simu info from an hdf5 file
Returns
-------
`ctapipe.containers.MCHeaderContainer`
"""
with HDF5TableReader(filename) as reader:
mcheader = reader.read('/simulation/run_config', MCHeaderContainer())
mc = next(mcheader)
return mc
def read_simtel_energy_histogram(filename):
"""
Read the simtel energy histogram from a HDF5 file.
Parameters
----------
filename: path
Returns
-------
`lstchain.io.lstcontainers.ThrownEventsHistogram`
"""
with HDF5TableReader(filename=filename) as reader:
histtab = reader.read('/simulation/thrown_event_distribution', ThrownEventsHistogram())
hist = next(histtab)
return hist
def read_simu_info_hdf5(filename):
"""
Read simu info from an hdf5 file
Parameters
----------
filename: str
path to the HDF5 file
Returns
-------
`ctapipe.containers.SimulationConfigContainer`
"""
with HDF5TableReader(filename) as reader:
mc_reader = reader.read("/simulation/run_config", SimulationConfigContainer())
mc = next(mc_reader)
return mc
def read_simtel_energy_histogram_merged(filename):
"""
Read the simtel energy histogram from merged HDF5 file.
"""
with HDF5TableReader(filename=filename) as reader:
histtab = reader.read('/simulation/thrown_event_distribution', ThrownEventsHistogram())
hh = ThrownEventsHistogram()
for i in range(0, 100000):
try:
hist = next(histtab)
if i == 0:
hh.histogram = hist.histogram
else:
hh.histogram = hist.histogram + hh.histogram
#print(hist.obs_id, i)
except StopIteration:
print(hist.num_entries * i, np.sum(np.sum(hh.histogram, axis=1)))
break;
return hist, hh
def _parse_mc_headers(self):
"""
Construct a dict of SimulationConfigContainers from the
self.file_.root.configuration.simulation.run.
These are used to match the correct header to each event
"""
# Just returning next(reader) would work as long as there are no merged files
# The reader ignores obs_id making the setup somewhat tricky
# This is ugly but supports multiple headers so each event can have
# the correct mcheader assigned by matching the obs_id
# Alternatively this becomes a flat list
# and the obs_id matching part needs to be done in _generate_events()
mc_headers = {}
if "simulation" in self.file_.root.configuration:
reader = HDF5TableReader(self.file_).read(
"/configuration/simulation/run", SimulationConfigContainer()
)
row_iterator = self.file_.root.configuration.simulation.run.iterrows()
for row in row_iterator:
mc_headers[row["obs_id"]] = next(reader)
return mc_headers
def _parse_simulation_configs(self):
"""
Construct a dict of SimulationConfigContainers from the
self.file_.root.configuration.simulation.run.
These are used to match the correct header to each event
"""
# Just returning next(reader) would work as long as there are no merged files
# The reader ignores obs_id making the setup somewhat tricky
# This is ugly but supports multiple headers so each event can have
# the correct mcheader assigned by matching the obs_id
# Alternatively this becomes a flat list
# and the obs_id matching part needs to be done in _generate_events()
simulation_configs = {}
if "simulation" in self.file_.root.configuration:
reader = HDF5TableReader(self.file_).read(
"/configuration/simulation/run",
containers=[
SimulationConfigContainer(),
EventIndexContainer()
],
)
for (config, index) in reader:
simulation_configs[index.obs_id] = config
return simulation_configs
def _generate_events(self):
"""
Yield ArrayEventContainer to iterate through events.
"""
data = ArrayEventContainer()
# Maybe take some other metadata, but there are still some 'unknown'
# written out by the stage1 tool
data.meta["origin"] = self.file_.root._v_attrs["CTA PROCESS TYPE"]
data.meta["input_url"] = self.input_url
data.meta["max_events"] = self.max_events
if DataLevel.DL1_IMAGES in self.datalevels:
image_iterators = {
tel.name: self.file_.root.dl1.event.telescope.images[
tel.name
].iterrows()
for tel in self.file_.root.dl1.event.telescope.images
}
if self.has_simulated_dl1:
simulated_image_iterators = {
tel.name: self.file_.root.simulation.event.telescope.images[
tel.name
].iterrows()
for tel in self.file_.root.simulation.event.telescope.images
}
if DataLevel.DL1_PARAMETERS in self.datalevels:
param_readers = {
tel.name: HDF5TableReader(self.file_).read(
f"/dl1/event/telescope/parameters/{tel.name}",
containers=[
HillasParametersContainer(),
TimingParametersContainer(),
LeakageContainer(),
ConcentrationContainer(),
MorphologyContainer(),
IntensityStatisticsContainer(),
PeakTimeStatisticsContainer(),
],
prefixes=True,
)
for tel in self.file_.root.dl1.event.telescope.parameters
}
if self.has_simulated_dl1:
simulated_param_readers = {
tel.name: HDF5TableReader(self.file_).read(
f"/simulation/event/telescope/parameters/{tel.name}",
containers=[
HillasParametersContainer(),
LeakageContainer(),
ConcentrationContainer(),
MorphologyContainer(),
IntensityStatisticsContainer(),
],
prefixes=True,
)
for tel in self.file_.root.dl1.event.telescope.parameters
}
if self.is_simulation:
# simulated shower wide information
mc_shower_reader = HDF5TableReader(self.file_).read(
"/simulation/event/subarray/shower",
SimulatedShowerContainer(),
prefixes="true",
)
# Setup iterators for the array events
events = HDF5TableReader(self.file_).read(
"/dl1/event/subarray/trigger", [TriggerContainer(), EventIndexContainer()]
)
array_pointing_finder = IndexFinder(
self.file_.root.dl1.monitoring.subarray.pointing.col("time")
)
tel_pointing_finder = {
tel.name: IndexFinder(tel.col("time"))
for tel in self.file_.root.dl1.monitoring.telescope.pointing
}
for counter, array_event in enumerate(events):
data.dl1.tel.clear()
data.simulation.tel.clear()
data.pointing.tel.clear()
data.trigger.tel.clear()
data.count = counter
data.trigger, data.index = next(events)
data.trigger.tels_with_trigger = (
np.where(data.trigger.tels_with_trigger)[0] + 1
) # +1 to match array index to telescope id
# Maybe there is a simpler way to do this
# Beware: tels_with_trigger contains all triggered telescopes whereas
# the telescope trigger table contains only the subset of
# allowed_tels given during the creation of the dl1 file
for i in self.file_.root.dl1.event.telescope.trigger.where(
f"(obs_id=={data.index.obs_id}) & (event_id=={data.index.event_id})"
):
if self.allowed_tels and i["tel_id"] not in self.allowed_tels:
continue
if self.datamodel_version == "v1.0.0":
data.trigger.tel[i["tel_id"]].time = i["telescopetrigger_time"]
else:
data.trigger.tel[i["tel_id"]].time = i["time"]
self._fill_array_pointing(data, array_pointing_finder)
self._fill_telescope_pointing(data, tel_pointing_finder)
if self.is_simulation:
data.simulation.shower = next(mc_shower_reader)
for tel in data.trigger.tel.keys():
if self.allowed_tels and tel not in self.allowed_tels:
continue
if self.has_simulated_dl1:
simulated = data.simulation.tel[tel]
dl1 = data.dl1.tel[tel]
if DataLevel.DL1_IMAGES in self.datalevels:
if f"tel_{tel:03d}" not in image_iterators.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the image table."
)
continue
image_row = next(image_iterators[f"tel_{tel:03d}"])
dl1.image = image_row["image"]
dl1.peak_time = image_row["peak_time"]
dl1.image_mask = image_row["image_mask"]
if self.has_simulated_dl1:
if f"tel_{tel:03d}" not in simulated_image_iterators.keys():
logger.warning(
f"Triggered telescope {tel} is missing "
"from the simulated image table, but was present at the "
"reconstructed image table."
)
continue
simulated_image_row = next(
simulated_image_iterators[f"tel_{tel:03d}"]
)
simulated.true_image = simulated_image_row["true_image"]
if DataLevel.DL1_PARAMETERS in self.datalevels:
if f"tel_{tel:03d}" not in param_readers.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the parameters table."
)
continue
# Is there a smarter way to unpack this?
# Best would probbaly be if we could directly read
# into the ImageParametersContainer
params = next(param_readers[f"tel_{tel:03d}"])
dl1.parameters.hillas = params[0]
dl1.parameters.timing = params[1]
dl1.parameters.leakage = params[2]
dl1.parameters.concentration = params[3]
dl1.parameters.morphology = params[4]
dl1.parameters.intensity_statistics = params[5]
dl1.parameters.peak_time_statistics = params[6]
if self.has_simulated_dl1:
if f"tel_{tel:03d}" not in param_readers.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the simulated parameters table, but was "
"present at the reconstructed parameters table."
)
continue
simulated_params = next(
simulated_param_readers[f"tel_{tel:03d}"]
)
simulated.true_parameters.hillas = simulated_params[0]
simulated.true_parameters.leakage = simulated_params[1]
simulated.true_parameters.concentration = simulated_params[2]
simulated.true_parameters.morphology = simulated_params[3]
simulated.true_parameters.intensity_statistics = simulated_params[
4
]
yield data
