def test_h5_file(tmp_path_factory):
"""Test hdf5 file with some tables for the reader tests"""
path = tmp_path_factory.mktemp("hdf5") / "test.h5"
r0 = R0CameraContainer()
shower = SimulatedShowerContainer()
r0.waveform = np.random.uniform(size=(50, 10))
r0.meta["test_attribute"] = 3.14159
r0.meta["date"] = "2020-10-10"
with HDF5TableWriter(path,
group_name="R0",
filters=tables.Filters(complevel=7)) as writer:
for _ in range(100):
r0.waveform[:] = np.random.uniform(size=(50, 10))
shower.energy = 10**np.random.uniform(1, 2) * u.TeV
shower.core_x = np.random.uniform(-1, 1) * u.m
shower.core_y = np.random.uniform(-1, 1) * u.m
writer.write("tel_001", r0)
writer.write("tel_002", r0) # write a second table too
writer.write("sim_shower", shower)
return path
def test_read_container(temp_h5_file):
r0tel1 = R0CameraContainer()
r0tel2 = R0CameraContainer()
sim_shower = SimulatedShowerContainer()
with HDF5TableReader(temp_h5_file) as reader:
# get the generators for each table
# test supplying a single container as well as an
# iterable with one entry only
simtab = reader.read("/R0/sim_shower", (sim_shower, ))
r0tab1 = reader.read("/R0/tel_001", r0tel1)
r0tab2 = reader.read("/R0/tel_002", r0tel2)
# read all 3 tables in sync
for ii in range(3):
m = next(simtab)[0]
r0_1 = next(r0tab1)
r0_2 = next(r0tab2)
print("sim_shower:", m)
print("t0:", r0_1.waveform)
print("t1:", r0_2.waveform)
print("---------------------------")
assert "test_attribute" in r0_1.meta
assert r0_1.meta["date"] == "2020-10-10"
def test_read_whole_table(temp_h5_file):
sim_shower = SimulatedShowerContainer()
with HDF5TableReader(temp_h5_file) as reader:
for cont in reader.read("/R0/sim_shower", sim_shower):
print(cont)
def test_with_context_reader(temp_h5_file):
sim_shower = SimulatedShowerContainer()
with HDF5TableReader(temp_h5_file) as h5_table:
assert h5_table._h5file.isopen == 1
for cont in h5_table.read("/R0/sim_shower", sim_shower):
print(cont)
assert h5_table._h5file.isopen == 0
def test_write_container(temp_h5_file):
r0tel = R0CameraContainer()
simshower = SimulatedShowerContainer()
simshower.reset()
r0tel.waveform = np.random.uniform(size=(50, 10))
r0tel.meta["test_attribute"] = 3.14159
r0tel.meta["date"] = "2020-10-10"
with HDF5TableWriter(temp_h5_file,
group_name="R0",
filters=tables.Filters(complevel=7)) as writer:
for ii in range(100):
r0tel.waveform[:] = np.random.uniform(size=(50, 10))
simshower.energy = 10**np.random.uniform(1, 2) * u.TeV
simshower.core_x = np.random.uniform(-1, 1) * u.m
simshower.core_y = np.random.uniform(-1, 1) * u.m
writer.write("tel_001", r0tel)
writer.write("tel_002", r0tel) # write a second table too
writer.write("sim_shower", simshower)
def _generate_events(self):
"""
Yield ArrayEventContainer to iterate through events.
"""
data = ArrayEventContainer()
# Maybe take some other metadata, but there are still some 'unknown'
# written out by the stage1 tool
data.meta["origin"] = self.file_.root._v_attrs["CTA PROCESS TYPE"]
data.meta["input_url"] = self.input_url
data.meta["max_events"] = self.max_events
if DataLevel.DL1_IMAGES in self.datalevels:
image_iterators = {
tel.name: self.file_.root.dl1.event.telescope.images[
tel.name
].iterrows()
for tel in self.file_.root.dl1.event.telescope.images
}
if self.has_simulated_dl1:
simulated_image_iterators = {
tel.name: self.file_.root.simulation.event.telescope.images[
tel.name
].iterrows()
for tel in self.file_.root.simulation.event.telescope.images
}
if DataLevel.DL1_PARAMETERS in self.datalevels:
param_readers = {
tel.name: HDF5TableReader(self.file_).read(
f"/dl1/event/telescope/parameters/{tel.name}",
containers=[
HillasParametersContainer(),
TimingParametersContainer(),
LeakageContainer(),
ConcentrationContainer(),
MorphologyContainer(),
IntensityStatisticsContainer(),
PeakTimeStatisticsContainer(),
],
prefixes=True,
)
for tel in self.file_.root.dl1.event.telescope.parameters
}
if self.has_simulated_dl1:
simulated_param_readers = {
tel.name: HDF5TableReader(self.file_).read(
f"/simulation/event/telescope/parameters/{tel.name}",
containers=[
HillasParametersContainer(),
LeakageContainer(),
ConcentrationContainer(),
MorphologyContainer(),
IntensityStatisticsContainer(),
],
prefixes=True,
)
for tel in self.file_.root.dl1.event.telescope.parameters
}
if self.is_simulation:
# simulated shower wide information
mc_shower_reader = HDF5TableReader(self.file_).read(
"/simulation/event/subarray/shower",
SimulatedShowerContainer(),
prefixes="true",
)
# Setup iterators for the array events
events = HDF5TableReader(self.file_).read(
"/dl1/event/subarray/trigger", [TriggerContainer(), EventIndexContainer()]
)
array_pointing_finder = IndexFinder(
self.file_.root.dl1.monitoring.subarray.pointing.col("time")
)
tel_pointing_finder = {
tel.name: IndexFinder(tel.col("time"))
for tel in self.file_.root.dl1.monitoring.telescope.pointing
}
for counter, array_event in enumerate(events):
data.dl1.tel.clear()
data.simulation.tel.clear()
data.pointing.tel.clear()
data.trigger.tel.clear()
data.count = counter
data.trigger, data.index = next(events)
data.trigger.tels_with_trigger = (
np.where(data.trigger.tels_with_trigger)[0] + 1
) # +1 to match array index to telescope id
# Maybe there is a simpler way to do this
# Beware: tels_with_trigger contains all triggered telescopes whereas
# the telescope trigger table contains only the subset of
# allowed_tels given during the creation of the dl1 file
for i in self.file_.root.dl1.event.telescope.trigger.where(
f"(obs_id=={data.index.obs_id}) & (event_id=={data.index.event_id})"
):
if self.allowed_tels and i["tel_id"] not in self.allowed_tels:
continue
if self.datamodel_version == "v1.0.0":
data.trigger.tel[i["tel_id"]].time = i["telescopetrigger_time"]
else:
data.trigger.tel[i["tel_id"]].time = i["time"]
self._fill_array_pointing(data, array_pointing_finder)
self._fill_telescope_pointing(data, tel_pointing_finder)
if self.is_simulation:
data.simulation.shower = next(mc_shower_reader)
for tel in data.trigger.tel.keys():
if self.allowed_tels and tel not in self.allowed_tels:
continue
if self.has_simulated_dl1:
simulated = data.simulation.tel[tel]
dl1 = data.dl1.tel[tel]
if DataLevel.DL1_IMAGES in self.datalevels:
if f"tel_{tel:03d}" not in image_iterators.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the image table."
)
continue
image_row = next(image_iterators[f"tel_{tel:03d}"])
dl1.image = image_row["image"]
dl1.peak_time = image_row["peak_time"]
dl1.image_mask = image_row["image_mask"]
if self.has_simulated_dl1:
if f"tel_{tel:03d}" not in simulated_image_iterators.keys():
logger.warning(
f"Triggered telescope {tel} is missing "
"from the simulated image table, but was present at the "
"reconstructed image table."
)
continue
simulated_image_row = next(
simulated_image_iterators[f"tel_{tel:03d}"]
)
simulated.true_image = simulated_image_row["true_image"]
if DataLevel.DL1_PARAMETERS in self.datalevels:
if f"tel_{tel:03d}" not in param_readers.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the parameters table."
)
continue
# Is there a smarter way to unpack this?
# Best would probbaly be if we could directly read
# into the ImageParametersContainer
params = next(param_readers[f"tel_{tel:03d}"])
dl1.parameters.hillas = params[0]
dl1.parameters.timing = params[1]
dl1.parameters.leakage = params[2]
dl1.parameters.concentration = params[3]
dl1.parameters.morphology = params[4]
dl1.parameters.intensity_statistics = params[5]
dl1.parameters.peak_time_statistics = params[6]
if self.has_simulated_dl1:
if f"tel_{tel:03d}" not in param_readers.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the simulated parameters table, but was "
"present at the reconstructed parameters table."
)
continue
simulated_params = next(
simulated_param_readers[f"tel_{tel:03d}"]
)
simulated.true_parameters.hillas = simulated_params[0]
simulated.true_parameters.leakage = simulated_params[1]
simulated.true_parameters.concentration = simulated_params[2]
simulated.true_parameters.morphology = simulated_params[3]
simulated.true_parameters.intensity_statistics = simulated_params[
4
]
yield data
