def test_h5_file(tmp_path_factory): """Test hdf5 file with some tables for the reader tests""" path = tmp_path_factory.mktemp("hdf5") / "test.h5" r0 = R0CameraContainer() shower = SimulatedShowerContainer() r0.waveform = np.random.uniform(size=(50, 10)) r0.meta["test_attribute"] = 3.14159 r0.meta["date"] = "2020-10-10" with HDF5TableWriter(path, group_name="R0", filters=tables.Filters(complevel=7)) as writer: for _ in range(100): r0.waveform[:] = np.random.uniform(size=(50, 10)) shower.energy = 10**np.random.uniform(1, 2) * u.TeV shower.core_x = np.random.uniform(-1, 1) * u.m shower.core_y = np.random.uniform(-1, 1) * u.m writer.write("tel_001", r0) writer.write("tel_002", r0) # write a second table too writer.write("sim_shower", shower) return path
def test_read_container(temp_h5_file): r0tel1 = R0CameraContainer() r0tel2 = R0CameraContainer() sim_shower = SimulatedShowerContainer() with HDF5TableReader(temp_h5_file) as reader: # get the generators for each table # test supplying a single container as well as an # iterable with one entry only simtab = reader.read("/R0/sim_shower", (sim_shower, )) r0tab1 = reader.read("/R0/tel_001", r0tel1) r0tab2 = reader.read("/R0/tel_002", r0tel2) # read all 3 tables in sync for ii in range(3): m = next(simtab)[0] r0_1 = next(r0tab1) r0_2 = next(r0tab2) print("sim_shower:", m) print("t0:", r0_1.waveform) print("t1:", r0_2.waveform) print("---------------------------") assert "test_attribute" in r0_1.meta assert r0_1.meta["date"] == "2020-10-10"
def test_read_whole_table(temp_h5_file): sim_shower = SimulatedShowerContainer() with HDF5TableReader(temp_h5_file) as reader: for cont in reader.read("/R0/sim_shower", sim_shower): print(cont)
def test_with_context_reader(temp_h5_file): sim_shower = SimulatedShowerContainer() with HDF5TableReader(temp_h5_file) as h5_table: assert h5_table._h5file.isopen == 1 for cont in h5_table.read("/R0/sim_shower", sim_shower): print(cont) assert h5_table._h5file.isopen == 0
def test_write_container(temp_h5_file): r0tel = R0CameraContainer() simshower = SimulatedShowerContainer() simshower.reset() r0tel.waveform = np.random.uniform(size=(50, 10)) r0tel.meta["test_attribute"] = 3.14159 r0tel.meta["date"] = "2020-10-10" with HDF5TableWriter(temp_h5_file, group_name="R0", filters=tables.Filters(complevel=7)) as writer: for ii in range(100): r0tel.waveform[:] = np.random.uniform(size=(50, 10)) simshower.energy = 10**np.random.uniform(1, 2) * u.TeV simshower.core_x = np.random.uniform(-1, 1) * u.m simshower.core_y = np.random.uniform(-1, 1) * u.m writer.write("tel_001", r0tel) writer.write("tel_002", r0tel) # write a second table too writer.write("sim_shower", simshower)
def _generate_events(self): """ Yield ArrayEventContainer to iterate through events. """ data = ArrayEventContainer() # Maybe take some other metadata, but there are still some 'unknown' # written out by the stage1 tool data.meta["origin"] = self.file_.root._v_attrs["CTA PROCESS TYPE"] data.meta["input_url"] = self.input_url data.meta["max_events"] = self.max_events if DataLevel.DL1_IMAGES in self.datalevels: image_iterators = { tel.name: self.file_.root.dl1.event.telescope.images[ tel.name ].iterrows() for tel in self.file_.root.dl1.event.telescope.images } if self.has_simulated_dl1: simulated_image_iterators = { tel.name: self.file_.root.simulation.event.telescope.images[ tel.name ].iterrows() for tel in self.file_.root.simulation.event.telescope.images } if DataLevel.DL1_PARAMETERS in self.datalevels: param_readers = { tel.name: HDF5TableReader(self.file_).read( f"/dl1/event/telescope/parameters/{tel.name}", containers=[ HillasParametersContainer(), TimingParametersContainer(), LeakageContainer(), ConcentrationContainer(), MorphologyContainer(), IntensityStatisticsContainer(), PeakTimeStatisticsContainer(), ], prefixes=True, ) for tel in self.file_.root.dl1.event.telescope.parameters } if self.has_simulated_dl1: simulated_param_readers = { tel.name: HDF5TableReader(self.file_).read( f"/simulation/event/telescope/parameters/{tel.name}", containers=[ HillasParametersContainer(), LeakageContainer(), ConcentrationContainer(), MorphologyContainer(), IntensityStatisticsContainer(), ], prefixes=True, ) for tel in self.file_.root.dl1.event.telescope.parameters } if self.is_simulation: # simulated shower wide information mc_shower_reader = HDF5TableReader(self.file_).read( "/simulation/event/subarray/shower", SimulatedShowerContainer(), prefixes="true", ) # Setup iterators for the array events events = HDF5TableReader(self.file_).read( "/dl1/event/subarray/trigger", [TriggerContainer(), EventIndexContainer()] ) array_pointing_finder = IndexFinder( self.file_.root.dl1.monitoring.subarray.pointing.col("time") ) tel_pointing_finder = { tel.name: IndexFinder(tel.col("time")) for tel in self.file_.root.dl1.monitoring.telescope.pointing } for counter, array_event in enumerate(events): data.dl1.tel.clear() data.simulation.tel.clear() data.pointing.tel.clear() data.trigger.tel.clear() data.count = counter data.trigger, data.index = next(events) data.trigger.tels_with_trigger = ( np.where(data.trigger.tels_with_trigger)[0] + 1 ) # +1 to match array index to telescope id # Maybe there is a simpler way to do this # Beware: tels_with_trigger contains all triggered telescopes whereas # the telescope trigger table contains only the subset of # allowed_tels given during the creation of the dl1 file for i in self.file_.root.dl1.event.telescope.trigger.where( f"(obs_id=={data.index.obs_id}) & (event_id=={data.index.event_id})" ): if self.allowed_tels and i["tel_id"] not in self.allowed_tels: continue if self.datamodel_version == "v1.0.0": data.trigger.tel[i["tel_id"]].time = i["telescopetrigger_time"] else: data.trigger.tel[i["tel_id"]].time = i["time"] self._fill_array_pointing(data, array_pointing_finder) self._fill_telescope_pointing(data, tel_pointing_finder) if self.is_simulation: data.simulation.shower = next(mc_shower_reader) for tel in data.trigger.tel.keys(): if self.allowed_tels and tel not in self.allowed_tels: continue if self.has_simulated_dl1: simulated = data.simulation.tel[tel] dl1 = data.dl1.tel[tel] if DataLevel.DL1_IMAGES in self.datalevels: if f"tel_{tel:03d}" not in image_iterators.keys(): logger.debug( f"Triggered telescope {tel} is missing " "from the image table." ) continue image_row = next(image_iterators[f"tel_{tel:03d}"]) dl1.image = image_row["image"] dl1.peak_time = image_row["peak_time"] dl1.image_mask = image_row["image_mask"] if self.has_simulated_dl1: if f"tel_{tel:03d}" not in simulated_image_iterators.keys(): logger.warning( f"Triggered telescope {tel} is missing " "from the simulated image table, but was present at the " "reconstructed image table." ) continue simulated_image_row = next( simulated_image_iterators[f"tel_{tel:03d}"] ) simulated.true_image = simulated_image_row["true_image"] if DataLevel.DL1_PARAMETERS in self.datalevels: if f"tel_{tel:03d}" not in param_readers.keys(): logger.debug( f"Triggered telescope {tel} is missing " "from the parameters table." ) continue # Is there a smarter way to unpack this? # Best would probbaly be if we could directly read # into the ImageParametersContainer params = next(param_readers[f"tel_{tel:03d}"]) dl1.parameters.hillas = params[0] dl1.parameters.timing = params[1] dl1.parameters.leakage = params[2] dl1.parameters.concentration = params[3] dl1.parameters.morphology = params[4] dl1.parameters.intensity_statistics = params[5] dl1.parameters.peak_time_statistics = params[6] if self.has_simulated_dl1: if f"tel_{tel:03d}" not in param_readers.keys(): logger.debug( f"Triggered telescope {tel} is missing " "from the simulated parameters table, but was " "present at the reconstructed parameters table." ) continue simulated_params = next( simulated_param_readers[f"tel_{tel:03d}"] ) simulated.true_parameters.hillas = simulated_params[0] simulated.true_parameters.leakage = simulated_params[1] simulated.true_parameters.concentration = simulated_params[2] simulated.true_parameters.morphology = simulated_params[3] simulated.true_parameters.intensity_statistics = simulated_params[ 4 ] yield data