def test_read_without_prefixes(tmp_path):
path = tmp_path / "test.h5"
hillas_parameter_container = HillasParametersContainer(fov_lon=1 * u.deg,
fov_lat=1 * u.deg,
length=1 * u.deg,
width=1 * u.deg)
leakage_container = LeakageContainer(
pixels_width_1=0.1,
pixels_width_2=0.1,
intensity_width_1=0.1,
intensity_width_2=0.1,
)
with HDF5TableWriter(path, group_name="dl1", add_prefix=False) as writer:
writer.write("params", (hillas_parameter_container, leakage_container))
df = pd.read_hdf(path, key="/dl1/params")
assert "fov_lon" in df.columns
assert "pixels_width_1" in df.columns
# call with prefixes=False
with HDF5TableReader(path) as reader:
generator = reader.read(
"/dl1/params",
(HillasParametersContainer, LeakageContainer),
prefixes=False,
)
hillas_, leakage_ = next(generator)
for value, read_value in zip(leakage_container.as_dict().values(),
leakage_.as_dict().values()):
np.testing.assert_equal(value, read_value)
for value, read_value in zip(hillas_parameter_container.as_dict().values(),
hillas_.as_dict().values()):
np.testing.assert_equal(value, read_value)
# call with manually removed prefixes
with HDF5TableReader(path) as reader:
generator = reader.read(
"/dl1/params",
(HillasParametersContainer, LeakageContainer),
prefixes=["", ""],
)
hillas_, leakage_ = next(generator)
for value, read_value in zip(leakage_container.as_dict().values(),
leakage_.as_dict().values()):
np.testing.assert_equal(value, read_value)
for value, read_value in zip(hillas_parameter_container.as_dict().values(),
hillas_.as_dict().values()):
np.testing.assert_equal(value, read_value)
def test_read_multiple_containers(tmp_path):
path = tmp_path / "test_append.h5"
hillas_parameter_container = HillasParametersContainer(fov_lon=1 * u.deg,
fov_lat=1 * u.deg,
length=1 * u.deg,
width=1 * u.deg)
leakage_container = LeakageContainer(
pixels_width_1=0.1,
pixels_width_2=0.1,
intensity_width_1=0.1,
intensity_width_2=0.1,
)
with HDF5TableWriter(path, group_name="dl1", add_prefix=True) as writer:
writer.write("params", [hillas_parameter_container, leakage_container])
df = pd.read_hdf(path, key="/dl1/params")
assert "hillas_fov_lon" in df.columns
assert "leakage_pixels_width_1" in df.columns
# test reading both containers separately
with HDF5TableReader(path) as reader:
generator = reader.read("/dl1/params",
HillasParametersContainer,
prefixes=True)
hillas = next(generator)
for value, read_value in zip(hillas_parameter_container.as_dict().values(),
hillas.as_dict().values()):
np.testing.assert_equal(value, read_value)
with HDF5TableReader(path) as reader:
generator = reader.read("/dl1/params", LeakageContainer, prefixes=True)
leakage = next(generator)
for value, read_value in zip(leakage_container.as_dict().values(),
leakage.as_dict().values()):
np.testing.assert_equal(value, read_value)
# test reading both containers simultaneously
with HDF5TableReader(path) as reader:
generator = reader.read(
"/dl1/params",
(HillasParametersContainer, LeakageContainer),
prefixes=True,
)
hillas_, leakage_ = next(generator)
for value, read_value in zip(leakage_container.as_dict().values(),
leakage_.as_dict().values()):
np.testing.assert_equal(value, read_value)
for value, read_value in zip(hillas_parameter_container.as_dict().values(),
hillas_.as_dict().values()):
np.testing.assert_equal(value, read_value)
def test_read_multiple_containers():
hillas_parameter_container = HillasParametersContainer(x=1 * u.m,
y=1 * u.m,
length=1 * u.m,
width=1 * u.m)
leakage_container = LeakageContainer(
pixels_width_1=0.1,
pixels_width_2=0.1,
intensity_width_1=0.1,
intensity_width_2=0.1,
)
with tempfile.NamedTemporaryFile() as f:
with HDF5TableWriter(f.name, group_name="dl1",
add_prefix=True) as writer:
writer.write("params",
[hillas_parameter_container, leakage_container])
df = pd.read_hdf(f.name, key="/dl1/params")
assert "hillas_x" in df.columns
assert "leakage_pixels_width_1" in df.columns
# test reading both containers separately
with HDF5TableReader(f.name) as reader:
generator = reader.read("/dl1/params",
HillasParametersContainer(),
prefixes=True)
hillas = next(generator)
for value, read_value in zip(
hillas_parameter_container.as_dict().values(),
hillas.as_dict().values()):
np.testing.assert_equal(value, read_value)
with HDF5TableReader(f.name) as reader:
generator = reader.read("/dl1/params",
LeakageContainer(),
prefixes=True)
leakage = next(generator)
for value, read_value in zip(leakage_container.as_dict().values(),
leakage.as_dict().values()):
np.testing.assert_equal(value, read_value)
# test reading both containers simultaneously
with HDF5TableReader(f.name) as reader:
generator = reader.read(
"/dl1/params",
[HillasParametersContainer(),
LeakageContainer()],
prefixes=True,
)
hillas_, leakage_ = next(generator)
for value, read_value in zip(leakage_container.as_dict().values(),
leakage_.as_dict().values()):
np.testing.assert_equal(value, read_value)
for value, read_value in zip(
hillas_parameter_container.as_dict().values(),
hillas_.as_dict().values()):
np.testing.assert_equal(value, read_value)
def test_prefix(tmp_path):
tmp_file = tmp_path / "test_prefix.hdf5"
hillas_parameter_container = HillasParametersContainer(x=1 * u.m,
y=1 * u.m,
length=1 * u.m,
width=1 * u.m)
leakage_container = LeakageContainer(
pixels_width_1=0.1,
pixels_width_2=0.1,
intensity_width_1=0.1,
intensity_width_2=0.1,
)
with HDF5TableWriter(tmp_file.name, group_name="blabla",
add_prefix=True) as writer:
writer.write("events", [hillas_parameter_container, leakage_container])
df = pd.read_hdf(tmp_file.name, key="/blabla/events")
assert "hillas_x" in df.columns
assert "leakage_pixels_width_1" in df.columns
with HDF5TableReader(tmp_file.name) as reader:
generator = reader.read("/blabla/events",
HillasParametersContainer(),
prefix=True)
hillas = next(generator)
for value, read_value in zip(hillas_parameter_container.as_dict().values(),
hillas.as_dict().values()):
np.testing.assert_equal(value, read_value)
with HDF5TableReader(tmp_file.name) as reader:
generator = reader.read("/blabla/events",
LeakageContainer(),
prefix=True)
leakage = next(generator)
for value, read_value in zip(leakage_container.as_dict().values(),
leakage.as_dict().values()):
np.testing.assert_equal(value, read_value)
def test_read_without_prefixes():
hillas_parameter_container = HillasParametersContainer(
x=1 * u.m, y=1 * u.m, length=1 * u.m, width=1 * u.m
)
leakage_container = LeakageContainer(
pixels_width_1=0.1,
pixels_width_2=0.1,
intensity_width_1=0.1,
intensity_width_2=0.1,
)
with tempfile.NamedTemporaryFile() as f:
with HDF5TableWriter(f.name, group_name="dl1", add_prefix=False) as writer:
writer.write("params", [hillas_parameter_container, leakage_container])
df = pd.read_hdf(f.name, key="/dl1/params")
assert "x" in df.columns
assert "pixels_width_1" in df.columns
# call with prefixes=False
with HDF5TableReader(f.name) as reader:
generator = reader.read(
"/dl1/params",
[HillasParametersContainer(), LeakageContainer()],
prefixes=False,
)
hillas_, leakage_ = next(generator)
for value, read_value in zip(
leakage_container.as_dict().values(), leakage_.as_dict().values()
):
np.testing.assert_equal(value, read_value)
for value, read_value in zip(
hillas_parameter_container.as_dict().values(), hillas_.as_dict().values()
):
np.testing.assert_equal(value, read_value)
# call with manually removed prefixes
with HDF5TableReader(f.name) as reader:
generator = reader.read(
"/dl1/params",
[HillasParametersContainer(prefix=""), LeakageContainer(prefix="")],
prefixes=True,
)
hillas_, leakage_ = next(generator)
for value, read_value in zip(
leakage_container.as_dict().values(), leakage_.as_dict().values()
):
np.testing.assert_equal(value, read_value)
for value, read_value in zip(
hillas_parameter_container.as_dict().values(), hillas_.as_dict().values()
):
np.testing.assert_equal(value, read_value)
def test_prefix(tmp_path):
tmp_file = tmp_path / "test_prefix.hdf5"
hillas_parameter_container = HillasParametersContainer(
x=1 * u.m, y=1 * u.m, length=1 * u.m, width=1 * u.m
)
leakage_container = LeakageContainer(
pixels_width_1=0.1,
pixels_width_2=0.1,
intensity_width_1=0.1,
intensity_width_2=0.1,
)
with HDF5TableWriter(tmp_file.name, group_name="blabla", add_prefix=True) as writer:
writer.write("events", [hillas_parameter_container, leakage_container])
df = pd.read_hdf(tmp_file.name, key="/blabla/events")
assert "hillas_x" in df.columns
assert "leakage_pixels_width_1" in df.columns
def _generator(self):
"""
Stereo event generator. Yields DataContainer instances, filled
with the read event data.
Returns
-------
"""
counter = 0
data = DataContainer()
data.meta['origin'] = "MAGIC"
data.meta['input_url'] = self.input_url
data.meta['is_simulation'] = True
data.mcheader = self._mc_header
if self.calib_M1 is not None and self.calib_M2 is not None:
#Reading data from root file for Events table
eventid_M1 = np.asarray(
self.calib_M1["MRawEvtHeader.fStereoEvtNumber"].array())
eventid_M2 = np.asarray(
self.calib_M2["MRawEvtHeader.fStereoEvtNumber"].array())
zenith = np.asarray(self.calib_M1["MMcEvt.fTheta"].array())
pointing_altitude = np.asarray(
self.calib_M1["MPointingPos.fZd"].array())
azimuth = np.asarray(self.calib_M1["MMcEvt.fPhi"].array())
pointing_azimuth = np.asarray(
self.calib_M1["MPointingPos.fAz"].array())
core_x = np.asarray(self.calib_M1["MMcEvt.fCoreX"].array())
core_y = np.asarray(self.calib_M1["MMcEvt.fCoreY"].array())
mc_energy = np.asarray(
self.calib_M1["MMcEvt.fEnergy"].array()) / 1000.0
h_first_int = np.asarray(
self.calib_M1["MMcEvt.fZFirstInteraction"].array())
#Reading data from root file for Image table
charge_M1 = np.asarray(
self.calib_M1["MCerPhotEvt.fPixels.fPhot"].array())
peak_time_M1 = np.asarray(
self.calib_M1["MArrivalTime.fData"].array())
charge_M2 = np.asarray(
self.calib_M2["MCerPhotEvt.fPixels.fPhot"].array())
peak_time_M2 = np.asarray(
self.calib_M2["MArrivalTime.fData"].array())
if self.superstar is not None:
#Reading data from root file for Events table
eventid_M1 = np.asarray(
self.superstar["MRawEvtHeader_1.fStereoEvtNumber"].array())
eventid_M2 = np.asarray(
self.superstar["MRawEvtHeader_2.fStereoEvtNumber"].array())
zenith = np.asarray(self.superstar["MMcEvt_1.fTheta"].array())
pointing_altitude = np.asarray(
self.superstar["MPointingPos_1.fZd"].array())
azimuth = np.asarray(self.superstar["MMcEvt_1.fPhi"].array())
pointing_azimuth = np.asarray(
self.superstar["MPointingPos_1.fAz"].array())
core_x = np.asarray(self.superstar["MMcEvt_1.fCoreX"].array())
core_y = np.asarray(self.superstar["MMcEvt_1.fCoreY"].array())
mc_energy = np.asarray(
self.superstar["MMcEvt_1.fEnergy"].array()) / 1000.0
h_first_int = np.asarray(
self.superstar["MMcEvt_1.fZFirstInteraction"].array())
#Reading data from root file for Parameter table
hillas_intensity_M1 = np.asarray(
self.superstar["MHillas_1.fSize"].array())
hillas_intensity_M2 = np.asarray(
self.superstar["MHillas_2.fSize"].array())
hillas_x_M1 = np.asarray(
self.superstar["MHillas_1.fMeanX"].array())
hillas_x_M2 = np.asarray(
self.superstar["MHillas_2.fMeanX"].array())
hillas_y_M1 = np.asarray(
self.superstar["MHillas_1.fMeanY"].array())
hillas_y_M2 = np.asarray(
self.superstar["MHillas_2.fMeanY"].array())
hillas_r_M1 = np.sqrt(
np.power(hillas_x_M1, 2) + np.power(hillas_y_M1, 2))
hillas_r_M2 = np.sqrt(
np.power(hillas_x_M2, 2) + np.power(hillas_y_M2, 2))
hillas_phi_M1 = np.arctan2(hillas_y_M1, hillas_x_M1)
hillas_phi_M2 = np.arctan2(hillas_y_M2, hillas_x_M2)
hillas_length_M1 = np.asarray(
self.superstar["MHillas_1.fLength"].array())
hillas_length_M2 = np.asarray(
self.superstar["MHillas_2.fLength"].array())
hillas_width_M1 = np.asarray(
self.superstar["MHillas_1.fWidth"].array())
hillas_width_M2 = np.asarray(
self.superstar["MHillas_2.fWidth"].array())
hillas_psi_M1 = np.asarray(
self.superstar["MHillas_1.fDelta"].array())
hillas_psi_M2 = np.asarray(
self.superstar["MHillas_2.fDelta"].array())
hillas_skewness_M1 = np.asarray(
self.superstar["MHillasExt_1.fM3Long"].array())
hillas_skewness_M2 = np.asarray(
self.superstar["MHillasExt_2.fM3Long"].array())
leakage_intensity_1_M1 = np.asarray(
self.superstar["MNewImagePar_1.fLeakage1"].array())
leakage_intensity_1_M2 = np.asarray(
self.superstar["MNewImagePar_2.fLeakage1"].array())
leakage_intensity_2_M1 = np.asarray(
self.superstar["MNewImagePar_1.fLeakage2"].array())
leakage_intensity_2_M2 = np.asarray(
self.superstar["MNewImagePar_2.fLeakage2"].array())
num_islands_M1 = np.asarray(
self.superstar["MCerPhotEvt_1.fNumIslands"].array())
num_islands_M2 = np.asarray(
self.superstar["MCerPhotEvt_2.fNumIslands"].array())
#Reading data from root file for Image table (peak time and image mask not )
charge_M1 = np.asarray(
self.superstar["MCerPhotEvt_1.fPixels.fPhot"].array())
peak_time_M1 = np.zeros((charge_M1.shape[0], 1039))
image_mask_M1 = np.zeros((charge_M1.shape[0], 1039), dtype=bool)
charge_M2 = np.asarray(
self.superstar["MCerPhotEvt_2.fPixels.fPhot"].array())
peak_time_M2 = np.zeros((charge_M2.shape[0], 1039))
image_mask_M2 = np.zeros((charge_M2.shape[0], 1039), dtype=bool)
# Get the shower primary id
shower_primary_id = 1
if self.file_list[0].split("/")[-1].startswith("GA"):
shower_primary_id = 1
#Iterating over all events, and saving only stereo ones
total_events = min(len(charge_M1), len(charge_M2))
tels_with_data = {1, 2}
for i in range(0, total_events):
if eventid_M1[i] != 0:
obs_id = self.run_number
event_id = eventid_M1[i]
i2 = np.where(eventid_M2 == eventid_M1[i])
i2 = i2[0].astype(int)
data.count = counter
# Setting up the Data container
data.index.obs_id = obs_id
data.index.event_id = event_id
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
# Adding the array pointing in the pointing container
data.pointing.array_altitude = u.Quantity(
np.deg2rad(90.0 - pointing_altitude[i]), u.rad)
data.pointing.array_azimuth = u.Quantity(
np.deg2rad(pointing_azimuth[i]), u.rad)
# Filling the DL1 container with the event data
for tel_id in tels_with_data:
#Adding telescope pointing container
data.pointing.tel[tel_id].azimuth = u.Quantity(
np.deg2rad(pointing_azimuth[i]), u.rad)
data.pointing.tel[tel_id].altitude = u.Quantity(
np.deg2rad(90.0 - pointing_altitude[i]), u.rad)
#Adding MC data
data.mc.alt = Angle(np.pi / 2.0 - zenith[i], u.rad)
data.mc.az = Angle(
np.deg2rad(180.0 - 7.0) - azimuth[i], u.rad)
data.mc.x_max = u.Quantity(0, X_MAX_UNIT)
data.mc.h_first_int = u.Quantity(h_first_int[i], u.m)
data.mc.core_x = u.Quantity(core_x[i], u.m)
data.mc.core_y = u.Quantity(core_y[i], u.m)
data.mc.energy = u.Quantity(mc_energy[i], u.TeV)
data.mc.shower_primary_id = shower_primary_id
if self.superstar is not None:
leakage_values = LeakageContainer()
hillas_parameters_values = HillasParametersContainer()
concentration_values = ConcentrationContainer()
timing_values = TimingParametersContainer()
morphology_values = MorphologyContainer()
# Adding charge, peak time and parameters
if tel_id == 1:
data.dl1.tel[tel_id].image = charge_M1[i][:1039]
data.dl1.tel[tel_id].peak_time = peak_time_M1[i][:1039]
if self.superstar is not None:
data.dl1.tel[tel_id].image_mask = image_mask_M1[
i][:1039]
hillas_parameters_values[
"intensity"] = hillas_intensity_M1[i]
hillas_parameters_values["x"] = u.Quantity(
hillas_x_M1[i], unit=u.mm)
hillas_parameters_values["y"] = u.Quantity(
hillas_y_M1[i], unit=u.mm)
hillas_parameters_values["r"] = u.Quantity(
hillas_r_M1[i], unit=u.mm)
hillas_parameters_values["phi"] = u.Quantity(
hillas_phi_M1[i], unit=u.rad)
hillas_parameters_values["length"] = u.Quantity(
hillas_length_M1[i], unit=u.mm)
hillas_parameters_values["width"] = u.Quantity(
hillas_width_M1[i], unit=u.mm)
hillas_parameters_values["psi"] = u.Quantity(
hillas_psi_M1[i], unit=u.rad)
hillas_parameters_values[
"skewness"] = hillas_skewness_M1[i]
leakage_values[
"intensity_width_1"] = leakage_intensity_1_M1[
i]
leakage_values[
"intensity_width_2"] = leakage_intensity_2_M1[
i]
morphology_values["num_pixels"] = 1039
morphology_values["num_islands"] = num_islands_M1[
i]
else:
data.dl1.tel[tel_id].image = charge_M2[i][:1039]
data.dl1.tel[tel_id].peak_time = peak_time_M2[i][:1039]
if self.superstar is not None:
data.dl1.tel[tel_id].image_mask = image_mask_M2[
i][:1039]
hillas_parameters_values[
"intensity"] = hillas_intensity_M2[i]
hillas_parameters_values["x"] = u.Quantity(
hillas_x_M2[i], unit=u.mm)
hillas_parameters_values["y"] = u.Quantity(
hillas_y_M2[i], unit=u.mm)
hillas_parameters_values["r"] = u.Quantity(
hillas_r_M2[i], unit=u.mm)
hillas_parameters_values["phi"] = u.Quantity(
hillas_phi_M2[i], unit=u.rad)
hillas_parameters_values["length"] = u.Quantity(
hillas_length_M2[i], unit=u.mm)
hillas_parameters_values["width"] = u.Quantity(
hillas_width_M2[i], unit=u.mm)
hillas_parameters_values["psi"] = u.Quantity(
hillas_psi_M2[i], unit=u.rad)
hillas_parameters_values[
"skewness"] = hillas_skewness_M2[i]
leakage_values[
"intensity_width_1"] = leakage_intensity_1_M2[
i]
leakage_values[
"intensity_width_2"] = leakage_intensity_2_M2[
i]
morphology_values["num_pixels"] = 1039
morphology_values["num_islands"] = num_islands_M2[
i]
if self.superstar is not None:
data.dl1.tel[
tel_id].parameters.leakage = leakage_values
data.dl1.tel[
tel_id].parameters.hillas = hillas_parameters_values
data.dl1.tel[
tel_id].parameters.concentration = concentration_values
data.dl1.tel[tel_id].parameters.timing = timing_values
data.dl1.tel[
tel_id].parameters.morphology = morphology_values
# Setting the telescopes with data
data.r0.tels_with_data = tels_with_data
data.r1.tels_with_data = tels_with_data
data.dl0.tels_with_data = tels_with_data
yield data
counter += 1
return
def _generate_events(self):
"""
Yield ArrayEventContainer to iterate through events.
"""
data = ArrayEventContainer()
# Maybe take some other metadata, but there are still some 'unknown'
# written out by the stage1 tool
data.meta["origin"] = self.file_.root._v_attrs["CTA PROCESS TYPE"]
data.meta["input_url"] = self.input_url
data.meta["max_events"] = self.max_events
if DataLevel.DL1_IMAGES in self.datalevels:
image_iterators = {
tel.name: self.file_.root.dl1.event.telescope.images[
tel.name
].iterrows()
for tel in self.file_.root.dl1.event.telescope.images
}
if self.has_simulated_dl1:
simulated_image_iterators = {
tel.name: self.file_.root.simulation.event.telescope.images[
tel.name
].iterrows()
for tel in self.file_.root.simulation.event.telescope.images
}
if DataLevel.DL1_PARAMETERS in self.datalevels:
param_readers = {
tel.name: HDF5TableReader(self.file_).read(
f"/dl1/event/telescope/parameters/{tel.name}",
containers=[
HillasParametersContainer(),
TimingParametersContainer(),
LeakageContainer(),
ConcentrationContainer(),
MorphologyContainer(),
IntensityStatisticsContainer(),
PeakTimeStatisticsContainer(),
],
prefixes=True,
)
for tel in self.file_.root.dl1.event.telescope.parameters
}
if self.has_simulated_dl1:
simulated_param_readers = {
tel.name: HDF5TableReader(self.file_).read(
f"/simulation/event/telescope/parameters/{tel.name}",
containers=[
HillasParametersContainer(),
LeakageContainer(),
ConcentrationContainer(),
MorphologyContainer(),
IntensityStatisticsContainer(),
],
prefixes=True,
)
for tel in self.file_.root.dl1.event.telescope.parameters
}
if self.is_simulation:
# simulated shower wide information
mc_shower_reader = HDF5TableReader(self.file_).read(
"/simulation/event/subarray/shower",
SimulatedShowerContainer(),
prefixes="true",
)
# Setup iterators for the array events
events = HDF5TableReader(self.file_).read(
"/dl1/event/subarray/trigger", [TriggerContainer(), EventIndexContainer()]
)
array_pointing_finder = IndexFinder(
self.file_.root.dl1.monitoring.subarray.pointing.col("time")
)
tel_pointing_finder = {
tel.name: IndexFinder(tel.col("time"))
for tel in self.file_.root.dl1.monitoring.telescope.pointing
}
for counter, array_event in enumerate(events):
data.dl1.tel.clear()
data.simulation.tel.clear()
data.pointing.tel.clear()
data.trigger.tel.clear()
data.count = counter
data.trigger, data.index = next(events)
data.trigger.tels_with_trigger = (
np.where(data.trigger.tels_with_trigger)[0] + 1
) # +1 to match array index to telescope id
# Maybe there is a simpler way to do this
# Beware: tels_with_trigger contains all triggered telescopes whereas
# the telescope trigger table contains only the subset of
# allowed_tels given during the creation of the dl1 file
for i in self.file_.root.dl1.event.telescope.trigger.where(
f"(obs_id=={data.index.obs_id}) & (event_id=={data.index.event_id})"
):
if self.allowed_tels and i["tel_id"] not in self.allowed_tels:
continue
if self.datamodel_version == "v1.0.0":
data.trigger.tel[i["tel_id"]].time = i["telescopetrigger_time"]
else:
data.trigger.tel[i["tel_id"]].time = i["time"]
self._fill_array_pointing(data, array_pointing_finder)
self._fill_telescope_pointing(data, tel_pointing_finder)
if self.is_simulation:
data.simulation.shower = next(mc_shower_reader)
for tel in data.trigger.tel.keys():
if self.allowed_tels and tel not in self.allowed_tels:
continue
if self.has_simulated_dl1:
simulated = data.simulation.tel[tel]
dl1 = data.dl1.tel[tel]
if DataLevel.DL1_IMAGES in self.datalevels:
if f"tel_{tel:03d}" not in image_iterators.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the image table."
)
continue
image_row = next(image_iterators[f"tel_{tel:03d}"])
dl1.image = image_row["image"]
dl1.peak_time = image_row["peak_time"]
dl1.image_mask = image_row["image_mask"]
if self.has_simulated_dl1:
if f"tel_{tel:03d}" not in simulated_image_iterators.keys():
logger.warning(
f"Triggered telescope {tel} is missing "
"from the simulated image table, but was present at the "
"reconstructed image table."
)
continue
simulated_image_row = next(
simulated_image_iterators[f"tel_{tel:03d}"]
)
simulated.true_image = simulated_image_row["true_image"]
if DataLevel.DL1_PARAMETERS in self.datalevels:
if f"tel_{tel:03d}" not in param_readers.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the parameters table."
)
continue
# Is there a smarter way to unpack this?
# Best would probbaly be if we could directly read
# into the ImageParametersContainer
params = next(param_readers[f"tel_{tel:03d}"])
dl1.parameters.hillas = params[0]
dl1.parameters.timing = params[1]
dl1.parameters.leakage = params[2]
dl1.parameters.concentration = params[3]
dl1.parameters.morphology = params[4]
dl1.parameters.intensity_statistics = params[5]
dl1.parameters.peak_time_statistics = params[6]
if self.has_simulated_dl1:
if f"tel_{tel:03d}" not in param_readers.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the simulated parameters table, but was "
"present at the reconstructed parameters table."
)
continue
simulated_params = next(
simulated_param_readers[f"tel_{tel:03d}"]
)
simulated.true_parameters.hillas = simulated_params[0]
simulated.true_parameters.leakage = simulated_params[1]
simulated.true_parameters.concentration = simulated_params[2]
simulated.true_parameters.morphology = simulated_params[3]
simulated.true_parameters.intensity_statistics = simulated_params[
4
]
yield data
def _generator(self):
"""
Stereo event generator. Yields DataContainer instances, filled
with the read event data.
Returns
-------
"""
counter = 0
data = DataContainer()
data.meta["origin"] = "MAGIC"
data.meta["input_url"] = self.input_url
data.meta["is_simulation"] = self.mc
data.mcheader = self._header
if self.calib_M1 is not None and self.calib_M2 is not None:
# Reading data from root file for Events table
shower_primary_id = self.magic_to_cta_shower_primary_id[int(
self.superstar["MMcEvt_1.fPartId"].array()[0])]
eventid_M1 = np.asarray(
self.calib_M1["MRawEvtHeader.fStereoEvtNumber"].array())
eventid_M2 = np.asarray(
self.calib_M2["MRawEvtHeader.fStereoEvtNumber"].array())
zenith = np.asarray(self.calib_M1["MMcEvt.fTheta"].array())
pointing_altitude = np.asarray(
self.calib_M1["MPointingPos.fZd"].array())
azimuth = np.asarray(self.calib_M1["MMcEvt.fPhi"].array())
pointing_azimuth = np.asarray(
self.calib_M1["MPointingPos.fAz"].array())
core_x = np.asarray(self.calib_M1["MMcEvt.fCoreX"].array())
core_y = np.asarray(self.calib_M1["MMcEvt.fCoreY"].array())
mc_energy = np.asarray(
self.calib_M1["MMcEvt.fEnergy"].array()) / 1000.0
h_first_int = np.asarray(
self.calib_M1["MMcEvt.fZFirstInteraction"].array())
# Reading data from root file for Image table
charge_M1 = np.asarray(
self.calib_M1["MCerPhotEvt.fPixels.fPhot"].array())
peak_time_M1 = np.asarray(
self.calib_M1["MArrivalTime.fData"].array())
image_mask_M1 = np.asarray(self.calib_M1["CleanCharge"].array())
for i, mask in enumerate(image_mask_M1):
image_mask_M1[i] = np.array(mask) != 0
charge_M2 = np.asarray(
self.calib_M2["MCerPhotEvt.fPixels.fPhot"].array())
peak_time_M2 = np.asarray(
self.calib_M2["MArrivalTime.fData"].array())
image_mask_M2 = np.asarray(self.calib_M2["CleanCharge"].array())
for i, mask in enumerate(image_mask_M2):
image_mask_M2[i] = np.array(mask) != 0
if self.superstar is not None:
# Reading data from root file for Events table
# only read MC information if it exists
if self.is_simulation:
shower_primary_id = self.magic_to_cta_shower_primary_id[int(
self.superstar["MMcEvt_1.fPartId"].array()[0])]
src_pos_cam_Y = np.asarray(
self.superstar["MSrcPosCam_1.fY"].array())
src_pos_cam_X = np.asarray(
self.superstar["MSrcPosCam_1.fX"].array())
core_x = np.asarray(self.superstar["MMcEvt_1.fCoreX"].array())
core_y = np.asarray(self.superstar["MMcEvt_1.fCoreY"].array())
mc_energy = (
np.asarray(self.superstar["MMcEvt_1.fEnergy"].array()) /
1000.0)
h_first_int = np.asarray(
self.superstar["MMcEvt_1.fZFirstInteraction"].array())
eventid_M1 = np.asarray(
self.superstar["MRawEvtHeader_1.fStereoEvtNumber"].array())
eventid_M2 = np.asarray(
self.superstar["MRawEvtHeader_2.fStereoEvtNumber"].array())
pointing_altitude = np.asarray(
self.superstar["MPointingPos_1.fZd"].array())
pointing_azimuth = np.asarray(
self.superstar["MPointingPos_1.fAz"].array())
# Reading data from root file for Parameter table
hillas_intensity_M1 = np.asarray(
self.superstar["MHillas_1.fSize"].array())
hillas_intensity_M2 = np.asarray(
self.superstar["MHillas_2.fSize"].array())
hillas_x_M1 = np.asarray(
self.superstar["MHillas_1.fMeanX"].array())
hillas_x_M2 = np.asarray(
self.superstar["MHillas_2.fMeanX"].array())
hillas_y_M1 = np.asarray(
self.superstar["MHillas_1.fMeanY"].array())
hillas_y_M2 = np.asarray(
self.superstar["MHillas_2.fMeanY"].array())
hillas_r_M1 = np.sqrt(
np.power(hillas_x_M1, 2) + np.power(hillas_y_M1, 2))
hillas_r_M2 = np.sqrt(
np.power(hillas_x_M2, 2) + np.power(hillas_y_M2, 2))
hillas_phi_M1 = np.arctan2(hillas_y_M1, hillas_x_M1)
hillas_phi_M2 = np.arctan2(hillas_y_M2, hillas_x_M2)
hillas_length_M1 = np.asarray(
self.superstar["MHillas_1.fLength"].array())
hillas_length_M2 = np.asarray(
self.superstar["MHillas_2.fLength"].array())
hillas_width_M1 = np.asarray(
self.superstar["MHillas_1.fWidth"].array())
hillas_width_M2 = np.asarray(
self.superstar["MHillas_2.fWidth"].array())
hillas_psi_M1 = np.asarray(
self.superstar["MHillas_1.fDelta"].array())
hillas_psi_M2 = np.asarray(
self.superstar["MHillas_2.fDelta"].array())
hillas_skewness_M1 = np.asarray(
self.superstar["MHillasExt_1.fM3Long"].array())
hillas_skewness_M2 = np.asarray(
self.superstar["MHillasExt_2.fM3Long"].array())
leakage_intensity_1_M1 = np.asarray(
self.superstar["MNewImagePar_1.fLeakage1"].array())
leakage_intensity_1_M2 = np.asarray(
self.superstar["MNewImagePar_2.fLeakage1"].array())
leakage_intensity_2_M1 = np.asarray(
self.superstar["MNewImagePar_1.fLeakage2"].array())
leakage_intensity_2_M2 = np.asarray(
self.superstar["MNewImagePar_2.fLeakage2"].array())
num_islands_M1 = np.asarray(
self.superstar["MImagePar_1.fNumIslands"].array())
num_islands_M2 = np.asarray(
self.superstar["MImagePar_2.fNumIslands"].array())
x_max = np.asarray(self.superstar["MStereoPar.fXMax"].array())
# Reading data from root file for Image table (charge, peak time and image mask)
charge_M1 = np.asarray(self.superstar["UprootImageOrig_1"].array())
for i, charge in enumerate(charge_M1):
charge_M1[i] = np.array(charge)
peak_time_M1 = np.asarray(
self.superstar["MArrivalTime_1.fData"].array())
image_mask_M1 = np.asarray(
self.superstar["UprootImageOrigClean_1"].array())
for i, mask in enumerate(image_mask_M1):
image_mask_M1[i] = np.array(mask) != 0
charge_M2 = np.asarray(self.superstar["UprootImageOrig_2"].array())
for i, charge in enumerate(charge_M2):
charge_M2[i] = np.array(charge)
charge_M2 = np.asarray(
self.superstar["MCerPhotEvt_2.fPixels.fPhot"].array())
peak_time_M2 = np.asarray(
self.superstar["MArrivalTime_2.fData"].array())
image_mask_M2 = np.asarray(
self.superstar["UprootImageOrigClean_2"].array())
for i, mask in enumerate(image_mask_M2):
image_mask_M2[i] = np.array(mask) != 0
# Iterating over all events, and saving only stereo ones
total_events = min(len(charge_M1), len(charge_M2))
tels_with_data = {1, 2}
for i in range(0, total_events):
if eventid_M1[i] != 0:
obs_id = self.run_number
event_id = eventid_M1[i]
i2 = np.where(eventid_M2 == eventid_M1[i])
i2 = i2[0].astype(int)
data.count = counter
# Setting up the Data container
data.index.obs_id = obs_id
data.index.event_id = event_id
data.r0.tel.clear()
data.r1.tel.clear()
data.dl0.tel.clear()
# Adding the array pointing in the pointing container
data.pointing.array_altitude = u.Quantity(
np.deg2rad(90.0 - pointing_altitude[i]), u.rad)
data.pointing.array_azimuth = u.Quantity(
np.deg2rad(pointing_azimuth[i]), u.rad)
# Filling the DL1 container with the event data
for tel_id in tels_with_data:
# Adding telescope pointing container
data.pointing.tel[tel_id].azimuth = u.Quantity(
np.deg2rad(pointing_azimuth[i]), u.rad)
data.pointing.tel[tel_id].altitude = u.Quantity(
np.deg2rad(90.0 - pointing_altitude[i]), u.rad)
# Adding MC data
if self.is_simulation:
data.mc.alt = Angle(
np.deg2rad(src_pos_cam_Y[i] * 0.00337), u.rad)
data.mc.az = Angle(
np.deg2rad(src_pos_cam_X[i] * 0.00337), u.rad)
data.mc.x_max = u.Quantity(x_max[i], X_MAX_UNIT)
data.mc.h_first_int = u.Quantity(h_first_int[i], u.m)
data.mc.core_x = u.Quantity(core_x[i], u.m)
data.mc.core_y = u.Quantity(core_y[i], u.m)
data.mc.energy = u.Quantity(mc_energy[i], u.TeV)
data.mc.shower_primary_id = shower_primary_id
if self.superstar is not None:
leakage_values = LeakageContainer()
hillas_parameters_values = HillasParametersContainer()
concentration_values = ConcentrationContainer()
timing_values = TimingParametersContainer()
morphology_values = MorphologyContainer()
# Adding charge, peak time and parameters
if tel_id == 1:
data.dl1.tel[tel_id].image = charge_M1[i][:1039]
data.dl1.tel[tel_id].peak_time = peak_time_M1[i][:1039]
data.dl1.tel[tel_id].image_mask = image_mask_M1[
i][:1039]
if self.superstar is not None:
hillas_parameters_values[
"intensity"] = hillas_intensity_M1[i]
hillas_parameters_values["x"] = u.Quantity(
hillas_x_M1[i], unit=u.mm)
hillas_parameters_values["y"] = u.Quantity(
hillas_y_M1[i], unit=u.mm)
hillas_parameters_values["r"] = u.Quantity(
hillas_r_M1[i], unit=u.mm)
hillas_parameters_values["phi"] = u.Quantity(
hillas_phi_M1[i], unit=u.rad)
hillas_parameters_values["length"] = u.Quantity(
hillas_length_M1[i], unit=u.mm)
hillas_parameters_values["width"] = u.Quantity(
hillas_width_M1[i], unit=u.mm)
hillas_parameters_values["psi"] = u.Quantity(
hillas_psi_M1[i], unit=u.rad)
hillas_parameters_values[
"skewness"] = hillas_skewness_M1[i]
leakage_values[
"intensity_width_1"] = leakage_intensity_1_M1[
i]
leakage_values[
"intensity_width_2"] = leakage_intensity_2_M1[
i]
morphology_values["num_islands"] = num_islands_M1[
i]
else:
data.dl1.tel[tel_id].image = charge_M2[i][:1039]
data.dl1.tel[tel_id].peak_time = peak_time_M2[i][:1039]
data.dl1.tel[tel_id].image_mask = image_mask_M2[
i][:1039]
if self.superstar is not None:
hillas_parameters_values[
"intensity"] = hillas_intensity_M2[i]
hillas_parameters_values["x"] = u.Quantity(
hillas_x_M2[i], unit=u.mm)
hillas_parameters_values["y"] = u.Quantity(
hillas_y_M2[i], unit=u.mm)
hillas_parameters_values["r"] = u.Quantity(
hillas_r_M2[i], unit=u.mm)
hillas_parameters_values["phi"] = u.Quantity(
hillas_phi_M2[i], unit=u.rad)
hillas_parameters_values["length"] = u.Quantity(
hillas_length_M2[i], unit=u.mm)
hillas_parameters_values["width"] = u.Quantity(
hillas_width_M2[i], unit=u.mm)
hillas_parameters_values["psi"] = u.Quantity(
hillas_psi_M2[i], unit=u.rad)
hillas_parameters_values[
"skewness"] = hillas_skewness_M2[i]
leakage_values[
"intensity_width_1"] = leakage_intensity_1_M2[
i]
leakage_values[
"intensity_width_2"] = leakage_intensity_2_M2[
i]
morphology_values["num_islands"] = num_islands_M2[
i]
if self.superstar is not None:
data.dl1.tel[
tel_id].parameters.leakage = leakage_values
data.dl1.tel[
tel_id].parameters.hillas = hillas_parameters_values
data.dl1.tel[
tel_id].parameters.concentration = concentration_values
data.dl1.tel[tel_id].parameters.timing = timing_values
data.dl1.tel[
tel_id].parameters.morphology = morphology_values
# Setting the telescopes with data
data.r0.tels_with_data = tels_with_data
data.r1.tels_with_data = tels_with_data
data.dl0.tels_with_data = tels_with_data
yield data
counter += 1
return