def test_writing_nan_defaults():
from ctapipe.containers import ImageParametersContainer
params = ImageParametersContainer()
with tempfile.NamedTemporaryFile(suffix=".hdf5") as f:
with HDF5TableWriter(f.name, mode="w") as writer:
writer.write("params", params.values())
def test_writing_nan_defaults(tmp_path):
from ctapipe.containers import ImageParametersContainer
path = tmp_path / "test.h5"
params = ImageParametersContainer()
with HDF5TableWriter(path, mode="w") as writer:
writer.write("params", params.values())
def test_reconstructors(reconstructors):
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted"""
filename = get_dataset_path(
"gamma_LaPalma_baseline_20Zd_180Az_prod3b_test.simtel.gz")
source = EventSource(filename, max_events=10)
subarray = source.subarray
calib = CameraCalibrator(source.subarray)
horizon_frame = AltAz()
for event in source:
calib(event)
sim_shower = event.simulation.shower
array_pointing = SkyCoord(az=sim_shower.az,
alt=sim_shower.alt,
frame=horizon_frame)
telescope_pointings = {}
for tel_id, dl1 in event.dl1.tel.items():
geom = source.subarray.tel[tel_id].camera.geometry
telescope_pointings[tel_id] = SkyCoord(
alt=event.pointing.tel[tel_id].altitude,
az=event.pointing.tel[tel_id].azimuth,
frame=horizon_frame,
)
mask = tailcuts_clean(geom,
dl1.image,
picture_thresh=10.0,
boundary_thresh=5.0)
dl1.parameters = ImageParametersContainer()
try:
moments = hillas_parameters(geom[mask], dl1.image[mask])
except HillasParameterizationError:
dl1.parameters.hillas = HillasParametersContainer()
continue
# Make sure we provide only good images for the test
if np.isnan(moments.width.value) or (moments.width.value == 0):
dl1.parameters.hillas = HillasParametersContainer()
else:
dl1.parameters.hillas = moments
hillas_dict = {
tel_id: dl1.parameters.hillas
for tel_id, dl1 in event.dl1.tel.items()
if np.isfinite(dl1.parameters.hillas.intensity)
}
if len(hillas_dict) < 2:
continue
for count, reco_method in enumerate(reconstructors):
if reco_method is HillasReconstructor:
reconstructor = HillasReconstructor(subarray)
reconstructor(event)
event.dl2.stereo.geometry["HillasReconstructor"].alt.to(u.deg)
event.dl2.stereo.geometry["HillasReconstructor"].az.to(u.deg)
event.dl2.stereo.geometry["HillasReconstructor"].core_x.to(u.m)
assert event.dl2.stereo.geometry[
"HillasReconstructor"].is_valid
else:
reconstructor = reco_method()
try:
reconstructor_out = reconstructor.predict(
hillas_dict,
source.subarray,
array_pointing,
telescope_pointings,
)
except InvalidWidthException:
continue
reconstructor_out.alt.to(u.deg)
reconstructor_out.az.to(u.deg)
reconstructor_out.core_x.to(u.m)
assert reconstructor_out.is_valid
def test_array_display():
""" check that we can do basic array display functionality """
from ctapipe.visualization.mpl_array import ArrayDisplay
from ctapipe.image import timing_parameters
from ctapipe.containers import (
ArrayEventContainer,
DL1Container,
DL1CameraContainer,
ImageParametersContainer,
CoreParametersContainer,
)
# build a test subarray:
tels = dict()
tel_pos = dict()
for ii, pos in enumerate([[0, 0, 0], [100, 0, 0], [-100, 0, 0]] * u.m):
tels[ii + 1] = TelescopeDescription.from_name("MST", "NectarCam")
tel_pos[ii + 1] = pos
sub = SubarrayDescription(name="TestSubarray",
tel_positions=tel_pos,
tel_descriptions=tels)
# Create a fake event containing telescope-wise information about
# the image directions projected on the ground
event = ArrayEventContainer()
event.dl1 = DL1Container()
event.dl1.tel = {1: DL1CameraContainer(), 2: DL1CameraContainer()}
event.dl1.tel[1].parameters = ImageParametersContainer()
event.dl1.tel[2].parameters = ImageParametersContainer()
event.dl1.tel[2].parameters.core = CoreParametersContainer()
event.dl1.tel[1].parameters.core = CoreParametersContainer()
event.dl1.tel[1].parameters.core.psi = u.Quantity(2.0, unit=u.deg)
event.dl1.tel[2].parameters.core.psi = u.Quantity(1.0, unit=u.deg)
ad = ArrayDisplay(subarray=sub)
ad.set_vector_rho_phi(1 * u.m, 90 * u.deg)
# try setting a value
vals = np.ones(sub.num_tels)
ad.values = vals
assert (vals == ad.values).all()
# test UV field ...
# ...with colors by telescope type
ad.set_vector_uv(np.array([1, 2, 3]) * u.m, np.array([1, 2, 3]) * u.m)
# ...with scalar color
ad.set_vector_uv(np.array([1, 2, 3]) * u.m, np.array([1, 2, 3]) * u.m, c=3)
geom = CameraGeometry.from_name("LSTCam")
rot_angle = 20 * u.deg
hillas = CameraHillasParametersContainer(x=0 * u.m,
y=0 * u.m,
psi=rot_angle)
# test using hillas params CameraFrame:
hillas_dict = {
1: CameraHillasParametersContainer(length=100.0 * u.m, psi=90 * u.deg),
2: CameraHillasParametersContainer(length=20000 * u.cm, psi="95deg"),
}
grad = 2
intercept = 1
timing_rot20 = timing_parameters(
geom,
image=np.ones(geom.n_pixels),
peak_time=intercept + grad * geom.pix_x.value,
hillas_parameters=hillas,
cleaning_mask=np.ones(geom.n_pixels, dtype=bool),
)
gradient_dict = {1: timing_rot20.slope.value, 2: timing_rot20.slope.value}
core_dict = {
tel_id: dl1.parameters.core.psi
for tel_id, dl1 in event.dl1.tel.items()
}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
core_dict=core_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300)
# test using hillas params for divergent pointing in telescopeframe:
hillas_dict = {
1:
HillasParametersContainer(fov_lon=1.0 * u.deg,
fov_lat=1.0 * u.deg,
length=1.0 * u.deg,
psi=90 * u.deg),
2:
HillasParametersContainer(fov_lon=1.0 * u.deg,
fov_lat=1.0 * u.deg,
length=1.0 * u.deg,
psi=95 * u.deg),
}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
core_dict=core_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300)
# test using hillas params for parallel pointing in telescopeframe:
hillas_dict = {
1:
HillasParametersContainer(fov_lon=1.0 * u.deg,
fov_lat=1.0 * u.deg,
length=1.0 * u.deg,
psi=90 * u.deg),
2:
HillasParametersContainer(fov_lon=1.0 * u.deg,
fov_lat=1.0 * u.deg,
length=1.0 * u.deg,
psi=95 * u.deg),
}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
core_dict=core_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
# test negative time_gradients
gradient_dict = {1: -0.03, 2: -0.02}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
core_dict=core_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
# and very small
gradient_dict = {1: 0.003, 2: 0.002}
ad.set_vector_hillas(
hillas_dict=hillas_dict,
core_dict=core_dict,
length=500,
time_gradient=gradient_dict,
angle_offset=0 * u.deg,
)
# Test background contour
ad.background_contour(
x=np.array([0, 1, 2]),
y=np.array([0, 1, 2]),
background=np.array([[0, 1, 2], [0, 1, 2], [0, 1, 2]]),
)
ad.set_line_hillas(hillas_dict=hillas_dict, core_dict=core_dict, range=300)
ad.add_labels()
ad.remove_labels()
def _generate_events(self):
"""
Yield ArrayEventContainer to iterate through events.
"""
data = ArrayEventContainer()
# Maybe take some other metadata, but there are still some 'unknown'
# written out by the stage1 tool
data.meta["origin"] = self.file_.root._v_attrs["CTA PROCESS TYPE"]
data.meta["input_url"] = self.input_url
data.meta["max_events"] = self.max_events
if DataLevel.DL1_IMAGES in self.datalevels:
image_iterators = {
tel.name: self.file_.root.dl1.event.telescope.images[
tel.name].iterrows()
for tel in self.file_.root.dl1.event.telescope.images
}
if self.has_simulated_dl1:
simulated_image_iterators = {
tel.name:
self.file_.root.simulation.event.telescope.images[
tel.name].iterrows()
for tel in
self.file_.root.simulation.event.telescope.images
}
if DataLevel.DL1_PARAMETERS in self.datalevels:
param_readers = {
tel.name: HDF5TableReader(self.file_).read(
f"/dl1/event/telescope/parameters/{tel.name}",
containers=[
HillasParametersContainer(),
TimingParametersContainer(),
LeakageContainer(),
ConcentrationContainer(),
MorphologyContainer(),
IntensityStatisticsContainer(),
PeakTimeStatisticsContainer(),
],
prefixes=True,
)
for tel in self.file_.root.dl1.event.telescope.parameters
}
if self.has_simulated_dl1:
simulated_param_readers = {
tel.name: HDF5TableReader(self.file_).read(
f"/simulation/event/telescope/parameters/{tel.name}",
containers=[
HillasParametersContainer(),
LeakageContainer(),
ConcentrationContainer(),
MorphologyContainer(),
IntensityStatisticsContainer(),
],
prefixes=True,
)
for tel in self.file_.root.dl1.event.telescope.parameters
}
if self.is_simulation:
# simulated shower wide information
mc_shower_reader = HDF5TableReader(self.file_).read(
"/simulation/event/subarray/shower",
SimulatedShowerContainer(),
prefixes="true",
)
# Setup iterators for the array events
events = HDF5TableReader(self.file_).read(
"/dl1/event/subarray/trigger",
[TriggerContainer(), EventIndexContainer()])
array_pointing_finder = IndexFinder(
self.file_.root.dl1.monitoring.subarray.pointing.col("time"))
tel_pointing_finder = {
tel.name: IndexFinder(tel.col("time"))
for tel in self.file_.root.dl1.monitoring.telescope.pointing
}
for counter, (trigger, index) in enumerate(events):
data.dl1.tel.clear()
data.simulation.tel.clear()
data.pointing.tel.clear()
data.trigger.tel.clear()
data.count = counter
data.trigger = trigger
data.index = index
data.trigger.tels_with_trigger = self.subarray.tel_mask_to_tel_ids(
data.trigger.tels_with_trigger)
# Maybe there is a simpler way to do this
# Beware: tels_with_trigger contains all triggered telescopes whereas
# the telescope trigger table contains only the subset of
# allowed_tels given during the creation of the dl1 file
for i in self.file_.root.dl1.event.telescope.trigger.where(
f"(obs_id=={data.index.obs_id}) & (event_id=={data.index.event_id})"
):
if self.allowed_tels and i["tel_id"] not in self.allowed_tels:
continue
if self.datamodel_version == "v1.0.0":
data.trigger.tel[
i["tel_id"]].time = i["telescopetrigger_time"]
else:
data.trigger.tel[i["tel_id"]].time = i["time"]
self._fill_array_pointing(data, array_pointing_finder)
self._fill_telescope_pointing(data, tel_pointing_finder)
if self.is_simulation:
data.simulation.shower = next(mc_shower_reader)
for tel in data.trigger.tel.keys():
if self.allowed_tels and tel not in self.allowed_tels:
continue
if self.has_simulated_dl1:
simulated = data.simulation.tel[tel]
dl1 = data.dl1.tel[tel]
if DataLevel.DL1_IMAGES in self.datalevels:
if f"tel_{tel:03d}" not in image_iterators.keys():
logger.debug(f"Triggered telescope {tel} is missing "
"from the image table.")
continue
image_row = next(image_iterators[f"tel_{tel:03d}"])
dl1.image = image_row["image"]
dl1.peak_time = image_row["peak_time"]
dl1.image_mask = image_row["image_mask"]
if self.has_simulated_dl1:
if f"tel_{tel:03d}" not in simulated_image_iterators.keys(
):
logger.warning(
f"Triggered telescope {tel} is missing "
"from the simulated image table, but was present at the "
"reconstructed image table.")
continue
simulated_image_row = next(
simulated_image_iterators[f"tel_{tel:03d}"])
simulated.true_image = simulated_image_row[
"true_image"]
if DataLevel.DL1_PARAMETERS in self.datalevels:
if f"tel_{tel:03d}" not in param_readers.keys():
logger.debug(f"Triggered telescope {tel} is missing "
"from the parameters table.")
continue
# Is there a smarter way to unpack this?
# Best would probbaly be if we could directly read
# into the ImageParametersContainer
params = next(param_readers[f"tel_{tel:03d}"])
dl1.parameters = ImageParametersContainer(
hillas=params[0],
timing=params[1],
leakage=params[2],
concentration=params[3],
morphology=params[4],
intensity_statistics=params[5],
peak_time_statistics=params[6],
)
if self.has_simulated_dl1:
if f"tel_{tel:03d}" not in param_readers.keys():
logger.debug(
f"Triggered telescope {tel} is missing "
"from the simulated parameters table, but was "
"present at the reconstructed parameters table."
)
continue
simulated_params = next(
simulated_param_readers[f"tel_{tel:03d}"])
simulated.true_parameters = ImageParametersContainer(
hillas=simulated_params[0],
leakage=simulated_params[1],
concentration=simulated_params[2],
morphology=simulated_params[3],
intensity_statistics=simulated_params[4],
)
yield data
def test_CameraFrame_against_TelescopeFrame(filename):
input_file = get_dataset_path(
"gamma_divergent_LaPalma_baseline_20Zd_180Az_prod3_test.simtel.gz")
source = SimTelEventSource(input_file, max_events=10)
calib = CameraCalibrator(subarray=source.subarray)
reconstructor = HillasReconstructor(source.subarray)
reconstructed_events = 0
for event in source:
calib(event)
# make a copy of the calibrated event for the camera frame case
# later we clean and paramretrize the 2 events in the same way
# but in 2 different frames to check they return compatible results
event_camera_frame = deepcopy(event)
telescope_pointings = {}
hillas_dict_camera_frame = {}
hillas_dict_telescope_frame = {}
for tel_id, dl1 in event.dl1.tel.items():
event_camera_frame.dl1.tel[
tel_id].parameters = ImageParametersContainer()
event.dl1.tel[tel_id].parameters = ImageParametersContainer()
# this is needed only here to transform the camera geometries
telescope_pointings[tel_id] = SkyCoord(
alt=event.pointing.tel[tel_id].altitude,
az=event.pointing.tel[tel_id].azimuth,
frame=AltAz(),
)
geom_camera_frame = source.subarray.tel[tel_id].camera.geometry
# this could be done also out of this loop,
# but in case of real data each telescope would have a
# different telescope_pointing
geom_telescope_frame = geom_camera_frame.transform_to(
TelescopeFrame(telescope_pointing=telescope_pointings[tel_id]))
mask = tailcuts_clean(geom_telescope_frame,
dl1.image,
picture_thresh=10.0,
boundary_thresh=5.0)
try:
moments_camera_frame = hillas_parameters(
geom_camera_frame[mask], dl1.image[mask])
moments_telescope_frame = hillas_parameters(
geom_telescope_frame[mask], dl1.image[mask])
if (moments_camera_frame.width.value >
0) and (moments_telescope_frame.width.value > 0):
event_camera_frame.dl1.tel[
tel_id].parameters.hillas = moments_camera_frame
dl1.parameters.hillas = moments_telescope_frame
hillas_dict_camera_frame[tel_id] = moments_camera_frame
hillas_dict_telescope_frame[
tel_id] = moments_telescope_frame
else:
continue
except HillasParameterizationError as e:
print(e)
continue
if (len(hillas_dict_camera_frame) >
2) and (len(hillas_dict_telescope_frame) > 2):
reconstructor(event_camera_frame)
reconstructor(event)
reconstructed_events += 1
else: # this event was not good enough to be tested on
continue
# Compare old approach with new approach
result_camera_frame = event_camera_frame.dl2.stereo.geometry[
"HillasReconstructor"]
result_telescope_frame = event.dl2.stereo.geometry[
"HillasReconstructor"]
assert result_camera_frame.is_valid
assert result_telescope_frame.is_valid
for field in event.dl2.stereo.geometry["HillasReconstructor"].as_dict(
):
C = np.asarray(result_camera_frame.as_dict()[field])
T = np.asarray(result_telescope_frame.as_dict()[field])
assert (np.isclose(C, T, rtol=1e-03, atol=1e-03,
equal_nan=True)).all()
assert reconstructed_events > 0 # check that we reconstruct at least 1 event
def test_reconstruction_against_simulation(
subarray_and_event_gamma_off_axis_500_gev):
"""Reconstruction is here done only in the TelescopeFrame,
since the previous tests test already for the compatibility between
frames"""
# 4-LST bright event already calibrated
# we'll clean it and parametrize it again in the TelescopeFrame
subarray, event = subarray_and_event_gamma_off_axis_500_gev
# define reconstructor
reconstructor = HillasReconstructor(subarray)
hillas_dict = {}
telescope_pointings = {}
for tel_id, dl1 in event.dl1.tel.items():
telescope_pointings[tel_id] = SkyCoord(
alt=event.pointing.tel[tel_id].altitude,
az=event.pointing.tel[tel_id].azimuth,
frame=AltAz(),
)
geom_CameraFrame = subarray.tel[tel_id].camera.geometry
# this could be done also out of this loop,
# but in case of real data each telescope would have a
# different telescope_pointing
geom_TelescopeFrame = geom_CameraFrame.transform_to(
TelescopeFrame(telescope_pointing=telescope_pointings[tel_id]))
mask = tailcuts_clean(
geom_TelescopeFrame,
dl1.image,
picture_thresh=5.0,
boundary_thresh=2.5,
keep_isolated_pixels=False,
min_number_picture_neighbors=2,
)
try:
hillas_dict[tel_id] = hillas_parameters(geom_TelescopeFrame[mask],
dl1.image[mask])
# the original event is created from a
# pytest fixture with "session" scope, so it's always the same
# and if we used the same event we would overwrite the image
# parameters for the next tests, thus causing their failure
test_event = deepcopy(event)
test_event.dl1.tel[tel_id].parameters = ImageParametersContainer()
test_event.dl1.tel[tel_id].parameters.hillas = hillas_dict[tel_id]
except HillasParameterizationError as e:
print(e)
continue
# Get shower geometry
reconstructor(event)
# get the result from the correct DL2 container
result = event.dl2.stereo.geometry["HillasReconstructor"]
# get the reconstructed coordinates in the sky
reco_coord = SkyCoord(alt=result.alt, az=result.az, frame=AltAz())
# get the simulated coordinates in the sky
true_coord = SkyCoord(alt=event.simulation.shower.alt,
az=event.simulation.shower.az,
frame=AltAz())
# check that we are not more far than 0.1 degrees
assert reco_coord.separation(true_coord) < 0.1 * u.deg
def test_invalid_events(subarray_and_event_gamma_off_axis_500_gev):
"""
The HillasReconstructor is supposed to fail
in these cases:
- less than two teleskopes
- any width is NaN
- any width is 0
This test takes 1 shower from a test simtel file and modifies a-posteriori
some hillas dictionaries to make it non-reconstructable.
It is supposed to fail if no Exception or another Exception gets thrown.
"""
# 4-LST bright event already calibrated
# we'll clean it and parametrize it again in the TelescopeFrame
subarray, event = subarray_and_event_gamma_off_axis_500_gev
tel_azimuth = {}
tel_altitude = {}
#source = EventSource(filename, max_events=1)
#subarray = source.subarray
calib = CameraCalibrator(subarray)
fit = HillasReconstructor(subarray)
#for event in source:
calib(event)
hillas_dict = {}
for tel_id, dl1 in event.dl1.tel.items():
geom = subarray.tel[tel_id].camera.geometry
tel_azimuth[tel_id] = event.pointing.tel[tel_id].azimuth
tel_altitude[tel_id] = event.pointing.tel[tel_id].altitude
mask = tailcuts_clean(geom,
dl1.image,
picture_thresh=10.0,
boundary_thresh=5.0)
dl1.parameters = ImageParametersContainer()
try:
moments = hillas_parameters(geom[mask], dl1.image[mask])
hillas_dict[tel_id] = moments
dl1.parameters.hillas = moments
except HillasParameterizationError:
dl1.parameters.hillas = HillasParametersContainer()
continue
# copy event container to modify it
event_copy = deepcopy(event)
# overwrite all image parameters but the last one with dummy ones
for tel_id in list(event_copy.dl1.tel.keys())[:-1]:
event_copy.dl1.tel[
tel_id].parameters.hillas = HillasParametersContainer()
fit(event_copy)
assert event_copy.dl2.stereo.geometry[
"HillasReconstructor"].is_valid is False
# Now use the original event, but overwrite the last width to 0
event.dl1.tel[tel_id].parameters.hillas.width = 0 * u.m
fit(event)
assert event.dl2.stereo.geometry["HillasReconstructor"].is_valid is False
# Now use the original event, but overwrite the last width to NaN
event.dl1.tel[tel_id].parameters.hillas.width = np.nan * u.m
fit(event)
assert event.dl2.stereo.geometry["HillasReconstructor"].is_valid is False