def test_FitGammaHillas():
'''
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• GreatCircle creation
• direction fit
• position fit
in the end, proper units in the output are asserted '''
filename = get_dataset("gamma_test.simtel.gz")
fit = HillasReconstructor()
cam_geom = {}
tel_phi = {}
tel_theta = {}
source = hessio_event_source(filename)
for event in source:
hillas_dict = {}
for tel_id in event.dl0.tels_with_data:
if tel_id not in cam_geom:
cam_geom[tel_id] = CameraGeometry.guess(
event.inst.pixel_pos[tel_id][0],
event.inst.pixel_pos[tel_id][1],
event.inst.optical_foclen[tel_id])
tel_phi[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
tel_theta[tel_id] = (np.pi/2-event.mc.tel[tel_id].altitude_raw)*u.rad
pmt_signal = event.r0.tel[tel_id].adc_sums[0]
mask = tailcuts_clean(cam_geom[tel_id], pmt_signal,
picture_thresh=10., boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(event.inst.pixel_pos[tel_id][0],
event.inst.pixel_pos[tel_id][1],
pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2: continue
fit_result = fit.predict(hillas_dict, event.inst, tel_phi, tel_theta)
print(fit_result)
fit_result.alt.to(u.deg)
fit_result.az.to(u.deg)
fit_result.core_x.to(u.m)
assert fit_result.is_valid
return
def test_reconstruction():
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted """
filename = get_dataset_path("gamma_test.simtel.gz")
fit = HillasReconstructor()
tel_azimuth = {}
tel_altitude = {}
source = EventSourceFactory.produce(
input_url=filename,
product='HESSIOEventSource',
)
for event in source:
hillas_dict = {}
for tel_id in event.dl0.tels_with_data:
geom = event.inst.subarray.tel[tel_id].camera
tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
pmt_signal = event.r0.tel[tel_id].image[0]
mask = tailcuts_clean(geom,
pmt_signal,
picture_thresh=10.,
boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(geom, pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2:
continue
fit_result = fit.predict(hillas_dict, event.inst, tel_azimuth,
tel_altitude)
print(fit_result)
fit_result.alt.to(u.deg)
fit_result.az.to(u.deg)
fit_result.core_x.to(u.m)
assert fit_result.is_valid
return
def test_FitGammaHillas():
'''
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• GreatCircle creation
• direction fit
• position fit
in the end, proper units in the output are asserted '''
filename = get_dataset("gamma_test.simtel.gz")
fit = HillasReconstructor()
tel_phi = {}
tel_theta = {}
source = hessio_event_source(filename)
for event in source:
hillas_dict = {}
for tel_id in event.dl0.tels_with_data:
geom = event.inst.subarray.tel[tel_id].camera
tel_phi[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
tel_theta[tel_id] = (np.pi / 2 -
event.mc.tel[tel_id].altitude_raw) * u.rad
pmt_signal = event.r0.tel[tel_id].image[0]
mask = tailcuts_clean(geom,
pmt_signal,
picture_thresh=10.,
boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(geom, pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2: continue
fit_result = fit.predict(hillas_dict, event.inst, tel_phi, tel_theta)
print(fit_result)
fit_result.alt.to(u.deg)
fit_result.az.to(u.deg)
fit_result.core_x.to(u.m)
assert fit_result.is_valid
return
def test_reconstruction():
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted """
filename = get_dataset_path("gamma_test.simtel.gz")
fit = HillasReconstructor()
tel_azimuth = {}
tel_altitude = {}
source = EventSourceFactory.produce(input_url=filename)
for event in source:
hillas_dict = {}
for tel_id in event.dl0.tels_with_data:
geom = event.inst.subarray.tel[tel_id].camera
tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
pmt_signal = event.r0.tel[tel_id].image[0]
mask = tailcuts_clean(geom, pmt_signal,
picture_thresh=10., boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(geom, pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2:
continue
fit_result = fit.predict(hillas_dict, event.inst, tel_azimuth, tel_altitude)
print(fit_result)
fit_result.alt.to(u.deg)
fit_result.az.to(u.deg)
fit_result.core_x.to(u.m)
assert fit_result.is_valid
def test_reconstruction():
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted """
filename = get_dataset_path("gamma_test_large.simtel.gz")
source = EventSource(filename, max_events=10)
calib = CameraCalibrator(subarray=source.subarray)
horizon_frame = AltAz()
reconstructed_events = 0
for event in source:
calib(event)
mc = event.simulation.shower
array_pointing = SkyCoord(az=mc.az, alt=mc.alt, frame=horizon_frame)
hillas_dict = {}
telescope_pointings = {}
for tel_id, dl1 in event.dl1.tel.items():
geom = source.subarray.tel[tel_id].camera.geometry
telescope_pointings[tel_id] = SkyCoord(
alt=event.pointing.tel[tel_id].altitude,
az=event.pointing.tel[tel_id].azimuth,
frame=horizon_frame,
)
mask = tailcuts_clean(geom,
dl1.image,
picture_thresh=10.0,
boundary_thresh=5.0)
try:
moments = hillas_parameters(geom[mask], dl1.image[mask])
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2:
continue
else:
reconstructed_events += 1
# The three reconstructions below gives the same results
fit = HillasReconstructor()
fit_result_parall = fit.predict(hillas_dict, source.subarray,
array_pointing)
fit = HillasReconstructor()
fit_result_tel_point = fit.predict(hillas_dict, source.subarray,
array_pointing, telescope_pointings)
for key in fit_result_parall.keys():
print(key, fit_result_parall[key], fit_result_tel_point[key])
fit_result_parall.alt.to(u.deg)
fit_result_parall.az.to(u.deg)
fit_result_parall.core_x.to(u.m)
assert fit_result_parall.is_valid
assert reconstructed_events > 0
def test_invalid_events():
"""
The HillasReconstructor is supposed to fail
in these cases:
- less than two teleskopes
- any width is NaN
- any width is 0
This test uses the same sample simtel file as
test_reconstruction(). As there are no invalid events in this
file, multiple hillas_dicts are constructed to make sure
Exceptions get thrown in the mentioned edge cases.
Test will fail if no Exception or another Exception gets thrown."""
filename = get_dataset_path("gamma_test_large.simtel.gz")
fit = HillasReconstructor()
tel_azimuth = {}
tel_altitude = {}
source = EventSource(filename, max_events=10)
subarray = source.subarray
calib = CameraCalibrator(subarray)
for event in source:
calib(event)
hillas_dict = {}
for tel_id, dl1 in event.dl1.tel.items():
geom = source.subarray.tel[tel_id].camera.geometry
tel_azimuth[tel_id] = event.pointing.tel[tel_id].azimuth
tel_altitude[tel_id] = event.pointing.tel[tel_id].altitude
mask = tailcuts_clean(geom,
dl1.image,
picture_thresh=10.0,
boundary_thresh=5.0)
try:
moments = hillas_parameters(geom[mask], dl1.image[mask])
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
continue
# construct a dict only containing the last telescope events
# (#telescopes < 2)
hillas_dict_only_one_tel = dict()
hillas_dict_only_one_tel[tel_id] = hillas_dict[tel_id]
with pytest.raises(TooFewTelescopesException):
fit.predict(hillas_dict_only_one_tel, subarray, tel_azimuth,
tel_altitude)
# construct a hillas dict with the width of the last event set to 0
# (any width == 0)
hillas_dict_zero_width = hillas_dict.copy()
hillas_dict_zero_width[tel_id]["width"] = 0 * u.m
with pytest.raises(InvalidWidthException):
fit.predict(hillas_dict_zero_width, subarray, tel_azimuth,
tel_altitude)
# construct a hillas dict with the width of the last event set to np.nan
# (any width == nan)
hillas_dict_nan_width = hillas_dict.copy()
hillas_dict_zero_width[tel_id]["width"] = np.nan * u.m
with pytest.raises(InvalidWidthException):
fit.predict(hillas_dict_nan_width, subarray, tel_azimuth,
tel_altitude)
def test_reconstruction():
"""
a test of the complete fit procedure on one event including:
• tailcut cleaning
• hillas parametrisation
• HillasPlane creation
• direction fit
• position fit
in the end, proper units in the output are asserted """
filename = get_dataset_path("gamma_test_large.simtel.gz")
source = event_source(filename, max_events=10)
horizon_frame = AltAz()
reconstructed_events = 0
for event in source:
array_pointing = SkyCoord(
az=event.mc.az,
alt=event.mc.alt,
frame=horizon_frame
)
hillas_dict = {}
telescope_pointings = {}
for tel_id in event.dl0.tels_with_data:
geom = event.inst.subarray.tel[tel_id].camera
telescope_pointings[tel_id] = SkyCoord(alt=event.mc.tel[tel_id].altitude_raw * u.rad,
az=event.mc.tel[tel_id].azimuth_raw * u.rad,
frame=horizon_frame)
pmt_signal = event.r0.tel[tel_id].waveform[0].sum(axis=1)
mask = tailcuts_clean(geom, pmt_signal,
picture_thresh=10., boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(geom, pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
print(e)
continue
if len(hillas_dict) < 2:
continue
else:
reconstructed_events += 1
# The three reconstructions below gives the same results
fit = HillasReconstructor()
fit_result_parall = fit.predict(hillas_dict, event.inst, array_pointing)
fit = HillasReconstructor()
fit_result_tel_point = fit.predict(hillas_dict, event.inst, array_pointing, telescope_pointings)
for key in fit_result_parall.keys():
print(key, fit_result_parall[key], fit_result_tel_point[key])
fit_result_parall.alt.to(u.deg)
fit_result_parall.az.to(u.deg)
fit_result_parall.core_x.to(u.m)
assert fit_result_parall.is_valid
assert reconstructed_events > 0
def test_invalid_events():
"""
The HillasReconstructor is supposed to fail
in these cases:
- less than two teleskopes
- any width is NaN
- any width is 0
This test uses the same sample simtel file as
test_reconstruction(). As there are no invalid events in this
file, multiple hillas_dicts are constructed to make sure
Exceptions get thrown in the mentioned edge cases.
Test will fail if no Exception or another Exception gets thrown."""
filename = get_dataset_path("gamma_test_large.simtel.gz")
fit = HillasReconstructor()
tel_azimuth = {}
tel_altitude = {}
source = event_source(filename, max_events=10)
for event in source:
hillas_dict = {}
for tel_id in event.dl0.tels_with_data:
geom = event.inst.subarray.tel[tel_id].camera
tel_azimuth[tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
tel_altitude[tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
pmt_signal = event.r0.tel[tel_id].waveform[0].sum(axis=1)
mask = tailcuts_clean(geom, pmt_signal,
picture_thresh=10., boundary_thresh=5.)
pmt_signal[mask == 0] = 0
try:
moments = hillas_parameters(geom, pmt_signal)
hillas_dict[tel_id] = moments
except HillasParameterizationError as e:
continue
# construct a dict only containing the last telescope events
# (#telescopes < 2)
hillas_dict_only_one_tel = dict()
hillas_dict_only_one_tel[tel_id] = hillas_dict[tel_id]
with pytest.raises(TooFewTelescopesException):
fit.predict(hillas_dict_only_one_tel, event.inst, tel_azimuth, tel_altitude)
# construct a hillas dict with the width of the last event set to 0
# (any width == 0)
hillas_dict_zero_width = hillas_dict.copy()
hillas_dict_zero_width[tel_id]['width'] = 0 * u.m
with pytest.raises(InvalidWidthException):
fit.predict(hillas_dict_zero_width, event.inst, tel_azimuth, tel_altitude)
# construct a hillas dict with the width of the last event set to np.nan
# (any width == nan)
hillas_dict_nan_width = hillas_dict.copy()
hillas_dict_zero_width[tel_id]['width'] = np.nan * u.m
with pytest.raises(InvalidWidthException):
fit.predict(hillas_dict_nan_width, event.inst, tel_azimuth, tel_altitude)
class EventPreparer:
"""Class which loop on events and returns results stored in container.
The Class has several purposes. First of all, it prepares the images of the
event that will be further use for reconstruction by applying calibration,
cleaning and selection. Then, it reconstructs the geometry of the event and
then returns image (e.g. Hillas parameters)and event information
(e.g. results of the reconstruction).
Parameters
----------
config: dict
Configuration with analysis parameters
mode: str
Mode of the reconstruction, e.g. tail or wave
event_cutflow: ctapipe.utils.CutFlow
Statistic of events processed
image_cutflow: ctapipe.utils.CutFlow
Statistic of images processed
Returns: dict
Dictionnary of results
"""
def __init__(self, config, mode, event_cutflow=None, image_cutflow=None):
"""Initiliaze an EventPreparer object."""
# Cleaning for reconstruction
self.cleaner_reco = ImageCleaner( # for reconstruction
config=config["ImageCleaning"]["biggest"],
mode=mode)
# Cleaning for energy/score estimation
# Add possibility to force energy/score cleaning with tailcut analysis
force_mode = mode
try:
if config["General"]["force_tailcut_for_extended_cleaning"] is True:
force_mode = config["General"]["force_mode"]
print("> Activate force-mode for cleaning!!!!")
except:
pass # force_mode = mode
self.cleaner_extended = ImageCleaner( # for energy/score estimation
config=config["ImageCleaning"]["extended"],
mode=force_mode)
# Image book keeping
self.image_cutflow = image_cutflow or CutFlow("ImageCutFlow")
# Add quality cuts on images
charge_bounds = config["ImageSelection"]["charge"]
npix_bounds = config["ImageSelection"]["pixel"]
ellipticity_bounds = config["ImageSelection"]["ellipticity"]
nominal_distance_bounds = config["ImageSelection"]["nominal_distance"]
self.camera_radius = {
"LSTCam": 1.126,
"NectarCam": 1.126,
} # Average between max(xpix) and max(ypix), in meters
self.image_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min pixel", lambda s: np.count_nonzero(s) < npix_bounds[0]),
("min charge", lambda x: x < charge_bounds[0]),
# ("poor moments", lambda m: m.width <= 0 or m.length <= 0 or np.isnan(m.width) or np.isnan(m.length)),
# TBC, maybe we loose events without nan conditions
("poor moments", lambda m: m.width <= 0 or m.length <= 0),
(
"bad ellipticity",
lambda m: (m.width / m.length) < ellipticity_bounds[0] or
(m.width / m.length) > ellipticity_bounds[-1],
),
# ("close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * 1.12949101073069946))
# in meter
(
"close to the edge",
lambda m, cam_id: m.r.value >
(nominal_distance_bounds[-1] * self.camera_radius[cam_id]),
), # in meter
]))
# configuration for the camera calibrator
# modifies the integration window to be more like in MARS
# JLK, only for LST!!!!
cfg = Config()
cfg["ChargeExtractorFactory"]["window_width"] = 5
cfg["ChargeExtractorFactory"]["window_shift"] = 2
extractor = LocalPeakWindowSum(config=cfg)
self.calib = CameraCalibrator(config=cfg, image_extractor=extractor)
# Reconstruction
self.shower_reco = HillasReconstructor()
# Event book keeping
self.event_cutflow = event_cutflow or CutFlow("EventCutFlow")
# Add cuts on events
min_ntel = config["Reconstruction"]["min_tel"]
self.event_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min2Tels trig", lambda x: x < min_ntel),
("min2Tels reco", lambda x: x < min_ntel),
("direction nan", lambda x: x.is_valid == False),
]))
def prepare_event(self, source, return_stub=False, save_images=False):
for event in source:
self.event_cutflow.count("noCuts")
if self.event_cutflow.cut("min2Tels trig",
len(event.dl0.tels_with_data)):
if return_stub:
yield stub(event)
else:
continue
self.calib(event)
# telescope loop
tot_signal = 0
dl1_phe_image = None
mc_phe_image = None
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for direction reconstruction
hillas_dict = {} # for discrimination
n_tels = {
"tot": len(event.dl0.tels_with_data),
"LST_LST_LSTCam": 0,
"MST_MST_NectarCam": 0,
"SST": 0, # add later correct names when testing on Paranal
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
# Compute impact parameter in tilt system
run_array_direction = event.mcheader.run_array_direction
az, alt = run_array_direction[0], run_array_direction[1]
ground_frame = GroundFrame()
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count("noCuts")
camera = event.inst.subarray.tel[tel_id].camera
# count the current telescope according to its size
tel_type = str(event.inst.subarray.tel[tel_id])
# use ctapipe's functionality to get the calibrated image
pmt_signal = event.dl1.tel[tel_id].image
# Save the calibrated image after the gain has been chosen
# automatically by ctapipe, together with the simulated one
if save_images is True:
dl1_phe_image = pmt_signal
mc_phe_image = event.mc.tel[tel_id].photo_electron_image
if self.cleaner_reco.mode == "tail": # tail uses only ctapipe
# Cleaning used for direction reconstruction
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
# find all islands using this cleaning
num_islands, labels = number_of_islands(camera, mask_reco)
if num_islands == 1: # if only ONE islands is left ...
# ...use directly the old mask and reduce dimensions
# to make Hillas parametrization faster
camera_biggest = camera[mask_reco]
image_biggest = image_biggest[mask_reco]
elif num_islands > 1: # if more islands survived..
# ...find the biggest one
mask_biggest = largest_island(labels)
# and also reduce dimensions
camera_biggest = camera[mask_biggest]
image_biggest = image_biggest[mask_biggest]
else: # if no islands survived use old camera and image
camera_biggest = camera
# Cleaning used for score/energy estimation
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
# find all islands with this cleaning
# we will also register how many have been found
n_cluster_dict[tel_id], labels = number_of_islands(
camera, mask_extended)
# NOTE: the next check shouldn't be necessary if we keep
# all the isolated pixel clusters, but for now the
# two cleanings are set the same in analysis.yml because
# the performance of the extended one has never been really
# studied in model estimation.
# (This is a nice way to ask for volunteers :P)
# if some islands survived
if num_islands > 0:
# keep all of them and reduce dimensions
camera_extended = camera[mask_extended]
image_extended = image_extended[mask_extended]
else: # otherwise continue with the old camera and image
camera_extended = camera
# could this go into `hillas_parameters` ...?
# this is basically the charge of ALL islands
# not calculated later by the Hillas parametrization!
max_signals[tel_id] = np.max(image_extended)
else: # for wavelets we stick to old pywi-cta code
try: # "try except FileNotFoundError" not clear to me, but for now it stays...
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.cam_id)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.cam_id)
# Image for score/energy estimation (with clusters)
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
# This last part was outside the pywi-cta block
# before, but is indeed part of it because it uses
# pywi-cta functions in the "extended" case
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.cam_id)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
except FileNotFoundError as e: # JLK, WHAT?
print(e)
continue
# ==============================================================
# Apply some selection
if self.image_cutflow.cut("min pixel", image_biggest):
continue
if self.image_cutflow.cut("min charge", np.sum(image_biggest)):
continue
# do the hillas reconstruction of the images
# QUESTION should this change in numpy behaviour be done here
# or within `hillas_parameters` itself?
# JLK: make selection on biggest cluster
with np.errstate(invalid="raise", divide="raise"):
try:
moments_reco = hillas_parameters(
camera_biggest,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera_extended, image_extended
) # for discrimination and energy reconstruction
# if width and/or length are zero (e.g. when there is
# only only one pixel or when all pixel are exactly
# in one row), the parametrisation
# won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_reco):
continue
if self.image_cutflow.cut("close to the edge",
moments_reco, camera.cam_id):
continue
if self.image_cutflow.cut("bad ellipticity",
moments_reco):
continue
except (FloatingPointError,
hillas.HillasParameterizationError):
continue
point_azimuth_dict[
tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
point_altitude_dict[
tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
n_tels[tel_type] += 1
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0])
tot_signal += moments.intensity
n_tels["reco"] = len(hillas_dict_reco)
n_tels["discri"] = len(hillas_dict)
if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]):
if return_stub:
yield stub(event)
else:
continue
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Reconstruction results
reco_result = self.shower_reco.predict(
hillas_dict_reco,
event.inst,
SkyCoord(alt=alt, az=az, frame="altaz"),
{
tel_id: SkyCoord(
alt=point_altitude_dict[tel_id],
az=point_azimuth_dict[tel_id],
frame="altaz",
) # cycle only on tels which still have an image
for tel_id in point_altitude_dict.keys()
},
)
# Impact parameter for energy estimation (/ tel)
subarray = event.inst.subarray
for tel_id in hillas_dict.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# Should be better handled (tilted frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_ground.x - tel_ground.x)**2 +
(core_ground.y - tel_ground.y)**2)
except Exception as e:
print("exception in reconstruction:", e)
raise
if return_stub:
yield stub(event)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if return_stub:
yield stub(event)
else:
continue
yield PreparedEvent(
event=event,
dl1_phe_image=dl1_phe_image,
mc_phe_image=mc_phe_image,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
n_tels=n_tels,
tot_signal=tot_signal,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
)
class EventPreparer:
"""
Class which loop on events and returns results stored in container
The Class has several purposes. First of all, it prepares the images of the event
that will be further use for reconstruction by applying calibration, cleaning and
selection. Then, it reconstructs the geometry of the event and then returns
image (e.g. Hillas parameters)and event information (e.g. reults od the
reconstruction).
Parameters
----------
config: dict
Configuration with analysis parameters
mode: str
Mode of the reconstruction, e.g. tail or wave
event_cutflow: ctapipe.utils.CutFlow
Statistic of events processed
image_cutflow: ctapipe.utils.CutFlow
Statistic of images processed
Returns: dict
Dictionnary of results
"""
def __init__(self, config, mode, event_cutflow=None, image_cutflow=None):
# Cleaning for reconstruction
self.cleaner_reco = ImageCleaner( # for reconstruction
config=config["ImageCleaning"]["biggest"],
mode=mode)
# Cleaning for energy/score estimation
# Add possibility to force energy/score cleaning with tailcut analysis
force_mode = mode
try:
if config["General"]["force_tailcut_for_extended_cleaning"] is True:
force_mode = config["General"]["force_mode"]
print("> Activate force-mode for cleaning!!!!")
except:
pass # force_mode = mode
self.cleaner_extended = ImageCleaner( # for energy/score estimation
config=config["ImageCleaning"]["extended"],
mode=force_mode)
# Image book keeping
self.image_cutflow = image_cutflow or CutFlow("ImageCutFlow")
# Add quality cuts on images
charge_bounds = config["ImageSelection"]["charge"]
npix_bounds = config["ImageSelection"]["pixel"]
ellipticity_bounds = config["ImageSelection"]["ellipticity"]
nominal_distance_bounds = config["ImageSelection"]["nominal_distance"]
self.camera_radius = {
"LSTCam": 1.126,
"NectarCam": 1.126,
} # Average between max(xpix) and max(ypix), in meter
self.image_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min pixel", lambda s: np.count_nonzero(s) < npix_bounds[0]),
("min charge", lambda x: x < charge_bounds[0]),
# ("poor moments", lambda m: m.width <= 0 or m.length <= 0 or np.isnan(m.width) or np.isnan(m.length)), # TBC, maybe we loose events without nan conditions
("poor moments", lambda m: m.width <= 0 or m.length <= 0),
(
"bad ellipticity",
lambda m: (m.width / m.length) < ellipticity_bounds[0] or
(m.width / m.length) > ellipticity_bounds[-1],
),
# ("close to the edge", lambda m, cam_id: m.r.value > (nominal_distance_bounds[-1] * 1.12949101073069946)) # in meter
(
"close to the edge",
lambda m, cam_id: m.r.value >
(nominal_distance_bounds[-1] * self.camera_radius[cam_id]),
), # in meter
]))
# Reconstruction
self.shower_reco = HillasReconstructor()
# Event book keeping
self.event_cutflow = event_cutflow or CutFlow("EventCutFlow")
# Add cuts on events
min_ntel = config["Reconstruction"]["min_tel"]
self.event_cutflow.set_cuts(
OrderedDict([
("noCuts", None),
("min2Tels trig", lambda x: x < min_ntel),
("min2Tels reco", lambda x: x < min_ntel),
("direction nan", lambda x: x.is_valid == False),
]))
def prepare_event(self, source, return_stub=False):
# configuration for the camera calibrator
# modifies the integration window to be more like in MARS
# JLK, only for LST!!!!
# Option for integration correction is done above
cfg = Config()
cfg["ChargeExtractorFactory"]["window_width"] = 5
cfg["ChargeExtractorFactory"]["window_shift"] = 2
self.calib = CameraCalibrator(
config=cfg,
extractor_product="LocalPeakIntegrator",
eventsource=source,
tool=None,
)
for event in source:
self.event_cutflow.count("noCuts")
if self.event_cutflow.cut("min2Tels trig",
len(event.dl0.tels_with_data)):
if return_stub:
yield stub(event)
else:
continue
# calibrate the event
self.calib.calibrate(event)
# telescope loop
tot_signal = 0
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for geometry
hillas_dict = {} # for discrimination
n_tels = {
"tot": len(event.dl0.tels_with_data),
"LST": 0,
"MST": 0,
"SST": 0,
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
# To compute impact parameter in tilt system
run_array_direction = event.mcheader.run_array_direction
az, alt = run_array_direction[0], run_array_direction[1]
ground_frame = GroundFrame()
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count("noCuts")
camera = event.inst.subarray.tel[tel_id].camera
# count the current telescope according to its size
tel_type = event.inst.subarray.tel[tel_id].optics.tel_type
# JLK, N telescopes before cut selection are not really interesting for
# discrimination, too much fluctuations
# n_tels[tel_type] += 1
# the camera image as a 1D array and stuff needed for calibration
# Choose gain according to pywicta's procedure
image_1d = simtel_event_to_images(event=event,
tel_id=tel_id,
ctapipe_format=True)
pmt_signal = image_1d.input_image # calibrated image
# clean the image
try:
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.cam_id)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.cam_id)
# Image for score/energy estimation (with clusters)
image_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
except FileNotFoundError as e: # JLK, WHAT?
print(e)
continue
# Apply some selection
if self.image_cutflow.cut("min pixel", image_biggest):
continue
if self.image_cutflow.cut("min charge", np.sum(image_biggest)):
continue
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.cam_id)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
# do the hillas reconstruction of the images
# QUESTION should this change in numpy behaviour be done here
# or within `hillas_parameters` itself?
# JLK: make selection on biggest cluster
with np.errstate(invalid="raise", divide="raise"):
try:
moments_reco = hillas_parameters(
camera,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera, image_extended
) # for discrimination and energy reconstruction
# if width and/or length are zero (e.g. when there is only only one
# pixel or when all pixel are exactly in one row), the
# parametrisation won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_reco):
# print('poor moments')
continue
if self.image_cutflow.cut("close to the edge",
moments_reco, camera.cam_id):
# print('close to the edge')
continue
if self.image_cutflow.cut("bad ellipticity",
moments_reco):
# print('bad ellipticity: w={}, l={}'.format(moments_reco.width, moments_reco.length))
continue
except (FloatingPointError,
hillas.HillasParameterizationError):
continue
point_azimuth_dict[
tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
point_altitude_dict[
tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
n_tels[tel_type] += 1
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0])
tot_signal += moments.intensity
n_tels["reco"] = len(hillas_dict_reco)
n_tels["discri"] = len(hillas_dict)
if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]):
if return_stub:
yield stub(event)
else:
continue
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Reconstruction results
reco_result = self.shower_reco.predict(
hillas_dict_reco,
event.inst,
point_altitude_dict,
point_azimuth_dict,
)
# shower_sys = TiltedGroundFrame(pointing_direction=HorizonFrame(
# az=reco_result.az,
# alt=reco_result.alt
# ))
# Impact parameter for energy estimation (/ tel)
subarray = event.inst.subarray
for tel_id in hillas_dict.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
# tel_tilt = tel_ground.transform_to(shower_sys)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# core_tilt = core_ground.transform_to(shower_sys)
# Should be better handled (tilted frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_ground.x - tel_ground.x)**2 +
(core_ground.y - tel_ground.y)**2)
except Exception as e:
print("exception in reconstruction:", e)
raise
if return_stub:
yield stub(event)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if return_stub:
yield stub(event)
else:
continue
yield PreparedEvent(
event=event,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
n_tels=n_tels,
tot_signal=tot_signal,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
)
class PrepareList(Cutter):
'''
Prepare a feature list to save to table. It takes an event, does the
calibration, image cleaning, parametrization and reconstruction.
From this some basic features will be extracted and written to the file
which later on can be used for training of the classifiers or energy
regressors.
test
'''
true_az = {}
true_alt = {}
max_signal = {}
tot_signal = 0
impact = {}
def __init__(self,
event,
telescope_list,
camera_types,
ChargeExtration,
pe_thresh,
min_neighbors,
tail_thresholds,
DirReco,
quality_cuts,
LUT=None):
super().__init__()
'''
Parmeters
---------
event : ctapipe event container
calibrator : ctapipe camera calibrator
reconstructor : ctapipe hillas reconstructor
telescope_list : list with telescope configuration or "all"
pe_thresh : dict with thresholds for gain selection
tail_thresholds : dict with thresholds for image cleaning
quality_cuts : dict containing quality cuts
canera_types : list with camera types to analyze
'''
self.event = event
self.telescope_list = telescope_list
self.pe_thresh = pe_thresh
self.min_neighbors = min_neighbors
self.tail_thresholds = tail_thresholds
self.quality_cuts = quality_cuts
self.camera_types = camera_types
self.dirreco = DirReco
self.LUTgenerator = LUT
if (self.dirreco["weights"] == "LUT") | (self.dirreco["weights"]
== "doublepass"):
self.weights = {}
else:
self.weights = None
self.hillas_dict = {}
self.camera_dict = {}
self.edge_pixels = {}
# configurations for calibrator
cfg = Config()
cfg["ChargeExtractorFactory"]["product"] = \
ChargeExtration["ChargeExtractorProduct"]
cfg['WaveformCleanerFactory']['product'] = \
ChargeExtration["WaveformCleanerProduct"]
self.calibrator = CameraCalibrator(r1_product="HESSIOR1Calibrator",
config=cfg) # calibration
self.reconstructor = HillasReconstructor() # direction
def get_impact(self, hillas_dict):
'''
calculate impact parameters for all telescopes that were
used for parametrization.
Paremeters
----------
hillas_dict : dict with hillas HillasParameterContainers
Returnes
--------
impact : impact parameter or NaN if calculation failed
'''
# check if event was prepared before
try:
assert self.reco_result
except AssertionError:
self.prepare()
impact = {}
for tel_id in hillas_dict.keys():
try:
pred_core = np.array([
self.reco_result.core_x.value,
self.reco_result.core_y.value
]) * u.m
# tel_coords start at 0 instead of 1...
tel_position = np.array([
self.event.inst.subarray.tel_coords[tel_id - 1].x.value,
self.event.inst.subarray.tel_coords[tel_id - 1].y.value
]) * u.m
impact[tel_id] = linalg.length(pred_core - tel_position)
except AttributeError:
impact[tel_id] = np.nan
return impact
def get_offangle(self, tel_id, direction="reco"):
'''
Get the angular offset between the reconstructed direction and the
pointing direction of the telescope.
Parameters
----------
tel_id : integer
Telecope ID
true_off : string
if "mc", the true MC direction is taken for calculation. Otherwise,
if "reco" the reconstructed value will be taken
Returns
-------
off_angles : dictionary
dictionary with tel_ids as keys and the offangle as entries.
'''
if direction == "reco":
off_angle = angular_separation(
self.event.mc.tel[tel_id].azimuth_raw * u.rad,
self.event.mc.tel[tel_id].altitude_raw * u.rad,
self.reco_result.az, self.reco_result.alt)
elif direction == "mc":
off_angle = angular_separation(
self.event.mc.tel[tel_id].azimuth_raw * u.rad,
self.event.mc.tel[tel_id].altitude_raw * u.rad,
self.event.mc.az, self.event.mc.alt)
return off_angle
def get_weight(self, method, camera, tel_id, hillas_par, offangle=None):
"""
Get the weighting for HillasReconustructor. Possible methods are
'default', which will fall back to the standard weighting applied
in capipe, 'LUT' which will take the weights from a LUT and
'doublepass' which might be used for diffuse simulations. In this
case it returns the weights for the first pass.
method : sting
method to get the weighting.
camera: CameraDescription
tel_id: integer
hillas_par: HillasParameterContainer
"""
if method == "default":
pass
elif method == "LUT":
if np.isnan(hillas_par.width) & (not np.isnan(hillas_par.length)):
hillas_par.width = 0 * u.m
self.weights[tel_id] = self.LUTgenerator.get_weight_from_LUT(
hillas_par,
camera.cam_id,
min_stat=self.dirreco["min_stat"],
ratio_cut=self.dirreco["wl_ratio_cut"][camera.cam_id])
elif method == "doublepass":
# first pass with default weighting
if np.isnan(hillas_par.width) & (not np.isnan(hillas_par.length)):
hillas_par.width = 0 * u.m
self.weights[tel_id] = hillas_par.intensity * (
1 * u.m + hillas_par.length) / (1 * u.m + hillas_par.width)
elif method == "second_pass":
# weights for second pass
self.weights[
tel_id] = self.LUTgenerator.get_weight_from_diffuse_LUT(
self.hillas_dict[tel_id],
offangle,
camera.cam_id,
min_stat=self.dirreco["min_stat"],
ratio_cut=self.dirreco["wl_ratio_cut"][camera.cam_id])
else:
raise KeyError("Weighting method {} not known.".format(method))
def prepare(self):
'''
Prepare event performimng calibration, image cleaning,
hillas parametrization, hillas intersection for the single
event. Additionally, the impact distance will be calculated.
'''
tels_per_type = {}
no_weight = []
# calibrate event
self.calibrator.calibrate(self.event)
# loop over all telescopeswith data in it
for tel_id in self.event.r0.tels_with_data:
# check if telescope is selected for analysis
# This also could be done already in event_source when reading th data
if (tel_id in self.telescope_list) | (self.telescope_list
== "all"):
pass
else:
continue
# get camera information
camera = self.event.inst.subarray.tel[tel_id].camera
self.camera_dict[tel_id] = camera
image = self.event.dl1.tel[tel_id].image
if camera.cam_id in self.pe_thresh.keys():
image, select = pick_gain_channel(
image, self.pe_thresh[camera.cam_id], True)
else:
image = np.squeeze(image)
# image cleaning
mask = tailcuts_clean(
camera,
image,
picture_thresh=self.tail_thresholds[camera.cam_id][1],
boundary_thresh=self.tail_thresholds[camera.cam_id][0],
min_number_picture_neighbors=self.min_neighbors)
# go to next telescope if no pixels survived cleaning
if not any(mask):
continue
cleaned_image = np.copy(image)
cleaned_image[~mask] = 0
# calculate the hillas parameters
hillas_par = hillas_parameters(camera, cleaned_image)
# quality cuts
leakage = leakage = self.leakage_cut(
camera=camera,
hillas_parameters=hillas_par,
radius=self.quality_cuts["leakage_cut"]["radius"],
max_dist=self.quality_cuts["leakage_cut"]["dist"],
image=cleaned_image,
rows=self.quality_cuts["leakage_cut"]["rows"],
fraction=self.quality_cuts["leakage_cut"]["frac"],
method=self.quality_cuts["leakage_cut"]["method"],
)
size = self.size_cut(hillas_par, self.quality_cuts["size"])
if not (leakage & size):
# size or leakage cuts not passed
continue
# get the weighting for HillasReconstructor
try:
self.get_weight(self.dirreco["weights"], camera, tel_id,
hillas_par)
except LookupFailedError:
# this telescope will be ignored, should only happen for method LUT here
no_weight.append(tel_id)
continue
self.hillas_dict[tel_id] = hillas_par
self.max_signal[tel_id] = np.max(cleaned_image) # brightest pix
try:
tels_per_type[camera.cam_id].append(tel_id)
except KeyError:
tels_per_type[camera.cam_id] = [tel_id]
try:
assert tels_per_type
except AssertionError:
raise TooFewTelescopesException("No image survived the leakage "
"or size cuts.")
# collect some additional information
for tel_id in self.hillas_dict:
self.tot_signal += self.hillas_dict[tel_id].intensity # total size
self.true_az[
tel_id] = self.event.mc.tel[tel_id].azimuth_raw * u.rad
self.true_alt[
tel_id] = self.event.mc.tel[tel_id].altitude_raw * u.rad
# wil raise exception if cut was not passed
# self.multiplicity_cut(self.quality_cuts["multiplicity"]["cuts"],
# tels_per_type, method=self.quality_cuts["multiplicity"]["method"])
if self.dirreco["weights"] == "LUT":
# remove telescopes withough weights
print("Removed {} of {} telescopes due LUT problems".format(
len(no_weight),
len(self.hillas_dict) + len(no_weight)))
# do Hillas reconstruction
self.reco_result = self.reconstructor.predict(self.hillas_dict,
self.event.inst,
self.true_alt,
self.true_az,
ext_weight=self.weights)
if self.dirreco["weights"] == "doublepass":
# take the reconstructed direction to get an estimate of the offangle and
# get weights from the second pass from the diffuse LUT.
self.weights = {} # reset the weights from earlier
no_weight = []
for tel_id in self.hillas_dict:
predicted_offangle = self.get_offangle(tel_id,
direction="reco")
predicted_offangle = predicted_offangle.to(u.deg).value
camera = self.camera_dict[tel_id] # reload camera_information
# get the weighting for HillasReconstructor
try:
self.get_weight("second_pass", camera, tel_id,
self.hillas_dict[tel_id],
predicted_offangle)
except LookupFailedError:
no_weight.append(tel_id)
print("Removed {} of {} telescopes due LUT problems".format(
len(no_weight), len(self.hillas_dict)))
# remove those types from tels_per_type
for tel_id in no_weight:
del self.hillas_dict[tel_id]
for cam_id in tels_per_type:
if tel_id in tels_per_type[cam_id]:
index = np.where(
np.array(tels_per_type[cam_id]) == tel_id)
tels_per_type[cam_id].pop(int(
index[0])) # remove from list
# redo the multiplicity cut to check if it still fulfilled
# self.multiplicity_cut(self.quality_cuts["multiplicity"]["cuts"], tels_per_type,
# method=self.quality_cuts["multiplicity"]["method"])
# do the second pass with new weights
self.reco_result = self.reconstructor.predict(
self.hillas_dict,
self.event.inst,
self.true_alt,
self.true_az,
ext_weight=self.weights)
# Number of telescopes triggered per type
self.n_tels_per_type = {
tel: len(tels_per_type[tel])
for tel in tels_per_type
}
for tel_id in self.hillas_dict:
try:
agree = self.mc_offset == self.get_offangle(tel_id,
direction="mc")
except AttributeError:
self.mc_offset = self.get_offangle(tel_id, direction="mc")
continue
if not agree:
raise ValueError(
"The pointing of the telescopes seems to be different.")
self.impact = self.get_impact(self.hillas_dict) # impact parameter
def get_reconstructed_parameters(self):
'''
Return the parameters for writing to table.
Returns
-------
prepared parameters :
impact
max_signal
tot_signal
n_tels_per_type
hillas_dict
mc_offangle
reco_result
'''
# check if event was prepared before
try:
assert self.impact
except AssertionError:
self.prepare()
return (self.impact, self.max_signal, self.tot_signal,
self.n_tels_per_type, self.hillas_dict, self.mc_offset,
self.reco_result)