def test_remove_isolated_pixels_input_copy(self):
"""Check whether the input image is altered during process."""
# Input image #################
input_img = np.array([[1, 0, 1], [1, 0, 0], [1, 0, 1]])
input_img_copy = np.copy(input_img)
# Output image ################
output_img = filter_pixels_clusters(input_img)
# Check whether the input image has changed
np.testing.assert_array_equal(input_img_copy, input_img)
def test_remove_isolated_pixels_example1(self):
"""Check the output of the filter_pixels_clusters function."""
# Input image #################
input_img = np.array([[0, 0, 1, 1, 0, 0], [0, 0, 0, 1, 0, 0],
[1, 1, 0, 0, 1, 0], [0, 0, 0, 1, 0, 0]])
# Output image ################
output_img = filter_pixels_clusters(input_img)
# Expected output image #######
expected_output_img = np.array([[0, 0, 1, 1, 0, 0], [0, 0, 0, 1, 0, 0],
[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0,
0]])
np.testing.assert_array_equal(output_img, expected_output_img)
def test_whether_selection_is_based_on_pixels_value(self):
"""Check whether selection is based on pixels value or number of pixels in clusters."""
# Input image #################
input_img = np.array([[0, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0],
[1, 1, 1, 0, 10, 0], [1, 1, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0]])
# Output image ################
output_img = filter_pixels_clusters(input_img)
# Expected output image #######
expected_output_img = np.array([[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 10,
0], [0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0]])
np.testing.assert_array_equal(output_img, expected_output_img)
def test_remove_isolated_pixels_example_negative_threshold(self):
"""Check the output of the filter_pixels_clusters function."""
# Input image #################
input_img = np.array([[0, 0, 0, 0, 0, 0, 0], [0, 0, -1, -1, -1, 0, 0],
[0, 0, -1, -1, -1, 0, 0],
[0, 0, -1, -1, -1, 0, 0], [0, 0, 0, 0, 0, 0, 0]])
# Output image ################
output_img = filter_pixels_clusters(input_img, threshold=-2)
# Expected output image #######
expected_output_img = np.array(
[[0, 0, 0, 0, 0, 0, 0], [0, 0, -1, -1, -1, 0, 0],
[0, 0, -1, -1, -1, 0, 0], [0, 0, -1, -1, -1,
0, 0],
[0, 0, 0, 0, 0, 0, 0]]) # < Here there is only one big island!
np.testing.assert_array_equal(output_img, expected_output_img)
def test_remove_isolated_pixels_example_threshold(self):
"""Check that every values below threshold is set to 0."""
# Input image #################
input_img = np.array([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[1.0, 0.1, 1.0, 1.0, 1.0, 0.1, 1.0],
[0.0, 0.5, 1.0, -1.0, 1.0, 0.5, 0.0],
[1.0, 0.1, 1.0, 1.0, 1.0, 0.1, 1.0],
[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]])
# Output image ################
output_img = filter_pixels_clusters(input_img, threshold=0.2)
# Expected output image #######
expected_output_img = np.array([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0],
[0.0, 0.5, 1.0, 0.0, 1.0, 0.5, 0.0],
[0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]])
np.testing.assert_array_equal(output_img, expected_output_img)
def prepare_event(self,
source,
return_stub=True,
save_images=False,
debug=False):
"""
Calibrate, clean and reconstruct the direction of an event.
Parameters
----------
source : ctapipe.io.EventSource
A container of selected showers from a simtel file.
geom_cam_tel: dict
Dictionary of of MyCameraGeometry objects for each camera in the file
return_stub : bool
If True, yield also images from events that won't be reconstructed.
This feature is not currently available.
save_images : bool
If True, save photoelectron images from reconstructed events.
debug : bool
If True, print some debugging information (to be expanded).
Yields
------
PreparedEvent: dict
Dictionary containing event-image information to be written.
"""
# =============================================================
# TRANSFORMED CAMERA GEOMETRIES
# =============================================================
# These are the camera geometries were the Hillas parametrization will
# be performed.
# They are transformed to TelescopeFrame using the effective focal
# lengths
# These geometries could be used also to performe the image cleaning,
# but for the moment we still do that in the CameraFrame
geom_cam_tel = {}
for camera in source.subarray.camera_types:
# Original geometry of each camera
geom = camera.geometry
# Same but with focal length as an attribute
# This is planned to disappear and be imported by ctapipe
focal_length = effective_focal_lengths(camera.camera_name)
geom_cam_tel[camera.camera_name] = MyCameraGeometry(
camera_name=camera.camera_name,
pix_type=geom.pix_type,
pix_id=geom.pix_id,
pix_x=geom.pix_x,
pix_y=geom.pix_y,
pix_area=geom.pix_area,
cam_rotation=geom.cam_rotation,
pix_rotation=geom.pix_rotation,
frame=CameraFrame(focal_length=focal_length),
).transform_to(TelescopeFrame())
# =============================================================
ievt = 0
for event in source:
# Display event counts
if debug:
print(
bcolors.BOLD +
f"EVENT #{event.count}, EVENT_ID #{event.index.event_id}" +
bcolors.ENDC)
print(bcolors.BOLD +
f"has triggered telescopes {event.r1.tel.keys()}" +
bcolors.ENDC)
ievt += 1
# if (ievt < 10) or (ievt % 10 == 0):
# print(ievt)
self.event_cutflow.count("noCuts")
# LST stereo condition
# whenever there is only 1 LST in an event, we remove that telescope
# if the remaining telescopes are less than min_tel we remove the event
lst_tel_ids = set(
source.subarray.get_tel_ids_for_type("LST_LST_LSTCam"))
triggered_LSTs = set(event.r0.tel.keys()).intersection(lst_tel_ids)
n_triggered_LSTs = len(triggered_LSTs)
n_triggered_non_LSTs = len(event.r0.tel.keys()) - n_triggered_LSTs
bad_LST_stereo = False
if self.LST_stereo and self.event_cutflow.cut(
"no-LST-stereo + <2 other types", n_triggered_LSTs,
n_triggered_non_LSTs):
bad_LST_stereo = True
if return_stub:
print(
bcolors.WARNING +
"WARNING: LST_stereo is set to 'True'\n" +
f"This event has < {self.min_ntel_LST} triggered LSTs\n"
+
"and < 2 triggered telescopes from other telescope types.\n"
+ "The event will be processed up to DL1b." +
bcolors.ENDC)
# we show this, but we proceed to analyze the event up to
# DL1a/b for the associated benchmarks
# this checks for < 2 triggered telescopes of ANY type
if self.event_cutflow.cut("min2Tels trig",
len(event.r1.tel.keys())):
if return_stub:
print(
bcolors.WARNING +
f"WARNING : < {self.min_ntel} triggered telescopes!" +
bcolors.ENDC)
# we show this, but we proceed to analyze it
# =============================================================
# CALIBRATION
# =============================================================
if debug:
print(bcolors.OKBLUE + "Extracting all calibrated images..." +
bcolors.ENDC)
self.calib(event) # Calibrate the event
# =============================================================
# BEGINNING OF LOOP OVER TELESCOPES
# =============================================================
dl1_phe_image = {}
dl1_phe_image_mask_reco = {}
dl1_phe_image_mask_clusters = {}
mc_phe_image = {}
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for direction reconstruction
hillas_dict = {} # for discrimination
leakage_dict = {}
concentration_dict = {}
n_tels = {
"Triggered": len(event.r1.tel.keys()),
"LST_LST_LSTCam": 0,
"MST_MST_NectarCam": 0,
"MST_MST_FlashCam": 0,
"MST_SCT_SCTCam": 0,
"SST_1M_DigiCam": 0,
"SST_ASTRI_ASTRICam": 0,
"SST_GCT_CHEC": 0,
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
good_for_reco = {} # 1 = success, 0 = fail
# Array pointing in AltAz frame
az = event.pointing.array_azimuth
alt = event.pointing.array_altitude
array_pointing = SkyCoord(az, alt, frame=AltAz())
ground_frame = GroundFrame()
tilted_frame = TiltedGroundFrame(pointing_direction=array_pointing)
for tel_id in event.r1.tel.keys():
point_azimuth_dict[tel_id] = event.pointing.tel[tel_id].azimuth
point_altitude_dict[tel_id] = event.pointing.tel[
tel_id].altitude
if debug:
print(bcolors.OKBLUE +
f"Working on telescope #{tel_id}..." + bcolors.ENDC)
self.image_cutflow.count("noCuts")
camera = source.subarray.tel[tel_id].camera.geometry
# count the current telescope according to its type
tel_type = str(source.subarray.tel[tel_id])
n_tels[tel_type] += 1
# use ctapipe's functionality to get the calibrated image
# and scale the reconstructed values if required
pmt_signal = event.dl1.tel[tel_id].image / self.calibscale
# If required...
if save_images is True:
# Save the simulated and reconstructed image of the event
dl1_phe_image[tel_id] = pmt_signal
mc_phe_image[tel_id] = event.simulation.tel[
tel_id].true_image
# We now ASSUME that the event will be good
good_for_reco[tel_id] = 1
# later we change to 0 if any condition is NOT satisfied
if self.cleaner_reco.mode == "tail": # tail uses only ctapipe
# Cleaning used for direction reconstruction
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
# calculate the leakage (before filtering)
leakages = {} # this is needed by both cleanings
# The check on SIZE shouldn't be here, but for the moment
# I prefer to sacrifice elegancy...
if np.sum(image_biggest[mask_reco]) != 0.0:
leakage_biggest = leakage_parameters(
camera, image_biggest, mask_reco)
leakages["leak1_reco"] = leakage_biggest[
"intensity_width_1"]
leakages["leak2_reco"] = leakage_biggest[
"intensity_width_2"]
else:
leakages["leak1_reco"] = 0.0
leakages["leak2_reco"] = 0.0
# find all islands using this cleaning
num_islands, labels = number_of_islands(camera, mask_reco)
if num_islands == 1: # if only ONE islands is left ...
# ...use directly the old mask and reduce dimensions
# to make Hillas parametrization faster
camera_biggest = camera[mask_reco]
image_biggest = image_biggest[mask_reco]
if save_images is True:
dl1_phe_image_mask_reco[tel_id] = mask_reco
elif num_islands > 1: # if more islands survived..
# ...find the biggest one
mask_biggest = largest_island(labels)
# and also reduce dimensions
camera_biggest = camera[mask_biggest]
image_biggest = image_biggest[mask_biggest]
if save_images is True:
dl1_phe_image_mask_reco[tel_id] = mask_biggest
else: # if no islands survived use old camera and image
camera_biggest = camera
dl1_phe_image_mask_reco[tel_id] = mask_reco
# Cleaning used for score/energy estimation
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
dl1_phe_image_mask_clusters[tel_id] = mask_extended
# calculate the leakage (before filtering)
# this part is not well coded, but for the moment it works
if np.sum(image_extended[mask_extended]) != 0.0:
leakage_extended = leakage_parameters(
camera, image_extended, mask_extended)
leakages["leak1"] = leakage_extended[
"intensity_width_1"]
leakages["leak2"] = leakage_extended[
"intensity_width_2"]
else:
leakages["leak1"] = 0.0
leakages["leak2"] = 0.0
# find all islands with this cleaning
# we will also register how many have been found
n_cluster_dict[tel_id], labels = number_of_islands(
camera, mask_extended)
# NOTE: the next check shouldn't be necessary if we keep
# all the isolated pixel clusters, but for now the
# two cleanings are set the same in analysis.yml because
# the performance of the extended one has never been really
# studied in model estimation.
# if some islands survived
if n_cluster_dict[tel_id] > 0:
# keep all of them and reduce dimensions
camera_extended = camera[mask_extended]
image_extended = image_extended[mask_extended]
else: # otherwise continue with the old camera and image
camera_extended = camera
# could this go into `hillas_parameters` ...?
# this is basically the charge of ALL islands
# not calculated later by the Hillas parametrization!
max_signals[tel_id] = np.max(image_extended)
else: # for wavelets we stick to old pywi-cta code
try: # "try except FileNotFoundError" not clear to me, but for now it stays...
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.camera_name)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.camera_name)
# Image for score/energy estimation (with clusters)
(
image_extended,
mask_extended,
) = self.cleaner_extended.clean_image(
pmt_signal, camera)
# This last part was outside the pywi-cta block
# before, but is indeed part of it because it uses
# pywi-cta functions in the "extended" case
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.camera_name)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
except FileNotFoundError as e:
print(e)
continue
# =============================================================
# PRELIMINARY IMAGE SELECTION
# =============================================================
cleaned_image_is_good = True # we assume this
if self.image_selection_source == "extended":
cleaned_image_to_use = image_extended
elif self.image_selection_source == "biggest":
cleaned_image_to_use = image_biggest
else:
raise ValueError(
"Only supported cleanings are 'biggest' or 'extended'."
)
# Apply some selection
if self.image_cutflow.cut("min pixel", cleaned_image_to_use):
if debug:
print(bcolors.WARNING +
"WARNING : not enough pixels!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
cleaned_image_is_good = False
if self.image_cutflow.cut("min charge",
np.sum(cleaned_image_to_use)):
if debug:
print(bcolors.WARNING +
"WARNING : not enough charge!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
cleaned_image_is_good = False
if debug and (not cleaned_image_is_good): # BAD image quality
print(bcolors.WARNING +
"WARNING : The cleaned image didn't pass" +
" preliminary cuts.\n" +
"An attempt to parametrize it will be made," +
" but the image will NOT be used for" +
" direction reconstruction." + bcolors.ENDC)
# =============================================================
# IMAGE PARAMETRIZATION
# =============================================================
with np.errstate(invalid="raise", divide="raise"):
try:
# Filter the cameras in TelescopeFrame with the same
# cleaning masks
camera_biggest_tel = geom_cam_tel[camera.camera_name][
camera_biggest.pix_id]
camera_extended_tel = geom_cam_tel[camera.camera_name][
camera_extended.pix_id]
# Parametrize the image in the TelescopeFrame
moments_reco = hillas_parameters(
camera_biggest_tel,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera_extended_tel, image_extended
) # for discrimination and energy reconstruction
if debug:
print(
"Image parameters from main cluster cleaning:")
print(moments_reco)
print(
"Image parameters from all-clusters cleaning:")
print(moments)
# Add concentration parameters
concentrations = {}
concentrations_extended = concentration_parameters(
camera_extended_tel, image_extended, moments)
concentrations[
"concentration_cog"] = concentrations_extended[
"cog"]
concentrations[
"concentration_core"] = concentrations_extended[
"core"]
concentrations[
"concentration_pixel"] = concentrations_extended[
"pixel"]
# ===================================================
# PARAMETRIZED IMAGE SELECTION
# ===================================================
if self.image_selection_source == "extended":
moments_to_use = moments
else:
moments_to_use = moments_reco
# if width and/or length are zero (e.g. when there is
# only only one pixel or when all pixel are exactly
# in one row), the parametrisation
# won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_to_use):
if debug:
print(bcolors.WARNING +
"WARNING : poor moments!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
if self.image_cutflow.cut("close to the edge",
moments_to_use,
camera.camera_name):
if debug:
print(
bcolors.WARNING +
"WARNING : out of containment radius!\n" +
f"Camera radius = {self.camera_radius[camera.camera_name]}\n"
+ f"COG radius = {moments_to_use.r}" +
bcolors.ENDC)
good_for_reco[tel_id] = 0
if self.image_cutflow.cut("bad ellipticity",
moments_to_use):
if debug:
print(bcolors.WARNING +
"WARNING : bad ellipticity" +
bcolors.ENDC)
good_for_reco[tel_id] = 0
if debug and good_for_reco[tel_id] == 1:
print(
bcolors.OKGREEN +
"Image survived and correctly parametrized."
# + "\nIt will be used for direction reconstruction!"
+ bcolors.ENDC)
elif debug and good_for_reco[tel_id] == 0:
print(
bcolors.WARNING + "Image not survived or " +
"not good enough for parametrization."
# + "\nIt will be NOT used for direction reconstruction, "
# + "BUT it's information will be recorded."
+ bcolors.ENDC)
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(
np.where(image_extended > 0)[0])
leakage_dict[tel_id] = leakages
concentration_dict[tel_id] = concentrations
except (
FloatingPointError,
HillasParameterizationError,
ValueError,
) as e:
if debug:
print(bcolors.FAIL + "Parametrization error: " +
f"{e}\n" + "Dummy parameters recorded." +
bcolors.ENDC)
good_for_reco[tel_id] = 0
hillas_dict[
tel_id] = HillasParametersTelescopeFrameContainer(
)
hillas_dict_reco[
tel_id] = HillasParametersTelescopeFrameContainer(
)
n_pixel_dict[tel_id] = len(
np.where(image_extended > 0)[0])
leakage_dict[tel_id] = leakages
concentration_dict[tel_id] = concentrations
# END OF THE CYCLE OVER THE TELESCOPES
# =============================================================
# DIRECTION RECONSTRUCTION
# =============================================================
if bad_LST_stereo:
if debug:
print(
bcolors.WARNING +
"WARNING: This event was triggered with 1 LST image and <2 images from other telescope types."
+
"\nWARNING : direction reconstruction will not be performed."
+ bcolors.ENDC)
# Set all the involved images as NOT good for recosntruction
# even though they might have been
# but this is because of the LST stereo trigger....
for tel_id in event.r0.tel.keys():
good_for_reco[tel_id] = 0
# and set the number of good and bad images accordingly
n_tels["GOOD images"] = 0
n_tels[
"BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"]
# create a dummy container for direction reconstruction
reco_result = ReconstructedShowerContainer()
if return_stub: # if saving all events (default)
if debug:
print(bcolors.OKBLUE + "Recording event..." +
bcolors.ENDC)
print(
bcolors.WARNING +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
continue
else:
continue
# Now in case the only triggered telescopes were
# - < self.min_ntel_LST LST,
# - >=2 any other telescope type,
# we remove the single-LST image and continue reconstruction with
# the images from the other telescope types
if self.LST_stereo and (n_triggered_LSTs < self.min_ntel_LST) and (
n_triggered_LSTs != 0) and (n_triggered_non_LSTs >= 2):
if debug:
print(
bcolors.WARNING +
f"WARNING: LST stereo trigger condition is active.\n" +
f"This event triggered < {self.min_ntel_LST} LSTs " +
f"and {n_triggered_non_LSTs} images from other telescope types.\n"
+ bcolors.ENDC)
for tel_id in triggered_LSTs: # in case we test for min_ntel_LST>2
if good_for_reco[tel_id]:
# we don't use it for reconstruction
good_for_reco[tel_id] = 0
print(
bcolors.WARNING +
f"WARNING: LST image #{tel_id} removed, even though it passed quality cuts."
+ bcolors.ENDC)
# TODO: book-keeping of this kind of events doesn't seem easy
# convert dictionary in numpy array to get a "mask"
images_status = np.asarray(list(good_for_reco.values()))
# record how many images will be used for reconstruction
n_tels["GOOD images"] = len(
np.extract(images_status == 1, images_status))
n_tels["BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"]
if self.event_cutflow.cut("min2Tels reco", n_tels["GOOD images"]):
if debug:
print(
bcolors.FAIL +
f"WARNING: < {self.min_ntel} ({n_tels['GOOD images']}) images remaining!"
+
"\nWARNING : direction reconstruction is not possible!"
+ bcolors.ENDC)
# create a dummy container for direction reconstruction
reco_result = ReconstructedShowerContainer()
if return_stub: # if saving all events (default)
if debug:
print(bcolors.OKBLUE + "Recording event..." +
bcolors.ENDC)
print(
bcolors.WARNING +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
continue
else:
continue
if debug:
print(bcolors.OKBLUE + "Starting direction reconstruction..." +
bcolors.ENDC)
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
if self.image_selection_source == "extended":
hillas_dict_to_use = hillas_dict
else:
hillas_dict_to_use = hillas_dict_reco
# use only the successfully parametrized images
# to reconstruct the direction of this event
successfull_hillas = np.where(images_status == 1)[0]
all_images = np.asarray(list(good_for_reco.keys()))
good_images = set(all_images[successfull_hillas])
good_hillas_dict = {
k: v
for k, v in hillas_dict_to_use.items()
if k in good_images
}
if debug:
print(
bcolors.PURPLE +
f"{len(good_hillas_dict)} images " +
f"(from telescopes #{list(good_hillas_dict.keys())}) will be "
+ "used to recover the shower's direction..." +
bcolors.ENDC)
# Reconstruction results
reco_result = self.shower_reco.predict(
good_hillas_dict,
source.subarray,
SkyCoord(alt=alt, az=az, frame="altaz"),
None, # use the array direction
)
# Impact parameter for telescope-wise energy estimation
subarray = source.subarray
for tel_id in hillas_dict_to_use.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# Go back to the tilted frame
# this should be the same...
tel_tilted = tel_ground.transform_to(tilted_frame)
# but this not
core_tilted = SkyCoord(x=core_ground.x,
y=core_ground.y,
frame=tilted_frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_tilted.x - tel_tilted.x)**2 +
(core_tilted.y - tel_tilted.y)**2)
except (Exception, TooFewTelescopesException,
InvalidWidthException) as e:
if debug:
print("exception in reconstruction:", e)
raise
if return_stub:
if debug:
print(
bcolors.FAIL +
"Shower could NOT be correctly reconstructed! " +
"Recording event..." +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if debug:
print(bcolors.WARNING + "WARNING: undefined direction!" +
bcolors.ENDC)
if return_stub:
if debug:
print(
bcolors.FAIL +
"Shower could NOT be correctly reconstructed! " +
"Recording event..." +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
else:
continue
if debug:
print(bcolors.BOLDGREEN + "Shower correctly reconstructed! " +
"Recording event..." + bcolors.ENDC)
yield PreparedEvent(
event=event,
dl1_phe_image=dl1_phe_image,
dl1_phe_image_mask_reco=dl1_phe_image_mask_reco,
dl1_phe_image_mask_clusters=dl1_phe_image_mask_clusters,
mc_phe_image=mc_phe_image,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
leakage_dict=leakage_dict,
concentration_dict=concentration_dict,
n_tels=n_tels,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
good_event=True,
good_for_reco=good_for_reco,
)
def clean_image(
input_image,
type_of_filtering=hard_filter.DEFAULT_TYPE_OF_FILTERING,
filter_thresholds=hard_filter.DEFAULT_FILTER_THRESHOLDS,
last_scale_treatment=mrtransform_wrapper.DEFAULT_LAST_SCALE_TREATMENT,
detect_only_positive_structures=False,
kill_isolated_pixels=False,
noise_distribution=None,
tmp_files_directory=".",
output_data_dict=None,
**kwargs):
"""Clean the `input_image` image.
Apply the wavelet transform, filter planes and return the reverse
transformed image.
Parameters
----------
input_image : array_like
The image to clean.
type_of_filtering : str
Type of filtering: 'hard_filtering' or 'ksigma_hard_filtering'.
filter_thresholds : list of float
Thresholds used for the plane filtering.
last_scale_treatment : str
Last plane treatment: 'keep', 'drop' or 'mask'.
detect_only_positive_structures : bool
Detect only positive structures.
kill_isolated_pixels : bool
Suppress isolated pixels in the support.
noise_distribution : bool
The JSON file containing the Cumulated Distribution Function of the
noise model used to inject artificial noise in blank pixels (those
with a NaN value).
tmp_files_directory : str
The path of the directory where temporary files are written.
output_data_dict : dict
A dictionary used to return results and intermediate results.
Returns
-------
Return the cleaned image.
"""
if DEBUG:
print("Filter thresholds:", filter_thresholds)
number_of_scales = len(filter_thresholds) + 1
if DEBUG:
print("Number of scales:", number_of_scales)
# COMPUTE THE WAVELET TRANSFORM #######################################
wavelet_planes = wavelet_transform(input_image,
number_of_scales=number_of_scales,
tmp_files_directory=tmp_files_directory,
noise_distribution=noise_distribution)
if DEBUG:
for index, plane in enumerate(wavelet_planes):
images.plot(plane, "Plane " + str(index))
# FILTER WAVELET PLANES ###############################################
filtered_wavelet_planes = filter_planes(
wavelet_planes,
method=type_of_filtering,
thresholds=filter_thresholds,
detect_only_positive_structures=detect_only_positive_structures)
#if DEBUG:
# for index, plane in enumerate(filtered_wavelet_planes):
# images.plot(plane, "Filtered plane " + str(index))
# COMPUTE THE INVERSE TRANSFORM #######################################
cleaned_image = inverse_wavelet_transform(filtered_wavelet_planes,
last_plane=last_scale_treatment)
if DEBUG:
images.plot(cleaned_image, "Cleaned image")
# REMOVE ISOLATED PIXELS ##############################################
if output_data_dict is not None:
img_cleaned_clusters_delta_intensity, img_cleaned_clusters_delta_abs_intensity, img_cleaned_clusters_delta_num_pixels = filter_pixels_clusters_stats(
cleaned_image)
img_cleaned_num_islands = number_of_pixels_clusters(cleaned_image)
output_data_dict[
"img_cleaned_clusters_delta_intensity"] = img_cleaned_clusters_delta_intensity
output_data_dict[
"img_cleaned_clusters_delta_abs_intensity"] = img_cleaned_clusters_delta_abs_intensity
output_data_dict[
"img_cleaned_clusters_delta_num_pixels"] = img_cleaned_clusters_delta_num_pixels
output_data_dict["img_cleaned_num_islands"] = img_cleaned_num_islands
if kill_isolated_pixels:
cleaned_image = filter_pixels_clusters(cleaned_image)
if DEBUG:
images.plot(cleaned_image, "Cleaned image after cluster filtering")
return cleaned_image
def prepare_event(self, source, return_stub=False, save_images=False):
for event in source:
self.event_cutflow.count("noCuts")
if self.event_cutflow.cut("min2Tels trig",
len(event.dl0.tels_with_data)):
if return_stub:
yield stub(event)
else:
continue
self.calib(event)
# telescope loop
tot_signal = 0
dl1_phe_image = None
mc_phe_image = None
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for direction reconstruction
hillas_dict = {} # for discrimination
n_tels = {
"tot": len(event.dl0.tels_with_data),
"LST_LST_LSTCam": 0,
"MST_MST_NectarCam": 0,
"SST": 0, # add later correct names when testing on Paranal
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
# Compute impact parameter in tilt system
run_array_direction = event.mcheader.run_array_direction
az, alt = run_array_direction[0], run_array_direction[1]
ground_frame = GroundFrame()
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count("noCuts")
camera = event.inst.subarray.tel[tel_id].camera
# count the current telescope according to its size
tel_type = str(event.inst.subarray.tel[tel_id])
# use ctapipe's functionality to get the calibrated image
pmt_signal = event.dl1.tel[tel_id].image
# Save the calibrated image after the gain has been chosen
# automatically by ctapipe, together with the simulated one
if save_images is True:
dl1_phe_image = pmt_signal
mc_phe_image = event.mc.tel[tel_id].photo_electron_image
if self.cleaner_reco.mode == "tail": # tail uses only ctapipe
# Cleaning used for direction reconstruction
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
# find all islands using this cleaning
num_islands, labels = number_of_islands(camera, mask_reco)
if num_islands == 1: # if only ONE islands is left ...
# ...use directly the old mask and reduce dimensions
# to make Hillas parametrization faster
camera_biggest = camera[mask_reco]
image_biggest = image_biggest[mask_reco]
elif num_islands > 1: # if more islands survived..
# ...find the biggest one
mask_biggest = largest_island(labels)
# and also reduce dimensions
camera_biggest = camera[mask_biggest]
image_biggest = image_biggest[mask_biggest]
else: # if no islands survived use old camera and image
camera_biggest = camera
# Cleaning used for score/energy estimation
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
# find all islands with this cleaning
# we will also register how many have been found
n_cluster_dict[tel_id], labels = number_of_islands(
camera, mask_extended)
# NOTE: the next check shouldn't be necessary if we keep
# all the isolated pixel clusters, but for now the
# two cleanings are set the same in analysis.yml because
# the performance of the extended one has never been really
# studied in model estimation.
# (This is a nice way to ask for volunteers :P)
# if some islands survived
if num_islands > 0:
# keep all of them and reduce dimensions
camera_extended = camera[mask_extended]
image_extended = image_extended[mask_extended]
else: # otherwise continue with the old camera and image
camera_extended = camera
# could this go into `hillas_parameters` ...?
# this is basically the charge of ALL islands
# not calculated later by the Hillas parametrization!
max_signals[tel_id] = np.max(image_extended)
else: # for wavelets we stick to old pywi-cta code
try: # "try except FileNotFoundError" not clear to me, but for now it stays...
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.cam_id)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.cam_id)
# Image for score/energy estimation (with clusters)
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
# This last part was outside the pywi-cta block
# before, but is indeed part of it because it uses
# pywi-cta functions in the "extended" case
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.cam_id)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
except FileNotFoundError as e: # JLK, WHAT?
print(e)
continue
# ==============================================================
# Apply some selection
if self.image_cutflow.cut("min pixel", image_biggest):
continue
if self.image_cutflow.cut("min charge", np.sum(image_biggest)):
continue
# do the hillas reconstruction of the images
# QUESTION should this change in numpy behaviour be done here
# or within `hillas_parameters` itself?
# JLK: make selection on biggest cluster
with np.errstate(invalid="raise", divide="raise"):
try:
moments_reco = hillas_parameters(
camera_biggest,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera_extended, image_extended
) # for discrimination and energy reconstruction
# if width and/or length are zero (e.g. when there is
# only only one pixel or when all pixel are exactly
# in one row), the parametrisation
# won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_reco):
continue
if self.image_cutflow.cut("close to the edge",
moments_reco, camera.cam_id):
continue
if self.image_cutflow.cut("bad ellipticity",
moments_reco):
continue
except (FloatingPointError,
hillas.HillasParameterizationError):
continue
point_azimuth_dict[
tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
point_altitude_dict[
tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
n_tels[tel_type] += 1
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0])
tot_signal += moments.intensity
n_tels["reco"] = len(hillas_dict_reco)
n_tels["discri"] = len(hillas_dict)
if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]):
if return_stub:
yield stub(event)
else:
continue
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Reconstruction results
reco_result = self.shower_reco.predict(
hillas_dict_reco,
event.inst,
SkyCoord(alt=alt, az=az, frame="altaz"),
{
tel_id: SkyCoord(
alt=point_altitude_dict[tel_id],
az=point_azimuth_dict[tel_id],
frame="altaz",
) # cycle only on tels which still have an image
for tel_id in point_altitude_dict.keys()
},
)
# Impact parameter for energy estimation (/ tel)
subarray = event.inst.subarray
for tel_id in hillas_dict.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# Should be better handled (tilted frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_ground.x - tel_ground.x)**2 +
(core_ground.y - tel_ground.y)**2)
except Exception as e:
print("exception in reconstruction:", e)
raise
if return_stub:
yield stub(event)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if return_stub:
yield stub(event)
else:
continue
yield PreparedEvent(
event=event,
dl1_phe_image=dl1_phe_image,
mc_phe_image=mc_phe_image,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
n_tels=n_tels,
tot_signal=tot_signal,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
)
def filter_planes(wavelet_planes,
method=DEFAULT_TYPE_OF_FILTERING,
thresholds=DEFAULT_FILTER_THRESHOLDS,
detect_only_positive_structures=False):
"""Filter the wavelet planes.
The last plane (called residuals) is kept unmodified.
Parameters
----------
wavelet_planes : list of array_like
The wavelet planes to filter, including the last *residual* plane.
method : str, optional
The filtering method to use. So far, only the 'hard_filtering' and
'ksigma_hard_filtering' methods are implemented.
thresholds : list of float
Thresholds used for the plane filtering.
detect_only_positive_structures : bool
Detect only positive structures.
Returns
-------
list
Return a list containing the filtered wavelet planes.
"""
filtered_wavelet_planes = copy.deepcopy(wavelet_planes)
# The last plane is kept unmodified
for plane_index, plane in enumerate(wavelet_planes[0:-1]):
if method in ('hard_filtering', 'common_hard_filtering'):
with np.errstate(
invalid='ignore'
): # TODO: to disable warnings on images containing "NaN" values (temporary solution)
if detect_only_positive_structures:
plane_mask = plane > thresholds[plane_index]
else:
plane_mask = abs(plane) > thresholds[plane_index]
filtered_plane = plane * plane_mask
elif method == 'ksigma_hard_filtering':
# Compute the standard deviation of the plane ##
plane_noise_std = np.std(
plane
) # TODO: this is wrong... it should be the estimated std of the **noise**
# Apply a threshold on the plane ###############
# Remark: "abs(plane) > (plane_noise_std * 3.)" should be the correct way to
# make the image mask, but sometimes results looks better when all
# negative coefficients are dropped ("plane > (plane_noise_std * 3.)")
if detect_only_positive_structures:
plane_mask = plane > (plane_noise_std *
thresholds[plane_index])
else:
plane_mask = abs(plane) > (plane_noise_std *
thresholds[plane_index])
filtered_plane = plane * plane_mask
elif method == 'cluster_filtering':
if plane_index == 0:
plane_mask = plane > thresholds[plane_index]
filtered_plane = plane * plane_mask
else:
filtered_plane = filter_pixels_clusters(
plane, threshold=thresholds[plane_index])
else:
raise ValueError(
'Unknown method "{}". Should be "hard_filtering" or "ksigma_hard_filtering".'
.format(method))
filtered_wavelet_planes[plane_index] = filtered_plane
if DEBUG:
images.plot(plane, title="Plane {}".format(plane_index))
images.plot(plane_mask,
title="Binary mask for plane {}".format(plane_index))
images.plot(filtered_plane,
title="Filtered plane {}".format(plane_index))
if method == 'common_hard_filtering':
# Use the same significant pixels on each plane
# Init the common pixel mask to "all pixels rejected"
common_significant_pixels_mask = np.zeros(wavelet_planes[0].shape)
for filtered_plane in filtered_wavelet_planes[0:-1]:
current_significant_pixels_mask = (np.isfinite(filtered_plane) *
(filtered_plane != 0))
common_significant_pixels_mask = np.logical_or(
common_significant_pixels_mask,
current_significant_pixels_mask)
for plane_index, plane in enumerate(wavelet_planes[0:-1]):
filtered_plane = plane * common_significant_pixels_mask
filtered_wavelet_planes[plane_index] = filtered_plane
# The next commented part is actually quite useless as a post processing island filtering does more or less the same job...
#elif method == 'cluster_filtering':
# # Only keep first plane's pixels that are *significant* in the others planes
# significant_pixels_mask = np.zeros(filtered_wavelet_planes[0].shape)
# for filtered_plane in filtered_wavelet_planes[1:-1]:
# significant_pixels_mask[filtered_plane != 0] = 1
# filtered_wavelet_planes[0] += filtered_wavelet_planes[0] * significant_pixels_mask
return filtered_wavelet_planes
def prepare_event(self, source, return_stub=False):
# configuration for the camera calibrator
# modifies the integration window to be more like in MARS
# JLK, only for LST!!!!
# Option for integration correction is done above
cfg = Config()
cfg["ChargeExtractorFactory"]["window_width"] = 5
cfg["ChargeExtractorFactory"]["window_shift"] = 2
self.calib = CameraCalibrator(
config=cfg,
extractor_product="LocalPeakIntegrator",
eventsource=source,
tool=None,
)
for event in source:
self.event_cutflow.count("noCuts")
if self.event_cutflow.cut("min2Tels trig",
len(event.dl0.tels_with_data)):
if return_stub:
yield stub(event)
else:
continue
# calibrate the event
self.calib.calibrate(event)
# telescope loop
tot_signal = 0
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for geometry
hillas_dict = {} # for discrimination
n_tels = {
"tot": len(event.dl0.tels_with_data),
"LST": 0,
"MST": 0,
"SST": 0,
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
# To compute impact parameter in tilt system
run_array_direction = event.mcheader.run_array_direction
az, alt = run_array_direction[0], run_array_direction[1]
ground_frame = GroundFrame()
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count("noCuts")
camera = event.inst.subarray.tel[tel_id].camera
# count the current telescope according to its size
tel_type = event.inst.subarray.tel[tel_id].optics.tel_type
# JLK, N telescopes before cut selection are not really interesting for
# discrimination, too much fluctuations
# n_tels[tel_type] += 1
# the camera image as a 1D array and stuff needed for calibration
# Choose gain according to pywicta's procedure
image_1d = simtel_event_to_images(event=event,
tel_id=tel_id,
ctapipe_format=True)
pmt_signal = image_1d.input_image # calibrated image
# clean the image
try:
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.cam_id)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.cam_id)
# Image for score/energy estimation (with clusters)
image_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
except FileNotFoundError as e: # JLK, WHAT?
print(e)
continue
# Apply some selection
if self.image_cutflow.cut("min pixel", image_biggest):
continue
if self.image_cutflow.cut("min charge", np.sum(image_biggest)):
continue
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.cam_id)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
# do the hillas reconstruction of the images
# QUESTION should this change in numpy behaviour be done here
# or within `hillas_parameters` itself?
# JLK: make selection on biggest cluster
with np.errstate(invalid="raise", divide="raise"):
try:
moments_reco = hillas_parameters(
camera,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera, image_extended
) # for discrimination and energy reconstruction
# if width and/or length are zero (e.g. when there is only only one
# pixel or when all pixel are exactly in one row), the
# parametrisation won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_reco):
# print('poor moments')
continue
if self.image_cutflow.cut("close to the edge",
moments_reco, camera.cam_id):
# print('close to the edge')
continue
if self.image_cutflow.cut("bad ellipticity",
moments_reco):
# print('bad ellipticity: w={}, l={}'.format(moments_reco.width, moments_reco.length))
continue
except (FloatingPointError,
hillas.HillasParameterizationError):
continue
point_azimuth_dict[
tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
point_altitude_dict[
tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
n_tels[tel_type] += 1
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0])
tot_signal += moments.intensity
n_tels["reco"] = len(hillas_dict_reco)
n_tels["discri"] = len(hillas_dict)
if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]):
if return_stub:
yield stub(event)
else:
continue
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Reconstruction results
reco_result = self.shower_reco.predict(
hillas_dict_reco,
event.inst,
point_altitude_dict,
point_azimuth_dict,
)
# shower_sys = TiltedGroundFrame(pointing_direction=HorizonFrame(
# az=reco_result.az,
# alt=reco_result.alt
# ))
# Impact parameter for energy estimation (/ tel)
subarray = event.inst.subarray
for tel_id in hillas_dict.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
# tel_tilt = tel_ground.transform_to(shower_sys)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# core_tilt = core_ground.transform_to(shower_sys)
# Should be better handled (tilted frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_ground.x - tel_ground.x)**2 +
(core_ground.y - tel_ground.y)**2)
except Exception as e:
print("exception in reconstruction:", e)
raise
if return_stub:
yield stub(event)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if return_stub:
yield stub(event)
else:
continue
yield PreparedEvent(
event=event,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
n_tels=n_tels,
tot_signal=tot_signal,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
)
