def test_number_of_islands():
from ctapipe.image import number_of_islands
# test with LST geometry (1855 pixels)
geom = CameraGeometry.from_name("LSTCam")
# create 18 triggered pixels grouped to 5 clusters
mask = np.zeros(geom.n_pixels).astype("bool")
triggered_pixels = np.array(
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 37, 38, 111, 222])
mask[triggered_pixels] = True
island_labels_true = np.zeros(geom.n_pixels, dtype=np.int16)
island_labels_true[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]] = 1
island_labels_true[14] = 2
island_labels_true[[37, 38]] = 3
island_labels_true[111] = 4
island_labels_true[222] = 5
n_islands, island_labels = number_of_islands(geom, mask)
n_islands_true = 5
# non cleaning pixels should be zero
assert np.all(island_labels[~mask] == 0)
# all other should have some label
assert np.all(island_labels[mask] > 0)
assert n_islands == n_islands_true
assert_allclose(island_labels, island_labels_true)
assert island_labels.dtype == np.int16
def test_largest_island():
"""Test selection of largest island in imagea with given cleaning masks."""
from ctapipe.image import number_of_islands, largest_island
# Create a simple rectangular camera made of 17 pixels
camera = CameraGeometry.make_rectangular(17, 1)
# Assume a simple image (flattened array) made of 0, 1 or 2 photoelectrons
# [2, 1, 1, 1, 1, 2, 2, 1, 0, 2, 1, 1, 1, 0, 2, 2, 2]
# Assume a virtual tailcut cleaning that requires:
# - picture_threshold = 2 photoelectrons,
# - boundary_threshold = 1 photoelectron,
# - min_number_picture_neighbors = 0
# There will be 4 islands left after cleaning:
clean_mask = np.zeros(camera.n_pixels).astype("bool") # initialization
clean_mask[[0, 1]] = 1
clean_mask[[4, 5, 6, 7]] = 2 # this is the biggest
clean_mask[[9, 10]] = 3
clean_mask[[14, 15, 16]] = 4
# Label islands (their number is not important here)
_, islands_labels = number_of_islands(camera, clean_mask)
# Create the true mask which takes into account only the biggest island
# Pixels with no signal are labelled with a 0
true_mask_largest = np.zeros(camera.n_pixels).astype("bool")
true_mask_largest[[4, 5, 6, 7]] = 1
# Apply the function to test
mask_largest = largest_island(islands_labels)
# Now the degenerate case of only one island surviving
# Same process as before
clean_mask_one = np.zeros(camera.n_pixels).astype("bool")
clean_mask_one[[0, 1]] = 1
_, islands_labels_one = number_of_islands(camera, clean_mask_one)
true_mask_largest_one = np.zeros(camera.n_pixels).astype("bool")
true_mask_largest_one[[0, 1]] = 1
mask_largest_one = largest_island(islands_labels_one)
# Last the case of no islands surviving
clean_mask_0 = np.zeros(camera.n_pixels).astype("bool")
_, islands_labels_0 = number_of_islands(camera, clean_mask_0)
true_mask_largest_0 = np.zeros(camera.n_pixels).astype("bool")
mask_largest_0 = largest_island(islands_labels_0)
# Check if the function recovers the ground truth in all of the three cases
assert (mask_largest_one == true_mask_largest_one).all()
assert (mask_largest_0 == true_mask_largest_0).all()
assert_allclose(mask_largest, true_mask_largest)
def mask_from_biggest_island(charge: np.array, geometry: CameraGeometry, mask):
n_islands, labels = number_of_islands(geometry, mask)
if n_islands == 1:
camera_biggest = geometry[mask]
charge_biggest = charge[mask]
elif n_islands > 1:
mask_biggest = largest_island(labels)
camera_biggest = geometry[mask_biggest]
charge_biggest = charge[mask_biggest]
else:
camera_biggest = geometry
charge_biggest = charge
return charge_biggest, camera_biggest, n_islands
def test_number_of_islands():
from ctapipe.image import number_of_islands
# test with LST geometry (1855 pixels)
geom = CameraGeometry.from_name("LSTCam")
# create 18 triggered pixels grouped to 5 clusters
island_mask_true = np.zeros(geom.n_pixels)
mask = np.zeros(geom.n_pixels).astype("bool")
triggered_pixels = np.array(
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 37, 38, 111, 222])
island_mask_true[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]] = 1
island_mask_true[14] = 2
island_mask_true[[37, 38]] = 3
island_mask_true[111] = 4
island_mask_true[222] = 5
mask[triggered_pixels] = 1
n_islands, island_mask = number_of_islands(geom, mask)
n_islands_true = 5
assert n_islands == n_islands_true
assert_allclose(island_mask, island_mask_true)
assert island_mask.dtype == np.int32
def get_dl1(
calibrated_event,
subarray,
telescope_id,
dl1_container=None,
custom_config={},
):
"""
Return a DL1ParametersContainer of extracted features from a calibrated event.
The DL1ParametersContainer can be passed to be filled if created outside the function
(faster for multiple event processing)
Parameters
----------
calibrated_event: ctapipe event container
subarray: `ctapipe.instrument.subarray.SubarrayDescription`
telescope_id: `int`
dl1_container: DL1ParametersContainer
custom_config: path to a configuration file
configuration used for tailcut cleaning
supersedes the standard configuration
Returns
-------
DL1ParametersContainer
"""
config = replace_config(standard_config, custom_config)
# pop delta_time and use_main_island, so we can pass cleaning_parameters to
# tailcuts
cleaning_parameters = config["tailcut"].copy()
delta_time = cleaning_parameters.pop("delta_time", None)
use_main_island = cleaning_parameters.pop("use_only_main_island", True)
use_dynamic_cleaning = False
if "apply" in config["dynamic_cleaning"]:
use_dynamic_cleaning = config["dynamic_cleaning"]["apply"]
dl1_container = DL1ParametersContainer(
) if dl1_container is None else dl1_container
dl1 = calibrated_event.dl1.tel[telescope_id]
telescope = subarray.tel[telescope_id]
camera_geometry = telescope.camera.geometry
optics = telescope.optics
image = dl1.image
peak_time = dl1.peak_time
signal_pixels = cleaning_method(camera_geometry, image,
**cleaning_parameters)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
if delta_time is not None:
signal_pixels = apply_time_delta_cleaning(camera_geometry,
signal_pixels,
peak_time,
min_number_neighbors=1,
time_limit=delta_time)
if use_dynamic_cleaning:
threshold_dynamic = config['dynamic_cleaning']['threshold']
fraction_dynamic = config['dynamic_cleaning'][
'fraction_cleaning_intensity']
signal_pixels = apply_dynamic_cleaning(image, signal_pixels,
threshold_dynamic,
fraction_dynamic)
# check the number of islands
num_islands, island_labels = number_of_islands(camera_geometry,
signal_pixels)
dl1_container.n_islands = num_islands
if use_main_island:
n_pixels_on_island = np.bincount(island_labels)
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
signal_pixels[island_labels != max_island_label] = False
# count surviving pixels
n_pixels = np.count_nonzero(signal_pixels)
dl1_container.n_pixels = n_pixels
if n_pixels > 0:
parametrize_image(
image=image,
peak_time=peak_time,
signal_pixels=signal_pixels,
camera_geometry=camera_geometry,
focal_length=optics.equivalent_focal_length,
dl1_container=dl1_container,
)
# We set other fields which still make sense for a non-parametrized
# image:
dl1_container.set_telescope_info(subarray, telescope_id)
# save the applied cleaning mask
calibrated_event.dl1.tel[telescope_id].image_mask = signal_pixels
return dl1_container
def prepare_event(self,
source,
return_stub=True,
save_images=False,
debug=False):
"""
Calibrate, clean and reconstruct the direction of an event.
Parameters
----------
source : ctapipe.io.EventSource
A container of selected showers from a simtel file.
geom_cam_tel: dict
Dictionary of of MyCameraGeometry objects for each camera in the file
return_stub : bool
If True, yield also images from events that won't be reconstructed.
This feature is not currently available.
save_images : bool
If True, save photoelectron images from reconstructed events.
debug : bool
If True, print some debugging information (to be expanded).
Yields
------
PreparedEvent: dict
Dictionary containing event-image information to be written.
"""
# =============================================================
# TRANSFORMED CAMERA GEOMETRIES
# =============================================================
# These are the camera geometries were the Hillas parametrization will
# be performed.
# They are transformed to TelescopeFrame using the effective focal
# lengths
# These geometries could be used also to performe the image cleaning,
# but for the moment we still do that in the CameraFrame
geom_cam_tel = {}
for camera in source.subarray.camera_types:
# Original geometry of each camera
geom = camera.geometry
# Same but with focal length as an attribute
# This is planned to disappear and be imported by ctapipe
focal_length = effective_focal_lengths(camera.camera_name)
geom_cam_tel[camera.camera_name] = MyCameraGeometry(
camera_name=camera.camera_name,
pix_type=geom.pix_type,
pix_id=geom.pix_id,
pix_x=geom.pix_x,
pix_y=geom.pix_y,
pix_area=geom.pix_area,
cam_rotation=geom.cam_rotation,
pix_rotation=geom.pix_rotation,
frame=CameraFrame(focal_length=focal_length),
).transform_to(TelescopeFrame())
# =============================================================
ievt = 0
for event in source:
# Display event counts
if debug:
print(
bcolors.BOLD +
f"EVENT #{event.count}, EVENT_ID #{event.index.event_id}" +
bcolors.ENDC)
print(bcolors.BOLD +
f"has triggered telescopes {event.r1.tel.keys()}" +
bcolors.ENDC)
ievt += 1
# if (ievt < 10) or (ievt % 10 == 0):
# print(ievt)
self.event_cutflow.count("noCuts")
# LST stereo condition
# whenever there is only 1 LST in an event, we remove that telescope
# if the remaining telescopes are less than min_tel we remove the event
lst_tel_ids = set(
source.subarray.get_tel_ids_for_type("LST_LST_LSTCam"))
triggered_LSTs = set(event.r0.tel.keys()).intersection(lst_tel_ids)
n_triggered_LSTs = len(triggered_LSTs)
n_triggered_non_LSTs = len(event.r0.tel.keys()) - n_triggered_LSTs
bad_LST_stereo = False
if self.LST_stereo and self.event_cutflow.cut(
"no-LST-stereo + <2 other types", n_triggered_LSTs,
n_triggered_non_LSTs):
bad_LST_stereo = True
if return_stub:
print(
bcolors.WARNING +
"WARNING: LST_stereo is set to 'True'\n" +
f"This event has < {self.min_ntel_LST} triggered LSTs\n"
+
"and < 2 triggered telescopes from other telescope types.\n"
+ "The event will be processed up to DL1b." +
bcolors.ENDC)
# we show this, but we proceed to analyze the event up to
# DL1a/b for the associated benchmarks
# this checks for < 2 triggered telescopes of ANY type
if self.event_cutflow.cut("min2Tels trig",
len(event.r1.tel.keys())):
if return_stub:
print(
bcolors.WARNING +
f"WARNING : < {self.min_ntel} triggered telescopes!" +
bcolors.ENDC)
# we show this, but we proceed to analyze it
# =============================================================
# CALIBRATION
# =============================================================
if debug:
print(bcolors.OKBLUE + "Extracting all calibrated images..." +
bcolors.ENDC)
self.calib(event) # Calibrate the event
# =============================================================
# BEGINNING OF LOOP OVER TELESCOPES
# =============================================================
dl1_phe_image = {}
dl1_phe_image_mask_reco = {}
dl1_phe_image_mask_clusters = {}
mc_phe_image = {}
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for direction reconstruction
hillas_dict = {} # for discrimination
leakage_dict = {}
concentration_dict = {}
n_tels = {
"Triggered": len(event.r1.tel.keys()),
"LST_LST_LSTCam": 0,
"MST_MST_NectarCam": 0,
"MST_MST_FlashCam": 0,
"MST_SCT_SCTCam": 0,
"SST_1M_DigiCam": 0,
"SST_ASTRI_ASTRICam": 0,
"SST_GCT_CHEC": 0,
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
good_for_reco = {} # 1 = success, 0 = fail
# Array pointing in AltAz frame
az = event.pointing.array_azimuth
alt = event.pointing.array_altitude
array_pointing = SkyCoord(az, alt, frame=AltAz())
ground_frame = GroundFrame()
tilted_frame = TiltedGroundFrame(pointing_direction=array_pointing)
for tel_id in event.r1.tel.keys():
point_azimuth_dict[tel_id] = event.pointing.tel[tel_id].azimuth
point_altitude_dict[tel_id] = event.pointing.tel[
tel_id].altitude
if debug:
print(bcolors.OKBLUE +
f"Working on telescope #{tel_id}..." + bcolors.ENDC)
self.image_cutflow.count("noCuts")
camera = source.subarray.tel[tel_id].camera.geometry
# count the current telescope according to its type
tel_type = str(source.subarray.tel[tel_id])
n_tels[tel_type] += 1
# use ctapipe's functionality to get the calibrated image
# and scale the reconstructed values if required
pmt_signal = event.dl1.tel[tel_id].image / self.calibscale
# If required...
if save_images is True:
# Save the simulated and reconstructed image of the event
dl1_phe_image[tel_id] = pmt_signal
mc_phe_image[tel_id] = event.simulation.tel[
tel_id].true_image
# We now ASSUME that the event will be good
good_for_reco[tel_id] = 1
# later we change to 0 if any condition is NOT satisfied
if self.cleaner_reco.mode == "tail": # tail uses only ctapipe
# Cleaning used for direction reconstruction
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
# calculate the leakage (before filtering)
leakages = {} # this is needed by both cleanings
# The check on SIZE shouldn't be here, but for the moment
# I prefer to sacrifice elegancy...
if np.sum(image_biggest[mask_reco]) != 0.0:
leakage_biggest = leakage_parameters(
camera, image_biggest, mask_reco)
leakages["leak1_reco"] = leakage_biggest[
"intensity_width_1"]
leakages["leak2_reco"] = leakage_biggest[
"intensity_width_2"]
else:
leakages["leak1_reco"] = 0.0
leakages["leak2_reco"] = 0.0
# find all islands using this cleaning
num_islands, labels = number_of_islands(camera, mask_reco)
if num_islands == 1: # if only ONE islands is left ...
# ...use directly the old mask and reduce dimensions
# to make Hillas parametrization faster
camera_biggest = camera[mask_reco]
image_biggest = image_biggest[mask_reco]
if save_images is True:
dl1_phe_image_mask_reco[tel_id] = mask_reco
elif num_islands > 1: # if more islands survived..
# ...find the biggest one
mask_biggest = largest_island(labels)
# and also reduce dimensions
camera_biggest = camera[mask_biggest]
image_biggest = image_biggest[mask_biggest]
if save_images is True:
dl1_phe_image_mask_reco[tel_id] = mask_biggest
else: # if no islands survived use old camera and image
camera_biggest = camera
dl1_phe_image_mask_reco[tel_id] = mask_reco
# Cleaning used for score/energy estimation
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
dl1_phe_image_mask_clusters[tel_id] = mask_extended
# calculate the leakage (before filtering)
# this part is not well coded, but for the moment it works
if np.sum(image_extended[mask_extended]) != 0.0:
leakage_extended = leakage_parameters(
camera, image_extended, mask_extended)
leakages["leak1"] = leakage_extended[
"intensity_width_1"]
leakages["leak2"] = leakage_extended[
"intensity_width_2"]
else:
leakages["leak1"] = 0.0
leakages["leak2"] = 0.0
# find all islands with this cleaning
# we will also register how many have been found
n_cluster_dict[tel_id], labels = number_of_islands(
camera, mask_extended)
# NOTE: the next check shouldn't be necessary if we keep
# all the isolated pixel clusters, but for now the
# two cleanings are set the same in analysis.yml because
# the performance of the extended one has never been really
# studied in model estimation.
# if some islands survived
if n_cluster_dict[tel_id] > 0:
# keep all of them and reduce dimensions
camera_extended = camera[mask_extended]
image_extended = image_extended[mask_extended]
else: # otherwise continue with the old camera and image
camera_extended = camera
# could this go into `hillas_parameters` ...?
# this is basically the charge of ALL islands
# not calculated later by the Hillas parametrization!
max_signals[tel_id] = np.max(image_extended)
else: # for wavelets we stick to old pywi-cta code
try: # "try except FileNotFoundError" not clear to me, but for now it stays...
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.camera_name)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.camera_name)
# Image for score/energy estimation (with clusters)
(
image_extended,
mask_extended,
) = self.cleaner_extended.clean_image(
pmt_signal, camera)
# This last part was outside the pywi-cta block
# before, but is indeed part of it because it uses
# pywi-cta functions in the "extended" case
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.camera_name)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
except FileNotFoundError as e:
print(e)
continue
# =============================================================
# PRELIMINARY IMAGE SELECTION
# =============================================================
cleaned_image_is_good = True # we assume this
if self.image_selection_source == "extended":
cleaned_image_to_use = image_extended
elif self.image_selection_source == "biggest":
cleaned_image_to_use = image_biggest
else:
raise ValueError(
"Only supported cleanings are 'biggest' or 'extended'."
)
# Apply some selection
if self.image_cutflow.cut("min pixel", cleaned_image_to_use):
if debug:
print(bcolors.WARNING +
"WARNING : not enough pixels!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
cleaned_image_is_good = False
if self.image_cutflow.cut("min charge",
np.sum(cleaned_image_to_use)):
if debug:
print(bcolors.WARNING +
"WARNING : not enough charge!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
cleaned_image_is_good = False
if debug and (not cleaned_image_is_good): # BAD image quality
print(bcolors.WARNING +
"WARNING : The cleaned image didn't pass" +
" preliminary cuts.\n" +
"An attempt to parametrize it will be made," +
" but the image will NOT be used for" +
" direction reconstruction." + bcolors.ENDC)
# =============================================================
# IMAGE PARAMETRIZATION
# =============================================================
with np.errstate(invalid="raise", divide="raise"):
try:
# Filter the cameras in TelescopeFrame with the same
# cleaning masks
camera_biggest_tel = geom_cam_tel[camera.camera_name][
camera_biggest.pix_id]
camera_extended_tel = geom_cam_tel[camera.camera_name][
camera_extended.pix_id]
# Parametrize the image in the TelescopeFrame
moments_reco = hillas_parameters(
camera_biggest_tel,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera_extended_tel, image_extended
) # for discrimination and energy reconstruction
if debug:
print(
"Image parameters from main cluster cleaning:")
print(moments_reco)
print(
"Image parameters from all-clusters cleaning:")
print(moments)
# Add concentration parameters
concentrations = {}
concentrations_extended = concentration_parameters(
camera_extended_tel, image_extended, moments)
concentrations[
"concentration_cog"] = concentrations_extended[
"cog"]
concentrations[
"concentration_core"] = concentrations_extended[
"core"]
concentrations[
"concentration_pixel"] = concentrations_extended[
"pixel"]
# ===================================================
# PARAMETRIZED IMAGE SELECTION
# ===================================================
if self.image_selection_source == "extended":
moments_to_use = moments
else:
moments_to_use = moments_reco
# if width and/or length are zero (e.g. when there is
# only only one pixel or when all pixel are exactly
# in one row), the parametrisation
# won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_to_use):
if debug:
print(bcolors.WARNING +
"WARNING : poor moments!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
if self.image_cutflow.cut("close to the edge",
moments_to_use,
camera.camera_name):
if debug:
print(
bcolors.WARNING +
"WARNING : out of containment radius!\n" +
f"Camera radius = {self.camera_radius[camera.camera_name]}\n"
+ f"COG radius = {moments_to_use.r}" +
bcolors.ENDC)
good_for_reco[tel_id] = 0
if self.image_cutflow.cut("bad ellipticity",
moments_to_use):
if debug:
print(bcolors.WARNING +
"WARNING : bad ellipticity" +
bcolors.ENDC)
good_for_reco[tel_id] = 0
if debug and good_for_reco[tel_id] == 1:
print(
bcolors.OKGREEN +
"Image survived and correctly parametrized."
# + "\nIt will be used for direction reconstruction!"
+ bcolors.ENDC)
elif debug and good_for_reco[tel_id] == 0:
print(
bcolors.WARNING + "Image not survived or " +
"not good enough for parametrization."
# + "\nIt will be NOT used for direction reconstruction, "
# + "BUT it's information will be recorded."
+ bcolors.ENDC)
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(
np.where(image_extended > 0)[0])
leakage_dict[tel_id] = leakages
concentration_dict[tel_id] = concentrations
except (
FloatingPointError,
HillasParameterizationError,
ValueError,
) as e:
if debug:
print(bcolors.FAIL + "Parametrization error: " +
f"{e}\n" + "Dummy parameters recorded." +
bcolors.ENDC)
good_for_reco[tel_id] = 0
hillas_dict[
tel_id] = HillasParametersTelescopeFrameContainer(
)
hillas_dict_reco[
tel_id] = HillasParametersTelescopeFrameContainer(
)
n_pixel_dict[tel_id] = len(
np.where(image_extended > 0)[0])
leakage_dict[tel_id] = leakages
concentration_dict[tel_id] = concentrations
# END OF THE CYCLE OVER THE TELESCOPES
# =============================================================
# DIRECTION RECONSTRUCTION
# =============================================================
if bad_LST_stereo:
if debug:
print(
bcolors.WARNING +
"WARNING: This event was triggered with 1 LST image and <2 images from other telescope types."
+
"\nWARNING : direction reconstruction will not be performed."
+ bcolors.ENDC)
# Set all the involved images as NOT good for recosntruction
# even though they might have been
# but this is because of the LST stereo trigger....
for tel_id in event.r0.tel.keys():
good_for_reco[tel_id] = 0
# and set the number of good and bad images accordingly
n_tels["GOOD images"] = 0
n_tels[
"BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"]
# create a dummy container for direction reconstruction
reco_result = ReconstructedShowerContainer()
if return_stub: # if saving all events (default)
if debug:
print(bcolors.OKBLUE + "Recording event..." +
bcolors.ENDC)
print(
bcolors.WARNING +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
continue
else:
continue
# Now in case the only triggered telescopes were
# - < self.min_ntel_LST LST,
# - >=2 any other telescope type,
# we remove the single-LST image and continue reconstruction with
# the images from the other telescope types
if self.LST_stereo and (n_triggered_LSTs < self.min_ntel_LST) and (
n_triggered_LSTs != 0) and (n_triggered_non_LSTs >= 2):
if debug:
print(
bcolors.WARNING +
f"WARNING: LST stereo trigger condition is active.\n" +
f"This event triggered < {self.min_ntel_LST} LSTs " +
f"and {n_triggered_non_LSTs} images from other telescope types.\n"
+ bcolors.ENDC)
for tel_id in triggered_LSTs: # in case we test for min_ntel_LST>2
if good_for_reco[tel_id]:
# we don't use it for reconstruction
good_for_reco[tel_id] = 0
print(
bcolors.WARNING +
f"WARNING: LST image #{tel_id} removed, even though it passed quality cuts."
+ bcolors.ENDC)
# TODO: book-keeping of this kind of events doesn't seem easy
# convert dictionary in numpy array to get a "mask"
images_status = np.asarray(list(good_for_reco.values()))
# record how many images will be used for reconstruction
n_tels["GOOD images"] = len(
np.extract(images_status == 1, images_status))
n_tels["BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"]
if self.event_cutflow.cut("min2Tels reco", n_tels["GOOD images"]):
if debug:
print(
bcolors.FAIL +
f"WARNING: < {self.min_ntel} ({n_tels['GOOD images']}) images remaining!"
+
"\nWARNING : direction reconstruction is not possible!"
+ bcolors.ENDC)
# create a dummy container for direction reconstruction
reco_result = ReconstructedShowerContainer()
if return_stub: # if saving all events (default)
if debug:
print(bcolors.OKBLUE + "Recording event..." +
bcolors.ENDC)
print(
bcolors.WARNING +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
continue
else:
continue
if debug:
print(bcolors.OKBLUE + "Starting direction reconstruction..." +
bcolors.ENDC)
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
if self.image_selection_source == "extended":
hillas_dict_to_use = hillas_dict
else:
hillas_dict_to_use = hillas_dict_reco
# use only the successfully parametrized images
# to reconstruct the direction of this event
successfull_hillas = np.where(images_status == 1)[0]
all_images = np.asarray(list(good_for_reco.keys()))
good_images = set(all_images[successfull_hillas])
good_hillas_dict = {
k: v
for k, v in hillas_dict_to_use.items()
if k in good_images
}
if debug:
print(
bcolors.PURPLE +
f"{len(good_hillas_dict)} images " +
f"(from telescopes #{list(good_hillas_dict.keys())}) will be "
+ "used to recover the shower's direction..." +
bcolors.ENDC)
# Reconstruction results
reco_result = self.shower_reco.predict(
good_hillas_dict,
source.subarray,
SkyCoord(alt=alt, az=az, frame="altaz"),
None, # use the array direction
)
# Impact parameter for telescope-wise energy estimation
subarray = source.subarray
for tel_id in hillas_dict_to_use.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# Go back to the tilted frame
# this should be the same...
tel_tilted = tel_ground.transform_to(tilted_frame)
# but this not
core_tilted = SkyCoord(x=core_ground.x,
y=core_ground.y,
frame=tilted_frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_tilted.x - tel_tilted.x)**2 +
(core_tilted.y - tel_tilted.y)**2)
except (Exception, TooFewTelescopesException,
InvalidWidthException) as e:
if debug:
print("exception in reconstruction:", e)
raise
if return_stub:
if debug:
print(
bcolors.FAIL +
"Shower could NOT be correctly reconstructed! " +
"Recording event..." +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if debug:
print(bcolors.WARNING + "WARNING: undefined direction!" +
bcolors.ENDC)
if return_stub:
if debug:
print(
bcolors.FAIL +
"Shower could NOT be correctly reconstructed! " +
"Recording event..." +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
else:
continue
if debug:
print(bcolors.BOLDGREEN + "Shower correctly reconstructed! " +
"Recording event..." + bcolors.ENDC)
yield PreparedEvent(
event=event,
dl1_phe_image=dl1_phe_image,
dl1_phe_image_mask_reco=dl1_phe_image_mask_reco,
dl1_phe_image_mask_clusters=dl1_phe_image_mask_clusters,
mc_phe_image=mc_phe_image,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
leakage_dict=leakage_dict,
concentration_dict=concentration_dict,
n_tels=n_tels,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
good_event=True,
good_for_reco=good_for_reco,
)
def main():
args = parser.parse_args()
log.setLevel(logging.INFO)
handler = logging.StreamHandler()
logging.getLogger().addHandler(handler)
if Path(args.output_file).exists():
log.critical(f'Output file {args.output_file} already exists')
sys.exit(1)
std_config = get_standard_config()
if args.config_file is not None:
config = replace_config(std_config,
read_configuration_file(args.config_file))
else:
config = std_config
with tables.open_file(args.input_file, 'r') as f:
is_simulation = 'simulation' in f.root
increase_nsb = False
increase_psf = False
if "image_modifier" in config:
imconfig = config["image_modifier"]
increase_nsb = imconfig["increase_nsb"]
increase_psf = imconfig["increase_psf"]
if increase_nsb or increase_psf:
log.info(f"image_modifier configuration: {imconfig}")
extra_noise_in_dim_pixels = imconfig["extra_noise_in_dim_pixels"]
extra_bias_in_dim_pixels = imconfig["extra_bias_in_dim_pixels"]
transition_charge = imconfig["transition_charge"]
extra_noise_in_bright_pixels = imconfig["extra_noise_in_bright_pixels"]
smeared_light_fraction = imconfig["smeared_light_fraction"]
if (increase_nsb or increase_psf):
log.info(
"NOTE: Using the image_modifier options means images will "
"not be saved.")
args.no_image = True
if is_simulation:
args.pedestal_cleaning = False
if args.pedestal_cleaning:
log.info("Pedestal cleaning")
clean_method_name = 'tailcuts_clean_with_pedestal_threshold'
sigma = config[clean_method_name]['sigma']
pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma)
cleaning_params = get_cleaning_parameters(config, clean_method_name)
pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(
f"Fraction of pixel cleaning thresholds above picture thr.:"
f"{np.sum(pedestal_thresh > pic_th) / len(pedestal_thresh):.3f}")
picture_th = np.clip(pedestal_thresh, pic_th, None)
log.info(f"Tailcut clean with pedestal threshold config used:"
f"{config['tailcuts_clean_with_pedestal_threshold']}")
else:
clean_method_name = 'tailcut'
cleaning_params = get_cleaning_parameters(config, clean_method_name)
picture_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params
log.info(f"Tailcut config used: {config['tailcut']}")
use_dynamic_cleaning = False
if 'apply' in config['dynamic_cleaning']:
use_dynamic_cleaning = config['dynamic_cleaning']['apply']
if use_dynamic_cleaning:
THRESHOLD_DYNAMIC_CLEANING = config['dynamic_cleaning']['threshold']
FRACTION_CLEANING_SIZE = config['dynamic_cleaning'][
'fraction_cleaning_intensity']
log.info(
"Using dynamic cleaning for events with average size of the "
f"3 most brighest pixels > {config['dynamic_cleaning']['threshold']} p.e"
)
log.info(
"Remove from image pixels which have charge below "
f"= {config['dynamic_cleaning']['fraction_cleaning_intensity']} * average size"
)
use_only_main_island = True
if "use_only_main_island" in config[clean_method_name]:
use_only_main_island = config[clean_method_name][
"use_only_main_island"]
delta_time = None
if "delta_time" in config[clean_method_name]:
delta_time = config[clean_method_name]["delta_time"]
subarray_info = SubarrayDescription.from_hdf(args.input_file)
tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1
optics = subarray_info.tel[tel_id].optics
camera_geom = subarray_info.tel[tel_id].camera.geometry
dl1_container = DL1ParametersContainer()
parameters_to_update = [
'intensity', 'x', 'y', 'r', 'phi', 'length', 'width', 'psi',
'skewness', 'kurtosis', 'concentration_cog', 'concentration_core',
'concentration_pixel', 'leakage_intensity_width_1',
'leakage_intensity_width_2', 'leakage_pixels_width_1',
'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient',
'n_pixels', 'wl', 'log_intensity'
]
nodes_keys = get_dataset_keys(args.input_file)
if args.no_image:
nodes_keys.remove(dl1_images_lstcam_key)
metadata = global_metadata()
with tables.open_file(args.input_file, mode='r') as infile:
image_table = infile.root[dl1_images_lstcam_key]
dl1_params_input = infile.root[dl1_params_lstcam_key].colnames
disp_params = {
'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign'
}
if set(dl1_params_input).intersection(disp_params):
parameters_to_update.extend(disp_params)
uncertainty_params = {'width_uncertainty', 'length_uncertainty'}
if set(dl1_params_input).intersection(uncertainty_params):
parameters_to_update.extend(uncertainty_params)
if increase_nsb:
rng = np.random.default_rng(
infile.root.dl1.event.subarray.trigger.col('obs_id')[0])
if increase_psf:
set_numba_seed(
infile.root.dl1.event.subarray.trigger.col('obs_id')[0])
image_mask_save = not args.no_image and 'image_mask' in infile.root[
dl1_images_lstcam_key].colnames
with tables.open_file(args.output_file,
mode='a',
filters=HDF5_ZSTD_FILTERS) as outfile:
copy_h5_nodes(infile, outfile, nodes=nodes_keys)
add_source_filenames(outfile, [args.input_file])
params = outfile.root[dl1_params_lstcam_key].read()
if image_mask_save:
image_mask = outfile.root[dl1_images_lstcam_key].col(
'image_mask')
# need container to use lstchain.io.add_global_metadata and lstchain.io.add_config_metadata
for k, item in metadata.as_dict().items():
outfile.root[dl1_params_lstcam_key].attrs[k] = item
outfile.root[dl1_params_lstcam_key].attrs["config"] = str(config)
for ii, row in enumerate(image_table):
dl1_container.reset()
image = row['image']
peak_time = row['peak_time']
if increase_nsb:
# Add noise in pixels, to adjust MC to data noise levels.
# TO BE DONE: in case of "pedestal cleaning" (not used now
# in MC) we should recalculate picture_th above!
image = add_noise_in_pixels(rng, image,
extra_noise_in_dim_pixels,
extra_bias_in_dim_pixels,
transition_charge,
extra_noise_in_bright_pixels)
if increase_psf:
image = random_psf_smearer(
image, smeared_light_fraction,
camera_geom.neighbor_matrix_sparse.indices,
camera_geom.neighbor_matrix_sparse.indptr)
signal_pixels = tailcuts_clean(camera_geom, image, picture_th,
boundary_th, isolated_pixels,
min_n_neighbors)
n_pixels = np.count_nonzero(signal_pixels)
if n_pixels > 0:
# if delta_time has been set, we require at least one
# neighbor within delta_time to accept a pixel in the image:
if delta_time is not None:
cleaned_pixel_times = peak_time
# makes sure only signal pixels are used in the time
# check:
cleaned_pixel_times[~signal_pixels] = np.nan
new_mask = apply_time_delta_cleaning(
camera_geom, signal_pixels, cleaned_pixel_times, 1,
delta_time)
signal_pixels = new_mask
if use_dynamic_cleaning:
new_mask = apply_dynamic_cleaning(
image, signal_pixels, THRESHOLD_DYNAMIC_CLEANING,
FRACTION_CLEANING_SIZE)
signal_pixels = new_mask
# count a number of islands after all of the image cleaning steps
num_islands, island_labels = number_of_islands(
camera_geom, signal_pixels)
dl1_container.n_islands = num_islands
n_pixels_on_island = np.bincount(
island_labels.astype(np.int64))
n_pixels_on_island[
0] = 0 # first island is no-island and should not be considered
max_island_label = np.argmax(n_pixels_on_island)
if use_only_main_island:
signal_pixels[
island_labels != max_island_label] = False
# count the surviving pixels
n_pixels = np.count_nonzero(signal_pixels)
dl1_container.n_pixels = n_pixels
if n_pixels > 0:
parametrize_image(
image=image,
peak_time=peak_time,
signal_pixels=signal_pixels,
camera_geometry=camera_geom,
focal_length=optics.equivalent_focal_length,
dl1_container=dl1_container,
)
if set(dl1_params_input).intersection(disp_params):
disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp(
dl1_container['x'].to_value(u.m),
dl1_container['y'].to_value(u.m), params['src_x'][ii],
params['src_y'][ii])
dl1_container['disp_dx'] = disp_dx
dl1_container['disp_dy'] = disp_dy
dl1_container['disp_norm'] = disp_norm
dl1_container['disp_angle'] = disp_angle
dl1_container['disp_sign'] = disp_sign
for p in parameters_to_update:
params[ii][p] = u.Quantity(dl1_container[p]).value
if image_mask_save:
image_mask[ii] = signal_pixels
outfile.root[dl1_params_lstcam_key][:] = params
if image_mask_save:
outfile.root[dl1_images_lstcam_key].modify_column(
colname='image_mask', column=image_mask)
write_metadata(metadata, args.output_file)
def process_telescope(tel, dl1, stereo):
geom = tel.camera.geometry
camera_name = tel.camera.camera_name
image = dl1.image
peak_time = dl1.peak_time
# Cleaning using CHEC method
clean = obtain_cleaning_mask(geom, image, peak_time, camera_name)
# Skipping inadequate events
if clean.sum() == 0:
return None, None, None
if stereo and clean.sum() < 5:
return None, None, None
# Get hillas parameters
hillas_c = hillas_parameters(geom[clean], image[clean])
if hillas_c.width == 0 or np.isnan(hillas_c.width.value):
return None, None, None
# Get leakage and islands
leakage_c = leakage(geom, image, clean)
n_islands, island_ids = number_of_islands(geom, clean)
# Get time gradient
tgrad = np.nan
try:
timing_c = timing_parameters(geom, image, peak_time, hillas_c, clean)
tgrad = timing_c.slope.value
except BaseException:
print("Timing parameters didn't work. clean.sum() = " +
str(clean.sum()), "\n")
# Grab info for telescope_events
tel_data_dict = {
'nislands': n_islands,
'telescope_type': tel.type,
'camera_type': tel.camera.camera_name,
'focal_length': tel.optics.equivalent_focal_length.value,
'n_survived_pixels': clean.sum(),
'tgradient': tgrad,
'x': hillas_c.x.value,
'y': hillas_c.y.value,
'r': hillas_c.r.value,
'phi': hillas_c.phi.value,
'intensity': hillas_c.intensity,
'length': hillas_c.length.value,
'width': hillas_c.width.value,
'psi': hillas_c.psi.value,
'skewness': hillas_c.skewness,
'kurtosis': hillas_c.kurtosis,
'pixels_width_1': leakage_c.pixels_width_1,
'pixels_width_2': leakage_c.pixels_width_2,
'intensity_width_1': leakage_c.intensity_width_1,
'intensity_width_2': leakage_c.intensity_width_2}
return tel.type, tel_data_dict, hillas_c
