def get_observation_parameters(charge: np.array,
peak: np.array,
cam_name: str,
cutflow: CutFlow,
boundary_threshold: float = None,
picture_threshold: float = None,
min_neighbours: float = None,
plot: bool = False,
cut: bool = True):
"""
:param charge: Charge image
:param peak: Peak time image
:param cam_name: Camera name. e.g. FlashCam, ASTRICam, etc.
:param cutflow: Cutflow for selection
:param boundary_threshold: (Optional) Cleaning parameter: boundary threshold
:param picture_threshold: (Optional) Cleaning parameter: picture threshold
:param min_neighbours: (Optional) Cleaning parameter: minimum neighbours
:param plot: If True, for each observation a plot will be shown (Default: False)
:param cut: If true, tight else loose
:return: hillas containers, leakage container, number of islands, island IDs, timing container, timing gradient
"""
charge_biggest, mask = clean_charge(charge, cam_name, boundary_threshold,
picture_threshold, min_neighbours)
camera = get_camera(cam_name)
geometry = camera.geometry
charge_biggest, camera_biggest, n_islands = mask_from_biggest_island(
charge, geometry, mask)
if cut:
if cutflow.cut(CFO_MIN_PIXEL, charge_biggest):
return
if cutflow.cut(CFO_MIN_CHARGE, np.sum(charge_biggest)):
return
if cutflow.cut(CFO_NEGATIVE_CHARGE, charge_biggest):
return
leakage_c = leakage(geometry, charge, mask)
if plot:
_, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)
CameraDisplay(geometry, charge, ax=ax1).add_colorbar()
CameraDisplay(camera_biggest, charge_biggest, ax=ax2).add_colorbar()
plt.show()
moments = hillas_parameters(camera_biggest, charge_biggest)
if cut:
if cutflow.cut(CFO_CLOSE_EDGE, moments, camera.camera_name):
return
if cutflow.cut(CFO_BAD_ELLIP, moments):
return
if cutflow.cut(CFO_POOR_MOMENTS, moments):
return
timing_c = timing_parameters(geometry, charge, peak, moments, mask)
time_gradient = timing_c.slope.value if geometry.camera_name != 'ASTRICam' else moments.skewness
return moments, leakage_c, timing_c, time_gradient, n_islands
def leak(img):
cleanmask = cleaning.tailcuts_clean(geom, img, **opts)
mask = False
if any(cleanmask):
leakage_values = leakage(geom, img, cleanmask)
if hasattr(leakage_values,
'leakage{}_intensity'.format(leakage_number)):
mask = leakage_values['leakage{}_intensity'.format(
leakage_number)] <= leakage_value
elif hasattr(leakage_values,
'intensity_width_{}'.format(leakage_number)):
mask = leakage_values['intensity_width_{}'.format(
leakage_number)] <= leakage_value
return mask
def set_leakage(self, geom, image, clean):
leakage_c = leakage(geom, image, clean)
self.leakage_intensity_width_1 = leakage_c.intensity_width_1
self.leakage_intensity_width_2 = leakage_c.intensity_width_2
self.leakage_pixels_width_1 = leakage_c.pixels_width_1
self.leakage_pixels_width_2 = leakage_c.pixels_width_2
def set_leakage(self, geom, image, clean):
leakage_c = leakage(geom, image, clean)
self.leakage = leakage_c.leakage2_intensity
def func(paths, outpath, ro, rc, rn):
# iterate on each proton file & concatenate charge arrays
for n, f in enumerate(paths):
# get the data from the file
try:
print("Opening file #{}...".format(n))
data_p = tables.open_file(f)
_, LST_event_index, LST_image_charge, LST_image_peak_times = get_LST_data(data_p)
_, ei_alt, ei_az, ei_mc_energy = get_event_data(data_p)
# Excluding the 0th element - it's the empty one!!
LST_event_index = LST_event_index[1:]
LST_image_charge = LST_image_charge[1:]
LST_image_peak_times = LST_image_peak_times[1:]
# get camera geometry & camera pixels coordinates
camera = CameraGeometry.from_name("LSTCam")
points = np.array([np.array(camera.pix_x / u.m), np.array(camera.pix_y / u.m)]).T
# original choice by Nicola: 100x100 points in 2.5m x 2.5m
#grid_x, grid_y = np.mgrid[-1.25:1.25:100j, -1.25:1.25:100j]
# I choose instead 96x88 px in 2.40m x 2.20m: same spatial separation, less points
grid_x, grid_y = np.mgrid[-1.20:1.20:96j, -1.10:1.10:88j]
'''
was probably useless and wrong even with the old simulations
if alt_array > 90:
alt_array = 90
'''
# LST coordinates (pointing position)
#point = AltAz(alt=alt_array * u.deg, az=az_array * u.deg)
lst_image_charge_interp = []
lst_image_peak_times_interp = []
# alt az of the array [deg]
#az_array = 180. # before was ai_run_array_direction[0][0], now it is hardcoded as it's not present in new files
#alt_array = 70. # ai_run_array_direction[0][1]
#delta_az = []
#delta_alt = []
intensities = []
intensities_width_2 = []
acc_idxs = [] # accepted indexes # in principle it can be removed when cuts (line AAA) are not used here
cleaning_level = {'LSTCam': (3.5, 7.5, 2)}
count = 0
#rejected = open(f[:-3] + "_rejected.txt", "w")
for i in trange(0, len(LST_image_charge), desc="Images interpolation"):
image = LST_image_charge[i]
time = LST_image_peak_times[i]
boundary, picture, min_neighbors = cleaning_level['LSTCam']
clean = tailcuts_clean(
camera,
image,
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors
)
if len(np.where(clean > 0)[0]) != 0:
hillas = hillas_parameters(camera[clean], image[clean])
intensity = hillas['intensity']
l = leakage(camera, image, clean)
# print(l)
leakage2_intensity = l['intensity_width_2']
# if intensity > 50 and leakage2_intensity < 0.2: # ------>AAA --- CUT DIRECTLY DURING INTERP
# cubic interpolation
interp_img = griddata(points, image, (grid_x, grid_y), fill_value=0, method='cubic')
interp_time = griddata(points, time, (grid_x, grid_y), fill_value=0, method='cubic')
# delta az, delta alt computation
#az = ei_az[LST_event_index[i]]
#alt = ei_alt[LST_event_index[i]]
#src = AltAz(alt=alt * u.rad, az=az * u.rad)
#source_direction = src.transform_to(NominalFrame(origin=point))
# appending to arrays
lst_image_charge_interp.append(interp_img)
lst_image_peak_times_interp.append(interp_time)
#delta_az.append(source_direction.delta_az.deg)
#delta_alt.append(source_direction.delta_alt.deg)
intensities.append(intensity)
intensities_width_2.append(leakage2_intensity)
acc_idxs += [i] # also this one can be removed when no cuts here
else:
count += 1
# #rejected.write("{}.\tImage #{} rejected (no islands)!\n".format(count, i))
# print("No islands: image #{} rejected! (cumulative: {})\n".format(i, count), end='\r')
# lst_image_charge_interp = np.array(lst_image_charge_interp)
print("Number of rejected images: {} ({:.1f}%)".format(count, count / len(intensities) *100))
data_p.close()
#rejected.close()
fpath = Path(f)
newname = fpath.name[:-3] + '_interp.h5'
filename = str(PurePath(outpath, newname))
print("Writing file: " + filename + "\n")
data_file = h5py.File(filename, 'w')
data_file.create_dataset('Event_Info/ei_alt', data=np.array(ei_alt))
data_file.create_dataset('Event_Info/ei_az', data=np.array(ei_az))
data_file.create_dataset('Event_Info/ei_mc_energy', data=np.array(ei_mc_energy))
data_file.create_dataset('LST/LST_event_index', data=np.array(LST_event_index)[acc_idxs])
data_file.create_dataset('LST/LST_image_charge', data=np.array(LST_image_charge)[acc_idxs])
data_file.create_dataset('LST/LST_image_peak_times', data=np.array(LST_image_peak_times)[acc_idxs])
# data_file.create_dataset('LST/LST_event_index', data=np.array(LST_event_index))
# data_file.create_dataset('LST/LST_image_charge', data=np.array(LST_image_charge))
# data_file.create_dataset('LST/LST_image_peak_times', data=np.array(LST_image_peak_times))
data_file.create_dataset('LST/LST_image_charge_interp', data=np.array(lst_image_charge_interp))
data_file.create_dataset('LST/LST_image_peak_times_interp', data=np.array(lst_image_peak_times_interp))
#data_file.create_dataset('LST/delta_alt', data=np.array(delta_alt))
#data_file.create_dataset('LST/delta_az', data=np.array(delta_az))
data_file.create_dataset('LST/intensities', data=np.array(intensities))
data_file.create_dataset('LST/intensities_width_2', data=np.array(intensities_width_2))
data_file.close()
# in the interpolated files there will be all the original events
# but for the LST only the ones actually see at least from one LST (as in the original files)
# and that are above thresholds cuts
if ro == '1':
remove(f)
print('Removing original file')
except HDF5ExtError:
print('\nUnable to open file' + f)
if rc == '1':
print('Removing it...')
remove(f)
except NoSuchNodeError:
print('This file has a problem with the data structure: ' + f)
if rn == '1':
print('Removing it...')
remove(f)
def process_event(event, config):
'''
Processes
'''
reco_algorithm = config.reco_algorithm
features = {}
params = {}
pointing_azimuth = {}
pointing_altitude = {}
tel_x = {}
tel_y = {}
tel_focal_lengths = {}
cleaning_method = config.cleaning_method
valid_cleaning_methods = ['tailcuts_clean', 'fact_image_cleaning']
if cleaning_method not in valid_cleaning_methods:
print('Cleaning Method not implemented')
print('Please use one of ', valid_cleaning_methods)
return None
for telescope_id, dl1 in event.dl1.tel.items():
camera = event.inst.subarray.tels[telescope_id].camera
if camera.cam_id not in config.allowed_cameras:
continue
telescope_type_name = event.inst.subarray.tels[
telescope_id].optics.tel_type
if cleaning_method == 'tailcuts_clean':
boundary_thresh, picture_thresh, min_number_picture_neighbors = config.cleaning_level[
camera.cam_id]
mask = tailcuts_clean(camera, dl1.image[0],
*config.cleaning_level[camera.cam_id])
elif cleaning_method == 'fact_image_cleaning':
mask = fact_image_cleaning(
camera, dl1.image[0],
*config.cleaning_level_fact[camera.cam_id])
try:
cleaned = dl1.image[0].copy()
cleaned[~mask] = 0
hillas_container = hillas_parameters(
camera,
cleaned,
)
params[telescope_id] = hillas_container
except HillasParameterizationError:
continue
# probably wise to add try...except blocks here as well
# Add more Features here (look what ctapipe can do, timing?)
num_islands, island_labels = number_of_islands(camera, mask)
island_dict = {
'num_islands': num_islands,
'island_labels': island_labels
}
leakage_container = leakage(camera, dl1.image[0], mask)
timing_container = timing_parameters(camera, dl1.image[0],
dl1.peakpos[0], hillas_container)
pointing_azimuth[
telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad
pointing_altitude[
telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad
tel_x[telescope_id] = event.inst.subarray.positions[telescope_id][0]
tel_y[telescope_id] = event.inst.subarray.positions[telescope_id][1]
telescope_description = event.inst.subarray.tel[telescope_id]
tel_focal_lengths[
telescope_id] = telescope_description.optics.equivalent_focal_length
d = {
'array_event_id': event.dl0.event_id,
'telescope_id': int(telescope_id),
'camera_name': camera.cam_id,
'camera_id': config.names_to_id[camera.cam_id],
'run_id': event.r0.obs_id,
'telescope_type_name': telescope_type_name,
'telescope_type_id': config.types_to_id[telescope_type_name],
'pointing_azimuth': event.mc.tel[telescope_id].azimuth_raw,
'pointing_altitude': event.mc.tel[telescope_id].altitude_raw,
'mirror_area': telescope_description.optics.mirror_area,
'focal_length':
telescope_description.optics.equivalent_focal_length,
}
d.update(hillas_container.as_dict())
d.update(leakage_container.as_dict())
d.update(island_dict)
d.update(timing_container.as_dict())
features[telescope_id] = ({k: strip_unit(v) for k, v in d.items()})
if reco_algorithm == 'intersection':
reco = HillasIntersection()
array_direction = SkyCoord(alt=event.mcheader.run_array_direction[1],
az=event.mcheader.run_array_direction[0],
frame='altaz')
reconstruction = reco.predict(params, tel_x, tel_y, tel_focal_lengths,
array_direction)
elif reco_algorithm == 'planes':
reco = HillasReconstructor()
reconstruction = reco.predict(params, event.inst, pointing_altitude,
pointing_azimuth)
for telescope_id in event.dl1.tel.keys():
if telescope_id not in params:
continue
camera = event.inst.subarray.tels[telescope_id].camera
if camera.cam_id not in config.allowed_cameras:
continue
pos = event.inst.subarray.positions[telescope_id]
x, y = pos[0], pos[1]
core_x = reconstruction.core_x
core_y = reconstruction.core_y
d = np.sqrt((core_x - x)**2 + (core_y - y)**2)
features[telescope_id]['distance_to_core'] = d.value
return pd.DataFrame(list(features.values())), reconstruction, params
# cleaning
boundary, picture, min_neighbors = cleaning_level[geom.camera_name]
clean = tailcuts_clean(geom,
image,
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors)
# ignore images with less than 5 pixels after cleaning
if clean.sum() < 5:
continue
# image parameters
hillas_c = hillas_parameters(geom[clean], image[clean])
leakage_c = leakage(geom, image, clean)
n_islands, island_ids = number_of_islands(geom, clean)
timing_c = timing_parameters(
geom[clean],
image[clean],
peakpos[clean],
hillas_c,
)
# store parameters for stereo reconstruction
hillas_containers[telescope_id] = hillas_c
# store timegradients for plotting
# ASTRI has no timing in PROD3b, so we use skewness instead
if geom.camera_name != 'ASTRICam':
boundary, picture, min_neighbors = cleaning_level[camera.cam_id]
clean = tailcuts_clean(
camera,
image,
boundary_thresh=boundary,
picture_thresh=picture,
min_number_picture_neighbors=min_neighbors
)
# ignore images with less than 5 pixels after cleaning
if clean.sum() < 5:
continue
# image parameters
hillas_c = hillas_parameters(camera[clean], image[clean])
leakage_c = leakage(camera, image, clean)
n_islands, island_ids = number_of_islands(camera, clean)
timing_c = timing_parameters(
camera[clean], image[clean], peakpos[clean], hillas_c
)
# store parameters for stereo reconstruction
hillas_containers[telescope_id] = hillas_c
pointing_azimuth[telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad
pointing_altitude[telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad
# store timegradients for plotting
# ASTRI has no timing in PROD3b, so we use skewness instead
if camera.cam_id != 'ASTRICam':
time_gradients[telescope_id] = timing_c.slope.value
def set_leakage(self, geom, image, clean):
leakage_c = leakage(geom, image, clean)
self.leakage1_intensity = leakage_c.leakage1_intensity
self.leakage2_intensity = leakage_c.leakage2_intensity
self.leakage1_pixel = leakage_c.leakage1_pixel
self.leakage2_pixel = leakage_c.leakage2_pixel
def calculate_image_features(telescope_id, event, dl1, config):
''' Performs cleaning and adds the following image parameters:
- hillas
- leakage
- concentration
- timing
- number of islands
Make sure to adapt cleaning levels to the used algorithm (-> config)
- tailcuts:
picture_thresh, picture_thresh, min_number_picture_neighbors
- fact_image_cleaning:
picture_threshold, boundary_threshold, min_number_neighbors, time_limit
Returns:
--------
TelescopeParameterContainer
'''
array_event_id = event.dl0.event_id
run_id = event.r0.obs_id
telescope = event.inst.subarray.tels[telescope_id]
image = dl1.image
# might want to make the parameter names more consistent between methods
if config.cleaning_method == 'tailcuts_clean':
boundary_thresh, picture_thresh, min_number_picture_neighbors = config.cleaning_level[
telescope.camera.cam_id
]
mask = tailcuts_clean(
telescope.camera,
image,
boundary_thresh=boundary_thresh,
picture_thresh=picture_thresh,
min_number_picture_neighbors=min_number_picture_neighbors,
)
elif config.cleaning_method == 'fact_image_cleaning':
boundary_threshold, picture_threshold, time_limit, min_number_neighbors = config.cleaning_level[
telescope.camera.cam_id
]
mask = fact_image_cleaning(
telescope.camera,
image,
dl1.pulse_time,
boundary_threshhold=boundary_threshold,
picture_threshold=picture_threshold,
min_number_neighbors=min_number_neighbors,
time_limit=time_limit,
)
cleaned = image.copy()
cleaned[~mask] = 0
logging.debug(f'calculating hillas for event {array_event_id, telescope_id}')
hillas_container = hillas_parameters(telescope.camera, cleaned)
hillas_container.prefix = ''
logging.debug(f'calculating leakage for event {array_event_id, telescope_id}')
leakage_container = leakage(telescope.camera, image, mask)
leakage_container.prefix = ''
logging.debug(f'calculating concentration for event {array_event_id, telescope_id}')
concentration_container = concentration(telescope.camera, image, hillas_container)
concentration_container.prefix = ''
logging.debug(f'getting timing information for event {array_event_id, telescope_id}')
timing_container = timing_parameters(telescope.camera, image, dl1.pulse_time, hillas_container)
timing_container.prefix = ''
# membership missing for now as it causes problems with the hdf5tablewriter
# right now i dont need this anyway
logging.debug(f'calculating num_islands for event {array_event_id, telescope_id}')
num_islands, membership = number_of_islands(telescope.camera, mask)
island_container = IslandContainer(num_islands=num_islands)
island_container.prefix = ''
num_pixel_in_shower = mask.sum()
logging.debug(f'getting pointing container for event {array_event_id, telescope_id}')
# ctapipe requires this to be rad
pointing_container = TelescopePointingContainer(
azimuth=event.mc.tel[telescope_id].azimuth_raw * u.rad,
altitude=event.mc.tel[telescope_id].altitude_raw * u.rad,
prefix='pointing',
)
return TelescopeParameterContainer(
telescope_id=telescope_id,
run_id=run_id,
array_event_id=array_event_id,
leakage=leakage_container,
hillas=hillas_container,
concentration=concentration_container,
pointing=pointing_container,
timing=timing_container,
islands=island_container,
telescope_type_id=config.types_to_id[telescope.type],
camera_type_id=config.names_to_id[telescope.camera.cam_id],
focal_length=telescope.optics.equivalent_focal_length,
mirror_area=telescope.optics.mirror_area,
num_pixel_in_shower=num_pixel_in_shower,
)
def process_telescope(tel, dl1, stereo):
geom = tel.camera.geometry
camera_name = tel.camera.camera_name
image = dl1.image
peak_time = dl1.peak_time
# Cleaning using CHEC method
clean = obtain_cleaning_mask(geom, image, peak_time, camera_name)
# Skipping inadequate events
if clean.sum() == 0:
return None, None, None
if stereo and clean.sum() < 5:
return None, None, None
# Get hillas parameters
hillas_c = hillas_parameters(geom[clean], image[clean])
if hillas_c.width == 0 or np.isnan(hillas_c.width.value):
return None, None, None
# Get leakage and islands
leakage_c = leakage(geom, image, clean)
n_islands, island_ids = number_of_islands(geom, clean)
# Get time gradient
tgrad = np.nan
try:
timing_c = timing_parameters(geom, image, peak_time, hillas_c, clean)
tgrad = timing_c.slope.value
except BaseException:
print("Timing parameters didn't work. clean.sum() = " +
str(clean.sum()), "\n")
# Grab info for telescope_events
tel_data_dict = {
'nislands': n_islands,
'telescope_type': tel.type,
'camera_type': tel.camera.camera_name,
'focal_length': tel.optics.equivalent_focal_length.value,
'n_survived_pixels': clean.sum(),
'tgradient': tgrad,
'x': hillas_c.x.value,
'y': hillas_c.y.value,
'r': hillas_c.r.value,
'phi': hillas_c.phi.value,
'intensity': hillas_c.intensity,
'length': hillas_c.length.value,
'width': hillas_c.width.value,
'psi': hillas_c.psi.value,
'skewness': hillas_c.skewness,
'kurtosis': hillas_c.kurtosis,
'pixels_width_1': leakage_c.pixels_width_1,
'pixels_width_2': leakage_c.pixels_width_2,
'intensity_width_1': leakage_c.intensity_width_1,
'intensity_width_2': leakage_c.intensity_width_2}
return tel.type, tel_data_dict, hillas_c
def prepare_event(self,
source,
return_stub=True,
save_images=False,
debug=False):
"""
Calibrate, clean and reconstruct the direction of an event.
Parameters
----------
source : ctapipe.io.EventSource
A container of selected showers from a simtel file.
geom_cam_tel: dict
Dictionary of of MyCameraGeometry objects for each camera in the file
return_stub : bool
If True, yield also images from events that won't be reconstructed.
This feature is not currently available.
save_images : bool
If True, save photoelectron images from reconstructed events.
debug : bool
If True, print some debugging information (to be expanded).
Yields
------
PreparedEvent: dict
Dictionary containing event-image information to be written.
"""
# =============================================================
# TRANSFORMED CAMERA GEOMETRIES
# =============================================================
# These are the camera geometries were the Hillas parametrization will
# be performed.
# They are transformed to TelescopeFrame using the effective focal
# lengths
# These geometries could be used also to performe the image cleaning,
# but for the moment we still do that in the CameraFrame
geom_cam_tel = {}
for camera in source.subarray.camera_types:
# Original geometry of each camera
geom = camera.geometry
# Same but with focal length as an attribute
# This is planned to disappear and be imported by ctapipe
focal_length = effective_focal_lengths(camera.camera_name)
geom_cam_tel[camera.camera_name] = MyCameraGeometry(
camera_name=camera.camera_name,
pix_type=geom.pix_type,
pix_id=geom.pix_id,
pix_x=geom.pix_x,
pix_y=geom.pix_y,
pix_area=geom.pix_area,
cam_rotation=geom.cam_rotation,
pix_rotation=geom.pix_rotation,
frame=CameraFrame(focal_length=focal_length),
).transform_to(TelescopeFrame())
# =============================================================
ievt = 0
for event in source:
# Display event counts
if debug:
print(
bcolors.BOLD +
f"EVENT #{event.count}, EVENT_ID #{event.index.event_id}" +
bcolors.ENDC)
print(bcolors.BOLD +
f"has triggered telescopes {event.r0.tels_with_data}" +
bcolors.ENDC)
ievt += 1
# if (ievt < 10) or (ievt % 10 == 0):
# print(ievt)
self.event_cutflow.count("noCuts")
if self.event_cutflow.cut("min2Tels trig",
len(event.dl0.tels_with_data)):
if return_stub:
print(bcolors.WARNING +
f"WARNING : < {self.min_tel} triggered telescopes!" +
bcolors.ENDC)
pass # we register this, but we proceed to analyze it
# yield stub(event)
# else:
# continue
# =============================================================
# CALIBRATION
# =============================================================
if debug:
print(bcolors.OKBLUE + "Extracting all calibrated images..." +
bcolors.ENDC)
self.calib(event) # Calibrate the event
# =============================================================
# BEGINNING OF LOOP OVER TELESCOPES
# =============================================================
dl1_phe_image = {}
dl1_phe_image_mask_reco = {}
dl1_phe_image_mask_clusters = {}
mc_phe_image = {}
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for direction reconstruction
hillas_dict = {} # for discrimination
leakage_dict = {}
n_tels = {
"Triggered": len(event.dl0.tels_with_data),
"LST_LST_LSTCam": 0,
"MST_MST_NectarCam": 0,
"MST_MST_FlashCam": 0,
"MST_SCT_SCTCam": 0,
"SST_1M_DigiCam": 0,
"SST_ASTRI_ASTRICam": 0,
"SST_GCT_CHEC": 0,
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
good_for_reco = {} # 1 = success, 0 = fail
# Compute impact parameter in tilt system
run_array_direction = event.mcheader.run_array_direction
az, alt = run_array_direction[0], run_array_direction[1]
ground_frame = GroundFrame()
for tel_id in event.dl0.tels_with_data:
point_azimuth_dict[
tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
point_altitude_dict[
tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
if debug:
print(bcolors.OKBLUE +
f"Working on telescope #{tel_id}..." + bcolors.ENDC)
self.image_cutflow.count("noCuts")
camera = source.subarray.tel[tel_id].camera.geometry
# count the current telescope according to its type
tel_type = str(source.subarray.tel[tel_id])
n_tels[tel_type] += 1
# use ctapipe's functionality to get the calibrated image
pmt_signal = event.dl1.tel[tel_id].image
# If required...
if save_images is True:
# Save the simulated and reconstructed image of the event
dl1_phe_image[tel_id] = pmt_signal
mc_phe_image[tel_id] = event.mc.tel[tel_id].true_image
# We now ASSUME that the event will be good
good_for_reco[tel_id] = 1
# later we change to 0 if any condition is NOT satisfied
if self.cleaner_reco.mode == "tail": # tail uses only ctapipe
# Cleaning used for direction reconstruction
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
# calculate the leakage (before filtering)
leakages = {} # this is needed by both cleanings
# The check on SIZE shouldn't be here, but for the moment
# I prefer to sacrifice elegancy...
if np.sum(image_biggest[mask_reco]) != 0.0:
leakage_biggest = leakage(camera, image_biggest,
mask_reco)
leakages["leak1_reco"] = leakage_biggest[
"intensity_width_1"]
leakages["leak2_reco"] = leakage_biggest[
"intensity_width_2"]
else:
leakages["leak1_reco"] = 0.0
leakages["leak2_reco"] = 0.0
# find all islands using this cleaning
num_islands, labels = number_of_islands(camera, mask_reco)
if num_islands == 1: # if only ONE islands is left ...
# ...use directly the old mask and reduce dimensions
# to make Hillas parametrization faster
camera_biggest = camera[mask_reco]
image_biggest = image_biggest[mask_reco]
if save_images is True:
dl1_phe_image_mask_reco[tel_id] = mask_reco
elif num_islands > 1: # if more islands survived..
# ...find the biggest one
mask_biggest = largest_island(labels)
# and also reduce dimensions
camera_biggest = camera[mask_biggest]
image_biggest = image_biggest[mask_biggest]
if save_images is True:
dl1_phe_image_mask_reco[tel_id] = mask_biggest
else: # if no islands survived use old camera and image
camera_biggest = camera
dl1_phe_image_mask_reco[tel_id] = mask_reco
# Cleaning used for score/energy estimation
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
dl1_phe_image_mask_clusters[tel_id] = mask_extended
# calculate the leakage (before filtering)
# this part is not well coded, but for the moment it works
if np.sum(image_extended[mask_extended]) != 0.0:
leakage_extended = leakage(camera, image_extended,
mask_extended)
leakages["leak1"] = leakage_extended[
"intensity_width_1"]
leakages["leak2"] = leakage_extended[
"intensity_width_2"]
else:
leakages["leak1"] = 0.0
leakages["leak2"] = 0.0
# find all islands with this cleaning
# we will also register how many have been found
n_cluster_dict[tel_id], labels = number_of_islands(
camera, mask_extended)
# NOTE: the next check shouldn't be necessary if we keep
# all the isolated pixel clusters, but for now the
# two cleanings are set the same in analysis.yml because
# the performance of the extended one has never been really
# studied in model estimation.
# if some islands survived
if n_cluster_dict[tel_id] > 0:
# keep all of them and reduce dimensions
camera_extended = camera[mask_extended]
image_extended = image_extended[mask_extended]
else: # otherwise continue with the old camera and image
camera_extended = camera
# could this go into `hillas_parameters` ...?
# this is basically the charge of ALL islands
# not calculated later by the Hillas parametrization!
max_signals[tel_id] = np.max(image_extended)
else: # for wavelets we stick to old pywi-cta code
try: # "try except FileNotFoundError" not clear to me, but for now it stays...
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.camera_name)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.camera_name)
# Image for score/energy estimation (with clusters)
(
image_extended,
mask_extended,
) = self.cleaner_extended.clean_image(
pmt_signal, camera)
# This last part was outside the pywi-cta block
# before, but is indeed part of it because it uses
# pywi-cta functions in the "extended" case
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.camera_name)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
except FileNotFoundError as e: # JLK, WHAT?
print(e)
continue
# =============================================================
# PRELIMINARY IMAGE SELECTION
# =============================================================
cleaned_image_is_good = True # we assume this
# Apply some selection
if self.image_cutflow.cut("min pixel", image_biggest):
if debug:
print(bcolors.WARNING +
"WARNING : not enough pixels!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
cleaned_image_is_good = False
if self.image_cutflow.cut("min charge", np.sum(image_biggest)):
if debug:
print(bcolors.WARNING +
"WARNING : not enough charge!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
cleaned_image_is_good = False
if debug and (not cleaned_image_is_good): # BAD image quality
print(bcolors.WARNING +
"WARNING : The cleaned image didn't pass" +
" preliminary cuts.\n" +
"An attempt to parametrize it will be made," +
" but the image will NOT be used for" +
" direction reconstruction." + bcolors.ENDC)
# =============================================================
# IMAGE PARAMETRIZATION
# =============================================================
with np.errstate(invalid="raise", divide="raise"):
try:
# Filter the cameras in TelescopeFrame with the same
# cleaning masks
camera_biggest_tel = geom_cam_tel[camera.camera_name][
camera_biggest.pix_id]
camera_extended_tel = geom_cam_tel[camera.camera_name][
camera_extended.pix_id]
# Parametrize the image in the TelescopeFrame
moments_reco = hillas_parameters(
camera_biggest_tel,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera_extended_tel, image_extended
) # for discrimination and energy reconstruction
if debug:
print(
"Image parameters from main cluster cleaning:")
print(moments_reco)
print(
"Image parameters from all-clusters cleaning:")
print(moments)
# ===================================================
# PARAMETRIZED IMAGE SELECTION
# ===================================================
# if width and/or length are zero (e.g. when there is
# only only one pixel or when all pixel are exactly
# in one row), the parametrisation
# won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_reco):
if debug:
print(bcolors.WARNING +
"WARNING : poor moments!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
if self.image_cutflow.cut("close to the edge",
moments_reco,
camera.camera_name):
if debug:
print(
bcolors.WARNING +
"WARNING : out of containment radius!\n" +
f"Camera radius = {self.camera_radius[camera.camera_name]}\n"
+ f"COG radius = {moments_reco.r}" +
bcolors.ENDC)
good_for_reco[tel_id] = 0
if self.image_cutflow.cut("bad ellipticity",
moments_reco):
if debug:
print(bcolors.WARNING +
"WARNING : bad ellipticity" +
bcolors.ENDC)
good_for_reco[tel_id] = 0
if debug and good_for_reco[tel_id] == 1:
print(
bcolors.OKGREEN +
"Image survived and correctly parametrized."
# + "\nIt will be used for direction reconstruction!"
+ bcolors.ENDC)
elif debug and good_for_reco[tel_id] == 0:
print(
bcolors.WARNING + "Image not survived or " +
"not good enough for parametrization."
# + "\nIt will be NOT used for direction reconstruction, "
# + "BUT it's information will be recorded."
+ bcolors.ENDC)
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(
np.where(image_extended > 0)[0])
leakage_dict[tel_id] = leakages
except (
FloatingPointError,
HillasParameterizationError,
ValueError,
) as e:
if debug:
print(bcolors.FAIL + "Parametrization error: " +
f"{e}\n" + "Dummy parameters recorded." +
bcolors.ENDC)
good_for_reco[tel_id] = 0
hillas_dict[
tel_id] = HillasParametersTelescopeFrameContainer(
)
hillas_dict_reco[
tel_id] = HillasParametersTelescopeFrameContainer(
)
n_pixel_dict[tel_id] = len(
np.where(image_extended > 0)[0])
leakage_dict[tel_id] = leakages
# END OF THE CYCLE OVER THE TELESCOPES
# =============================================================
# DIRECTION RECONSTRUCTION
# =============================================================
# n_tels["reco"] = len(hillas_dict_reco)
# n_tels["discri"] = len(hillas_dict)
# convert dictionary in numpy array to get a "mask"
images_status = np.asarray(list(good_for_reco.values()))
# record how many images will used for reconstruction
n_tels["GOOD images"] = len(
np.extract(images_status == 1, images_status))
n_tels["BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"]
if self.event_cutflow.cut("min2Tels reco", n_tels["GOOD images"]):
if debug:
print(
bcolors.FAIL +
f"WARNING: < {self.min_ntel} ({n_tels['GOOD images']}) images remaining!"
+
"\nWARNING : direction reconstruction is not possible!"
+ bcolors.ENDC)
# create a dummy container for direction reconstruction
reco_result = ReconstructedShowerContainer()
if return_stub: # if saving all events (default)
if debug:
print(bcolors.OKBLUE + "Recording event..." +
bcolors.ENDC)
print(
bcolors.WARNING +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event,
mc_phe_image,
dl1_phe_image,
dl1_phe_image_mask_reco,
dl1_phe_image_mask_clusters,
good_for_reco,
hillas_dict,
hillas_dict_reco,
n_tels,
leakage_dict,
)
continue
else:
continue
if debug:
print(bcolors.OKBLUE + "Starting direction reconstruction..." +
bcolors.ENDC)
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# use only the successfully parametrized images
# to reconstruct the direction of this event
successfull_hillas = np.where(images_status == 1)[0]
all_images = np.asarray(list(good_for_reco.keys()))
good_images = set(all_images[successfull_hillas])
good_hillas_dict_reco = {
k: v
for k, v in hillas_dict_reco.items()
if k in good_images
}
if debug:
print(bcolors.PURPLE +
f"{len(good_hillas_dict_reco)} images will be " +
"used to recover the shower's direction..." +
bcolors.ENDC)
# Reconstruction results
reco_result = self.shower_reco.predict(
good_hillas_dict_reco,
source.subarray,
SkyCoord(alt=alt, az=az, frame="altaz"),
None, # use the array direction
)
# Impact parameter for energy estimation (/ tel)
subarray = source.subarray
for tel_id in hillas_dict.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# Should be better handled (tilted frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_ground.x - tel_ground.x)**2 +
(core_ground.y - tel_ground.y)**2)
except (Exception, TooFewTelescopesException,
InvalidWidthException) as e:
if debug:
print("exception in reconstruction:", e)
raise
if return_stub:
if debug:
print(
bcolors.FAIL +
"Shower could NOT be correctly reconstructed! " +
"Recording event..." +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event,
mc_phe_image,
dl1_phe_image,
dl1_phe_image_mask_reco,
dl1_phe_image_mask_clusters,
good_for_reco,
hillas_dict,
hillas_dict_reco,
n_tels,
leakage_dict,
)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if debug:
print(bcolors.WARNING + "WARNING: undefined direction!" +
bcolors.ENDC)
if return_stub:
if debug:
print(
bcolors.FAIL +
"Shower could NOT be correctly reconstructed! " +
"Recording event..." +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event,
mc_phe_image,
dl1_phe_image,
dl1_phe_image_mask_reco,
dl1_phe_image_mask_clusters,
good_for_reco,
hillas_dict,
hillas_dict_reco,
n_tels,
leakage_dict,
)
else:
continue
if debug:
print(bcolors.BOLDGREEN + "Shower correctly reconstructed! " +
"Recording event..." + bcolors.ENDC)
yield PreparedEvent(
event=event,
dl1_phe_image=dl1_phe_image,
dl1_phe_image_mask_reco=dl1_phe_image_mask_reco,
dl1_phe_image_mask_clusters=dl1_phe_image_mask_clusters,
mc_phe_image=mc_phe_image,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
leakage_dict=leakage_dict,
n_tels=n_tels,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
good_event=True,
good_for_reco=good_for_reco,
)
