def get_observation_parameters(charge: np.array, peak: np.array, cam_name: str, cutflow: CutFlow, boundary_threshold: float = None, picture_threshold: float = None, min_neighbours: float = None, plot: bool = False, cut: bool = True): """ :param charge: Charge image :param peak: Peak time image :param cam_name: Camera name. e.g. FlashCam, ASTRICam, etc. :param cutflow: Cutflow for selection :param boundary_threshold: (Optional) Cleaning parameter: boundary threshold :param picture_threshold: (Optional) Cleaning parameter: picture threshold :param min_neighbours: (Optional) Cleaning parameter: minimum neighbours :param plot: If True, for each observation a plot will be shown (Default: False) :param cut: If true, tight else loose :return: hillas containers, leakage container, number of islands, island IDs, timing container, timing gradient """ charge_biggest, mask = clean_charge(charge, cam_name, boundary_threshold, picture_threshold, min_neighbours) camera = get_camera(cam_name) geometry = camera.geometry charge_biggest, camera_biggest, n_islands = mask_from_biggest_island( charge, geometry, mask) if cut: if cutflow.cut(CFO_MIN_PIXEL, charge_biggest): return if cutflow.cut(CFO_MIN_CHARGE, np.sum(charge_biggest)): return if cutflow.cut(CFO_NEGATIVE_CHARGE, charge_biggest): return leakage_c = leakage(geometry, charge, mask) if plot: _, (ax1, ax2) = plt.subplots(nrows=1, ncols=2) CameraDisplay(geometry, charge, ax=ax1).add_colorbar() CameraDisplay(camera_biggest, charge_biggest, ax=ax2).add_colorbar() plt.show() moments = hillas_parameters(camera_biggest, charge_biggest) if cut: if cutflow.cut(CFO_CLOSE_EDGE, moments, camera.camera_name): return if cutflow.cut(CFO_BAD_ELLIP, moments): return if cutflow.cut(CFO_POOR_MOMENTS, moments): return timing_c = timing_parameters(geometry, charge, peak, moments, mask) time_gradient = timing_c.slope.value if geometry.camera_name != 'ASTRICam' else moments.skewness return moments, leakage_c, timing_c, time_gradient, n_islands
def leak(img): cleanmask = cleaning.tailcuts_clean(geom, img, **opts) mask = False if any(cleanmask): leakage_values = leakage(geom, img, cleanmask) if hasattr(leakage_values, 'leakage{}_intensity'.format(leakage_number)): mask = leakage_values['leakage{}_intensity'.format( leakage_number)] <= leakage_value elif hasattr(leakage_values, 'intensity_width_{}'.format(leakage_number)): mask = leakage_values['intensity_width_{}'.format( leakage_number)] <= leakage_value return mask
def set_leakage(self, geom, image, clean): leakage_c = leakage(geom, image, clean) self.leakage_intensity_width_1 = leakage_c.intensity_width_1 self.leakage_intensity_width_2 = leakage_c.intensity_width_2 self.leakage_pixels_width_1 = leakage_c.pixels_width_1 self.leakage_pixels_width_2 = leakage_c.pixels_width_2
def set_leakage(self, geom, image, clean): leakage_c = leakage(geom, image, clean) self.leakage = leakage_c.leakage2_intensity
def func(paths, outpath, ro, rc, rn): # iterate on each proton file & concatenate charge arrays for n, f in enumerate(paths): # get the data from the file try: print("Opening file #{}...".format(n)) data_p = tables.open_file(f) _, LST_event_index, LST_image_charge, LST_image_peak_times = get_LST_data(data_p) _, ei_alt, ei_az, ei_mc_energy = get_event_data(data_p) # Excluding the 0th element - it's the empty one!! LST_event_index = LST_event_index[1:] LST_image_charge = LST_image_charge[1:] LST_image_peak_times = LST_image_peak_times[1:] # get camera geometry & camera pixels coordinates camera = CameraGeometry.from_name("LSTCam") points = np.array([np.array(camera.pix_x / u.m), np.array(camera.pix_y / u.m)]).T # original choice by Nicola: 100x100 points in 2.5m x 2.5m #grid_x, grid_y = np.mgrid[-1.25:1.25:100j, -1.25:1.25:100j] # I choose instead 96x88 px in 2.40m x 2.20m: same spatial separation, less points grid_x, grid_y = np.mgrid[-1.20:1.20:96j, -1.10:1.10:88j] ''' was probably useless and wrong even with the old simulations if alt_array > 90: alt_array = 90 ''' # LST coordinates (pointing position) #point = AltAz(alt=alt_array * u.deg, az=az_array * u.deg) lst_image_charge_interp = [] lst_image_peak_times_interp = [] # alt az of the array [deg] #az_array = 180. # before was ai_run_array_direction[0][0], now it is hardcoded as it's not present in new files #alt_array = 70. # ai_run_array_direction[0][1] #delta_az = [] #delta_alt = [] intensities = [] intensities_width_2 = [] acc_idxs = [] # accepted indexes # in principle it can be removed when cuts (line AAA) are not used here cleaning_level = {'LSTCam': (3.5, 7.5, 2)} count = 0 #rejected = open(f[:-3] + "_rejected.txt", "w") for i in trange(0, len(LST_image_charge), desc="Images interpolation"): image = LST_image_charge[i] time = LST_image_peak_times[i] boundary, picture, min_neighbors = cleaning_level['LSTCam'] clean = tailcuts_clean( camera, image, boundary_thresh=boundary, picture_thresh=picture, min_number_picture_neighbors=min_neighbors ) if len(np.where(clean > 0)[0]) != 0: hillas = hillas_parameters(camera[clean], image[clean]) intensity = hillas['intensity'] l = leakage(camera, image, clean) # print(l) leakage2_intensity = l['intensity_width_2'] # if intensity > 50 and leakage2_intensity < 0.2: # ------>AAA --- CUT DIRECTLY DURING INTERP # cubic interpolation interp_img = griddata(points, image, (grid_x, grid_y), fill_value=0, method='cubic') interp_time = griddata(points, time, (grid_x, grid_y), fill_value=0, method='cubic') # delta az, delta alt computation #az = ei_az[LST_event_index[i]] #alt = ei_alt[LST_event_index[i]] #src = AltAz(alt=alt * u.rad, az=az * u.rad) #source_direction = src.transform_to(NominalFrame(origin=point)) # appending to arrays lst_image_charge_interp.append(interp_img) lst_image_peak_times_interp.append(interp_time) #delta_az.append(source_direction.delta_az.deg) #delta_alt.append(source_direction.delta_alt.deg) intensities.append(intensity) intensities_width_2.append(leakage2_intensity) acc_idxs += [i] # also this one can be removed when no cuts here else: count += 1 # #rejected.write("{}.\tImage #{} rejected (no islands)!\n".format(count, i)) # print("No islands: image #{} rejected! (cumulative: {})\n".format(i, count), end='\r') # lst_image_charge_interp = np.array(lst_image_charge_interp) print("Number of rejected images: {} ({:.1f}%)".format(count, count / len(intensities) *100)) data_p.close() #rejected.close() fpath = Path(f) newname = fpath.name[:-3] + '_interp.h5' filename = str(PurePath(outpath, newname)) print("Writing file: " + filename + "\n") data_file = h5py.File(filename, 'w') data_file.create_dataset('Event_Info/ei_alt', data=np.array(ei_alt)) data_file.create_dataset('Event_Info/ei_az', data=np.array(ei_az)) data_file.create_dataset('Event_Info/ei_mc_energy', data=np.array(ei_mc_energy)) data_file.create_dataset('LST/LST_event_index', data=np.array(LST_event_index)[acc_idxs]) data_file.create_dataset('LST/LST_image_charge', data=np.array(LST_image_charge)[acc_idxs]) data_file.create_dataset('LST/LST_image_peak_times', data=np.array(LST_image_peak_times)[acc_idxs]) # data_file.create_dataset('LST/LST_event_index', data=np.array(LST_event_index)) # data_file.create_dataset('LST/LST_image_charge', data=np.array(LST_image_charge)) # data_file.create_dataset('LST/LST_image_peak_times', data=np.array(LST_image_peak_times)) data_file.create_dataset('LST/LST_image_charge_interp', data=np.array(lst_image_charge_interp)) data_file.create_dataset('LST/LST_image_peak_times_interp', data=np.array(lst_image_peak_times_interp)) #data_file.create_dataset('LST/delta_alt', data=np.array(delta_alt)) #data_file.create_dataset('LST/delta_az', data=np.array(delta_az)) data_file.create_dataset('LST/intensities', data=np.array(intensities)) data_file.create_dataset('LST/intensities_width_2', data=np.array(intensities_width_2)) data_file.close() # in the interpolated files there will be all the original events # but for the LST only the ones actually see at least from one LST (as in the original files) # and that are above thresholds cuts if ro == '1': remove(f) print('Removing original file') except HDF5ExtError: print('\nUnable to open file' + f) if rc == '1': print('Removing it...') remove(f) except NoSuchNodeError: print('This file has a problem with the data structure: ' + f) if rn == '1': print('Removing it...') remove(f)
def process_event(event, config): ''' Processes ''' reco_algorithm = config.reco_algorithm features = {} params = {} pointing_azimuth = {} pointing_altitude = {} tel_x = {} tel_y = {} tel_focal_lengths = {} cleaning_method = config.cleaning_method valid_cleaning_methods = ['tailcuts_clean', 'fact_image_cleaning'] if cleaning_method not in valid_cleaning_methods: print('Cleaning Method not implemented') print('Please use one of ', valid_cleaning_methods) return None for telescope_id, dl1 in event.dl1.tel.items(): camera = event.inst.subarray.tels[telescope_id].camera if camera.cam_id not in config.allowed_cameras: continue telescope_type_name = event.inst.subarray.tels[ telescope_id].optics.tel_type if cleaning_method == 'tailcuts_clean': boundary_thresh, picture_thresh, min_number_picture_neighbors = config.cleaning_level[ camera.cam_id] mask = tailcuts_clean(camera, dl1.image[0], *config.cleaning_level[camera.cam_id]) elif cleaning_method == 'fact_image_cleaning': mask = fact_image_cleaning( camera, dl1.image[0], *config.cleaning_level_fact[camera.cam_id]) try: cleaned = dl1.image[0].copy() cleaned[~mask] = 0 hillas_container = hillas_parameters( camera, cleaned, ) params[telescope_id] = hillas_container except HillasParameterizationError: continue # probably wise to add try...except blocks here as well # Add more Features here (look what ctapipe can do, timing?) num_islands, island_labels = number_of_islands(camera, mask) island_dict = { 'num_islands': num_islands, 'island_labels': island_labels } leakage_container = leakage(camera, dl1.image[0], mask) timing_container = timing_parameters(camera, dl1.image[0], dl1.peakpos[0], hillas_container) pointing_azimuth[ telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad pointing_altitude[ telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad tel_x[telescope_id] = event.inst.subarray.positions[telescope_id][0] tel_y[telescope_id] = event.inst.subarray.positions[telescope_id][1] telescope_description = event.inst.subarray.tel[telescope_id] tel_focal_lengths[ telescope_id] = telescope_description.optics.equivalent_focal_length d = { 'array_event_id': event.dl0.event_id, 'telescope_id': int(telescope_id), 'camera_name': camera.cam_id, 'camera_id': config.names_to_id[camera.cam_id], 'run_id': event.r0.obs_id, 'telescope_type_name': telescope_type_name, 'telescope_type_id': config.types_to_id[telescope_type_name], 'pointing_azimuth': event.mc.tel[telescope_id].azimuth_raw, 'pointing_altitude': event.mc.tel[telescope_id].altitude_raw, 'mirror_area': telescope_description.optics.mirror_area, 'focal_length': telescope_description.optics.equivalent_focal_length, } d.update(hillas_container.as_dict()) d.update(leakage_container.as_dict()) d.update(island_dict) d.update(timing_container.as_dict()) features[telescope_id] = ({k: strip_unit(v) for k, v in d.items()}) if reco_algorithm == 'intersection': reco = HillasIntersection() array_direction = SkyCoord(alt=event.mcheader.run_array_direction[1], az=event.mcheader.run_array_direction[0], frame='altaz') reconstruction = reco.predict(params, tel_x, tel_y, tel_focal_lengths, array_direction) elif reco_algorithm == 'planes': reco = HillasReconstructor() reconstruction = reco.predict(params, event.inst, pointing_altitude, pointing_azimuth) for telescope_id in event.dl1.tel.keys(): if telescope_id not in params: continue camera = event.inst.subarray.tels[telescope_id].camera if camera.cam_id not in config.allowed_cameras: continue pos = event.inst.subarray.positions[telescope_id] x, y = pos[0], pos[1] core_x = reconstruction.core_x core_y = reconstruction.core_y d = np.sqrt((core_x - x)**2 + (core_y - y)**2) features[telescope_id]['distance_to_core'] = d.value return pd.DataFrame(list(features.values())), reconstruction, params
# cleaning boundary, picture, min_neighbors = cleaning_level[geom.camera_name] clean = tailcuts_clean(geom, image, boundary_thresh=boundary, picture_thresh=picture, min_number_picture_neighbors=min_neighbors) # ignore images with less than 5 pixels after cleaning if clean.sum() < 5: continue # image parameters hillas_c = hillas_parameters(geom[clean], image[clean]) leakage_c = leakage(geom, image, clean) n_islands, island_ids = number_of_islands(geom, clean) timing_c = timing_parameters( geom[clean], image[clean], peakpos[clean], hillas_c, ) # store parameters for stereo reconstruction hillas_containers[telescope_id] = hillas_c # store timegradients for plotting # ASTRI has no timing in PROD3b, so we use skewness instead if geom.camera_name != 'ASTRICam':
boundary, picture, min_neighbors = cleaning_level[camera.cam_id] clean = tailcuts_clean( camera, image, boundary_thresh=boundary, picture_thresh=picture, min_number_picture_neighbors=min_neighbors ) # ignore images with less than 5 pixels after cleaning if clean.sum() < 5: continue # image parameters hillas_c = hillas_parameters(camera[clean], image[clean]) leakage_c = leakage(camera, image, clean) n_islands, island_ids = number_of_islands(camera, clean) timing_c = timing_parameters( camera[clean], image[clean], peakpos[clean], hillas_c ) # store parameters for stereo reconstruction hillas_containers[telescope_id] = hillas_c pointing_azimuth[telescope_id] = event.mc.tel[telescope_id].azimuth_raw * u.rad pointing_altitude[telescope_id] = event.mc.tel[telescope_id].altitude_raw * u.rad # store timegradients for plotting # ASTRI has no timing in PROD3b, so we use skewness instead if camera.cam_id != 'ASTRICam': time_gradients[telescope_id] = timing_c.slope.value
def set_leakage(self, geom, image, clean): leakage_c = leakage(geom, image, clean) self.leakage1_intensity = leakage_c.leakage1_intensity self.leakage2_intensity = leakage_c.leakage2_intensity self.leakage1_pixel = leakage_c.leakage1_pixel self.leakage2_pixel = leakage_c.leakage2_pixel
def calculate_image_features(telescope_id, event, dl1, config): ''' Performs cleaning and adds the following image parameters: - hillas - leakage - concentration - timing - number of islands Make sure to adapt cleaning levels to the used algorithm (-> config) - tailcuts: picture_thresh, picture_thresh, min_number_picture_neighbors - fact_image_cleaning: picture_threshold, boundary_threshold, min_number_neighbors, time_limit Returns: -------- TelescopeParameterContainer ''' array_event_id = event.dl0.event_id run_id = event.r0.obs_id telescope = event.inst.subarray.tels[telescope_id] image = dl1.image # might want to make the parameter names more consistent between methods if config.cleaning_method == 'tailcuts_clean': boundary_thresh, picture_thresh, min_number_picture_neighbors = config.cleaning_level[ telescope.camera.cam_id ] mask = tailcuts_clean( telescope.camera, image, boundary_thresh=boundary_thresh, picture_thresh=picture_thresh, min_number_picture_neighbors=min_number_picture_neighbors, ) elif config.cleaning_method == 'fact_image_cleaning': boundary_threshold, picture_threshold, time_limit, min_number_neighbors = config.cleaning_level[ telescope.camera.cam_id ] mask = fact_image_cleaning( telescope.camera, image, dl1.pulse_time, boundary_threshhold=boundary_threshold, picture_threshold=picture_threshold, min_number_neighbors=min_number_neighbors, time_limit=time_limit, ) cleaned = image.copy() cleaned[~mask] = 0 logging.debug(f'calculating hillas for event {array_event_id, telescope_id}') hillas_container = hillas_parameters(telescope.camera, cleaned) hillas_container.prefix = '' logging.debug(f'calculating leakage for event {array_event_id, telescope_id}') leakage_container = leakage(telescope.camera, image, mask) leakage_container.prefix = '' logging.debug(f'calculating concentration for event {array_event_id, telescope_id}') concentration_container = concentration(telescope.camera, image, hillas_container) concentration_container.prefix = '' logging.debug(f'getting timing information for event {array_event_id, telescope_id}') timing_container = timing_parameters(telescope.camera, image, dl1.pulse_time, hillas_container) timing_container.prefix = '' # membership missing for now as it causes problems with the hdf5tablewriter # right now i dont need this anyway logging.debug(f'calculating num_islands for event {array_event_id, telescope_id}') num_islands, membership = number_of_islands(telescope.camera, mask) island_container = IslandContainer(num_islands=num_islands) island_container.prefix = '' num_pixel_in_shower = mask.sum() logging.debug(f'getting pointing container for event {array_event_id, telescope_id}') # ctapipe requires this to be rad pointing_container = TelescopePointingContainer( azimuth=event.mc.tel[telescope_id].azimuth_raw * u.rad, altitude=event.mc.tel[telescope_id].altitude_raw * u.rad, prefix='pointing', ) return TelescopeParameterContainer( telescope_id=telescope_id, run_id=run_id, array_event_id=array_event_id, leakage=leakage_container, hillas=hillas_container, concentration=concentration_container, pointing=pointing_container, timing=timing_container, islands=island_container, telescope_type_id=config.types_to_id[telescope.type], camera_type_id=config.names_to_id[telescope.camera.cam_id], focal_length=telescope.optics.equivalent_focal_length, mirror_area=telescope.optics.mirror_area, num_pixel_in_shower=num_pixel_in_shower, )
def process_telescope(tel, dl1, stereo): geom = tel.camera.geometry camera_name = tel.camera.camera_name image = dl1.image peak_time = dl1.peak_time # Cleaning using CHEC method clean = obtain_cleaning_mask(geom, image, peak_time, camera_name) # Skipping inadequate events if clean.sum() == 0: return None, None, None if stereo and clean.sum() < 5: return None, None, None # Get hillas parameters hillas_c = hillas_parameters(geom[clean], image[clean]) if hillas_c.width == 0 or np.isnan(hillas_c.width.value): return None, None, None # Get leakage and islands leakage_c = leakage(geom, image, clean) n_islands, island_ids = number_of_islands(geom, clean) # Get time gradient tgrad = np.nan try: timing_c = timing_parameters(geom, image, peak_time, hillas_c, clean) tgrad = timing_c.slope.value except BaseException: print("Timing parameters didn't work. clean.sum() = " + str(clean.sum()), "\n") # Grab info for telescope_events tel_data_dict = { 'nislands': n_islands, 'telescope_type': tel.type, 'camera_type': tel.camera.camera_name, 'focal_length': tel.optics.equivalent_focal_length.value, 'n_survived_pixels': clean.sum(), 'tgradient': tgrad, 'x': hillas_c.x.value, 'y': hillas_c.y.value, 'r': hillas_c.r.value, 'phi': hillas_c.phi.value, 'intensity': hillas_c.intensity, 'length': hillas_c.length.value, 'width': hillas_c.width.value, 'psi': hillas_c.psi.value, 'skewness': hillas_c.skewness, 'kurtosis': hillas_c.kurtosis, 'pixels_width_1': leakage_c.pixels_width_1, 'pixels_width_2': leakage_c.pixels_width_2, 'intensity_width_1': leakage_c.intensity_width_1, 'intensity_width_2': leakage_c.intensity_width_2} return tel.type, tel_data_dict, hillas_c
def prepare_event(self, source, return_stub=True, save_images=False, debug=False): """ Calibrate, clean and reconstruct the direction of an event. Parameters ---------- source : ctapipe.io.EventSource A container of selected showers from a simtel file. geom_cam_tel: dict Dictionary of of MyCameraGeometry objects for each camera in the file return_stub : bool If True, yield also images from events that won't be reconstructed. This feature is not currently available. save_images : bool If True, save photoelectron images from reconstructed events. debug : bool If True, print some debugging information (to be expanded). Yields ------ PreparedEvent: dict Dictionary containing event-image information to be written. """ # ============================================================= # TRANSFORMED CAMERA GEOMETRIES # ============================================================= # These are the camera geometries were the Hillas parametrization will # be performed. # They are transformed to TelescopeFrame using the effective focal # lengths # These geometries could be used also to performe the image cleaning, # but for the moment we still do that in the CameraFrame geom_cam_tel = {} for camera in source.subarray.camera_types: # Original geometry of each camera geom = camera.geometry # Same but with focal length as an attribute # This is planned to disappear and be imported by ctapipe focal_length = effective_focal_lengths(camera.camera_name) geom_cam_tel[camera.camera_name] = MyCameraGeometry( camera_name=camera.camera_name, pix_type=geom.pix_type, pix_id=geom.pix_id, pix_x=geom.pix_x, pix_y=geom.pix_y, pix_area=geom.pix_area, cam_rotation=geom.cam_rotation, pix_rotation=geom.pix_rotation, frame=CameraFrame(focal_length=focal_length), ).transform_to(TelescopeFrame()) # ============================================================= ievt = 0 for event in source: # Display event counts if debug: print( bcolors.BOLD + f"EVENT #{event.count}, EVENT_ID #{event.index.event_id}" + bcolors.ENDC) print(bcolors.BOLD + f"has triggered telescopes {event.r0.tels_with_data}" + bcolors.ENDC) ievt += 1 # if (ievt < 10) or (ievt % 10 == 0): # print(ievt) self.event_cutflow.count("noCuts") if self.event_cutflow.cut("min2Tels trig", len(event.dl0.tels_with_data)): if return_stub: print(bcolors.WARNING + f"WARNING : < {self.min_tel} triggered telescopes!" + bcolors.ENDC) pass # we register this, but we proceed to analyze it # yield stub(event) # else: # continue # ============================================================= # CALIBRATION # ============================================================= if debug: print(bcolors.OKBLUE + "Extracting all calibrated images..." + bcolors.ENDC) self.calib(event) # Calibrate the event # ============================================================= # BEGINNING OF LOOP OVER TELESCOPES # ============================================================= dl1_phe_image = {} dl1_phe_image_mask_reco = {} dl1_phe_image_mask_clusters = {} mc_phe_image = {} max_signals = {} n_pixel_dict = {} hillas_dict_reco = {} # for direction reconstruction hillas_dict = {} # for discrimination leakage_dict = {} n_tels = { "Triggered": len(event.dl0.tels_with_data), "LST_LST_LSTCam": 0, "MST_MST_NectarCam": 0, "MST_MST_FlashCam": 0, "MST_SCT_SCTCam": 0, "SST_1M_DigiCam": 0, "SST_ASTRI_ASTRICam": 0, "SST_GCT_CHEC": 0, } n_cluster_dict = {} impact_dict_reco = {} # impact distance measured in tilt system point_azimuth_dict = {} point_altitude_dict = {} good_for_reco = {} # 1 = success, 0 = fail # Compute impact parameter in tilt system run_array_direction = event.mcheader.run_array_direction az, alt = run_array_direction[0], run_array_direction[1] ground_frame = GroundFrame() for tel_id in event.dl0.tels_with_data: point_azimuth_dict[ tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad point_altitude_dict[ tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad if debug: print(bcolors.OKBLUE + f"Working on telescope #{tel_id}..." + bcolors.ENDC) self.image_cutflow.count("noCuts") camera = source.subarray.tel[tel_id].camera.geometry # count the current telescope according to its type tel_type = str(source.subarray.tel[tel_id]) n_tels[tel_type] += 1 # use ctapipe's functionality to get the calibrated image pmt_signal = event.dl1.tel[tel_id].image # If required... if save_images is True: # Save the simulated and reconstructed image of the event dl1_phe_image[tel_id] = pmt_signal mc_phe_image[tel_id] = event.mc.tel[tel_id].true_image # We now ASSUME that the event will be good good_for_reco[tel_id] = 1 # later we change to 0 if any condition is NOT satisfied if self.cleaner_reco.mode == "tail": # tail uses only ctapipe # Cleaning used for direction reconstruction image_biggest, mask_reco = self.cleaner_reco.clean_image( pmt_signal, camera) # calculate the leakage (before filtering) leakages = {} # this is needed by both cleanings # The check on SIZE shouldn't be here, but for the moment # I prefer to sacrifice elegancy... if np.sum(image_biggest[mask_reco]) != 0.0: leakage_biggest = leakage(camera, image_biggest, mask_reco) leakages["leak1_reco"] = leakage_biggest[ "intensity_width_1"] leakages["leak2_reco"] = leakage_biggest[ "intensity_width_2"] else: leakages["leak1_reco"] = 0.0 leakages["leak2_reco"] = 0.0 # find all islands using this cleaning num_islands, labels = number_of_islands(camera, mask_reco) if num_islands == 1: # if only ONE islands is left ... # ...use directly the old mask and reduce dimensions # to make Hillas parametrization faster camera_biggest = camera[mask_reco] image_biggest = image_biggest[mask_reco] if save_images is True: dl1_phe_image_mask_reco[tel_id] = mask_reco elif num_islands > 1: # if more islands survived.. # ...find the biggest one mask_biggest = largest_island(labels) # and also reduce dimensions camera_biggest = camera[mask_biggest] image_biggest = image_biggest[mask_biggest] if save_images is True: dl1_phe_image_mask_reco[tel_id] = mask_biggest else: # if no islands survived use old camera and image camera_biggest = camera dl1_phe_image_mask_reco[tel_id] = mask_reco # Cleaning used for score/energy estimation image_extended, mask_extended = self.cleaner_extended.clean_image( pmt_signal, camera) dl1_phe_image_mask_clusters[tel_id] = mask_extended # calculate the leakage (before filtering) # this part is not well coded, but for the moment it works if np.sum(image_extended[mask_extended]) != 0.0: leakage_extended = leakage(camera, image_extended, mask_extended) leakages["leak1"] = leakage_extended[ "intensity_width_1"] leakages["leak2"] = leakage_extended[ "intensity_width_2"] else: leakages["leak1"] = 0.0 leakages["leak2"] = 0.0 # find all islands with this cleaning # we will also register how many have been found n_cluster_dict[tel_id], labels = number_of_islands( camera, mask_extended) # NOTE: the next check shouldn't be necessary if we keep # all the isolated pixel clusters, but for now the # two cleanings are set the same in analysis.yml because # the performance of the extended one has never been really # studied in model estimation. # if some islands survived if n_cluster_dict[tel_id] > 0: # keep all of them and reduce dimensions camera_extended = camera[mask_extended] image_extended = image_extended[mask_extended] else: # otherwise continue with the old camera and image camera_extended = camera # could this go into `hillas_parameters` ...? # this is basically the charge of ALL islands # not calculated later by the Hillas parametrization! max_signals[tel_id] = np.max(image_extended) else: # for wavelets we stick to old pywi-cta code try: # "try except FileNotFoundError" not clear to me, but for now it stays... with warnings.catch_warnings(): # Image with biggest cluster (reco cleaning) image_biggest, mask_reco = self.cleaner_reco.clean_image( pmt_signal, camera) image_biggest2d = geometry_converter.image_1d_to_2d( image_biggest, camera.camera_name) image_biggest2d = filter_pixels_clusters( image_biggest2d) image_biggest = geometry_converter.image_2d_to_1d( image_biggest2d, camera.camera_name) # Image for score/energy estimation (with clusters) ( image_extended, mask_extended, ) = self.cleaner_extended.clean_image( pmt_signal, camera) # This last part was outside the pywi-cta block # before, but is indeed part of it because it uses # pywi-cta functions in the "extended" case # For cluster counts image_2d = geometry_converter.image_1d_to_2d( image_extended, camera.camera_name) n_cluster_dict[ tel_id] = pixel_clusters.number_of_pixels_clusters( array=image_2d, threshold=0) # could this go into `hillas_parameters` ...? max_signals[tel_id] = np.max(image_extended) except FileNotFoundError as e: # JLK, WHAT? print(e) continue # ============================================================= # PRELIMINARY IMAGE SELECTION # ============================================================= cleaned_image_is_good = True # we assume this # Apply some selection if self.image_cutflow.cut("min pixel", image_biggest): if debug: print(bcolors.WARNING + "WARNING : not enough pixels!" + bcolors.ENDC) good_for_reco[tel_id] = 0 # we record it as BAD cleaned_image_is_good = False if self.image_cutflow.cut("min charge", np.sum(image_biggest)): if debug: print(bcolors.WARNING + "WARNING : not enough charge!" + bcolors.ENDC) good_for_reco[tel_id] = 0 # we record it as BAD cleaned_image_is_good = False if debug and (not cleaned_image_is_good): # BAD image quality print(bcolors.WARNING + "WARNING : The cleaned image didn't pass" + " preliminary cuts.\n" + "An attempt to parametrize it will be made," + " but the image will NOT be used for" + " direction reconstruction." + bcolors.ENDC) # ============================================================= # IMAGE PARAMETRIZATION # ============================================================= with np.errstate(invalid="raise", divide="raise"): try: # Filter the cameras in TelescopeFrame with the same # cleaning masks camera_biggest_tel = geom_cam_tel[camera.camera_name][ camera_biggest.pix_id] camera_extended_tel = geom_cam_tel[camera.camera_name][ camera_extended.pix_id] # Parametrize the image in the TelescopeFrame moments_reco = hillas_parameters( camera_biggest_tel, image_biggest) # for geometry (eg direction) moments = hillas_parameters( camera_extended_tel, image_extended ) # for discrimination and energy reconstruction if debug: print( "Image parameters from main cluster cleaning:") print(moments_reco) print( "Image parameters from all-clusters cleaning:") print(moments) # =================================================== # PARAMETRIZED IMAGE SELECTION # =================================================== # if width and/or length are zero (e.g. when there is # only only one pixel or when all pixel are exactly # in one row), the parametrisation # won't be very useful: skip if self.image_cutflow.cut("poor moments", moments_reco): if debug: print(bcolors.WARNING + "WARNING : poor moments!" + bcolors.ENDC) good_for_reco[tel_id] = 0 # we record it as BAD if self.image_cutflow.cut("close to the edge", moments_reco, camera.camera_name): if debug: print( bcolors.WARNING + "WARNING : out of containment radius!\n" + f"Camera radius = {self.camera_radius[camera.camera_name]}\n" + f"COG radius = {moments_reco.r}" + bcolors.ENDC) good_for_reco[tel_id] = 0 if self.image_cutflow.cut("bad ellipticity", moments_reco): if debug: print(bcolors.WARNING + "WARNING : bad ellipticity" + bcolors.ENDC) good_for_reco[tel_id] = 0 if debug and good_for_reco[tel_id] == 1: print( bcolors.OKGREEN + "Image survived and correctly parametrized." # + "\nIt will be used for direction reconstruction!" + bcolors.ENDC) elif debug and good_for_reco[tel_id] == 0: print( bcolors.WARNING + "Image not survived or " + "not good enough for parametrization." # + "\nIt will be NOT used for direction reconstruction, " # + "BUT it's information will be recorded." + bcolors.ENDC) hillas_dict[tel_id] = moments hillas_dict_reco[tel_id] = moments_reco n_pixel_dict[tel_id] = len( np.where(image_extended > 0)[0]) leakage_dict[tel_id] = leakages except ( FloatingPointError, HillasParameterizationError, ValueError, ) as e: if debug: print(bcolors.FAIL + "Parametrization error: " + f"{e}\n" + "Dummy parameters recorded." + bcolors.ENDC) good_for_reco[tel_id] = 0 hillas_dict[ tel_id] = HillasParametersTelescopeFrameContainer( ) hillas_dict_reco[ tel_id] = HillasParametersTelescopeFrameContainer( ) n_pixel_dict[tel_id] = len( np.where(image_extended > 0)[0]) leakage_dict[tel_id] = leakages # END OF THE CYCLE OVER THE TELESCOPES # ============================================================= # DIRECTION RECONSTRUCTION # ============================================================= # n_tels["reco"] = len(hillas_dict_reco) # n_tels["discri"] = len(hillas_dict) # convert dictionary in numpy array to get a "mask" images_status = np.asarray(list(good_for_reco.values())) # record how many images will used for reconstruction n_tels["GOOD images"] = len( np.extract(images_status == 1, images_status)) n_tels["BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"] if self.event_cutflow.cut("min2Tels reco", n_tels["GOOD images"]): if debug: print( bcolors.FAIL + f"WARNING: < {self.min_ntel} ({n_tels['GOOD images']}) images remaining!" + "\nWARNING : direction reconstruction is not possible!" + bcolors.ENDC) # create a dummy container for direction reconstruction reco_result = ReconstructedShowerContainer() if return_stub: # if saving all events (default) if debug: print(bcolors.OKBLUE + "Recording event..." + bcolors.ENDC) print( bcolors.WARNING + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, ) continue else: continue if debug: print(bcolors.OKBLUE + "Starting direction reconstruction..." + bcolors.ENDC) try: with warnings.catch_warnings(): warnings.simplefilter("ignore") # use only the successfully parametrized images # to reconstruct the direction of this event successfull_hillas = np.where(images_status == 1)[0] all_images = np.asarray(list(good_for_reco.keys())) good_images = set(all_images[successfull_hillas]) good_hillas_dict_reco = { k: v for k, v in hillas_dict_reco.items() if k in good_images } if debug: print(bcolors.PURPLE + f"{len(good_hillas_dict_reco)} images will be " + "used to recover the shower's direction..." + bcolors.ENDC) # Reconstruction results reco_result = self.shower_reco.predict( good_hillas_dict_reco, source.subarray, SkyCoord(alt=alt, az=az, frame="altaz"), None, # use the array direction ) # Impact parameter for energy estimation (/ tel) subarray = source.subarray for tel_id in hillas_dict.keys(): pos = subarray.positions[tel_id] tel_ground = SkyCoord(pos[0], pos[1], pos[2], frame=ground_frame) core_ground = SkyCoord( reco_result.core_x, reco_result.core_y, 0 * u.m, frame=ground_frame, ) # Should be better handled (tilted frame) impact_dict_reco[tel_id] = np.sqrt( (core_ground.x - tel_ground.x)**2 + (core_ground.y - tel_ground.y)**2) except (Exception, TooFewTelescopesException, InvalidWidthException) as e: if debug: print("exception in reconstruction:", e) raise if return_stub: if debug: print( bcolors.FAIL + "Shower could NOT be correctly reconstructed! " + "Recording event..." + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, ) else: continue if self.event_cutflow.cut("direction nan", reco_result): if debug: print(bcolors.WARNING + "WARNING: undefined direction!" + bcolors.ENDC) if return_stub: if debug: print( bcolors.FAIL + "Shower could NOT be correctly reconstructed! " + "Recording event..." + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, ) else: continue if debug: print(bcolors.BOLDGREEN + "Shower correctly reconstructed! " + "Recording event..." + bcolors.ENDC) yield PreparedEvent( event=event, dl1_phe_image=dl1_phe_image, dl1_phe_image_mask_reco=dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters=dl1_phe_image_mask_clusters, mc_phe_image=mc_phe_image, n_pixel_dict=n_pixel_dict, hillas_dict=hillas_dict, hillas_dict_reco=hillas_dict_reco, leakage_dict=leakage_dict, n_tels=n_tels, max_signals=max_signals, n_cluster_dict=n_cluster_dict, reco_result=reco_result, impact_dict=impact_dict_reco, good_event=True, good_for_reco=good_for_reco, )