def test_number_of_islands(): from ctapipe.image import number_of_islands # test with LST geometry (1855 pixels) geom = CameraGeometry.from_name("LSTCam") # create 18 triggered pixels grouped to 5 clusters mask = np.zeros(geom.n_pixels).astype("bool") triggered_pixels = np.array( [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 37, 38, 111, 222]) mask[triggered_pixels] = True island_labels_true = np.zeros(geom.n_pixels, dtype=np.int16) island_labels_true[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]] = 1 island_labels_true[14] = 2 island_labels_true[[37, 38]] = 3 island_labels_true[111] = 4 island_labels_true[222] = 5 n_islands, island_labels = number_of_islands(geom, mask) n_islands_true = 5 # non cleaning pixels should be zero assert np.all(island_labels[~mask] == 0) # all other should have some label assert np.all(island_labels[mask] > 0) assert n_islands == n_islands_true assert_allclose(island_labels, island_labels_true) assert island_labels.dtype == np.int16
def test_largest_island(): """Test selection of largest island in imagea with given cleaning masks.""" from ctapipe.image import number_of_islands, largest_island # Create a simple rectangular camera made of 17 pixels camera = CameraGeometry.make_rectangular(17, 1) # Assume a simple image (flattened array) made of 0, 1 or 2 photoelectrons # [2, 1, 1, 1, 1, 2, 2, 1, 0, 2, 1, 1, 1, 0, 2, 2, 2] # Assume a virtual tailcut cleaning that requires: # - picture_threshold = 2 photoelectrons, # - boundary_threshold = 1 photoelectron, # - min_number_picture_neighbors = 0 # There will be 4 islands left after cleaning: clean_mask = np.zeros(camera.n_pixels).astype("bool") # initialization clean_mask[[0, 1]] = 1 clean_mask[[4, 5, 6, 7]] = 2 # this is the biggest clean_mask[[9, 10]] = 3 clean_mask[[14, 15, 16]] = 4 # Label islands (their number is not important here) _, islands_labels = number_of_islands(camera, clean_mask) # Create the true mask which takes into account only the biggest island # Pixels with no signal are labelled with a 0 true_mask_largest = np.zeros(camera.n_pixels).astype("bool") true_mask_largest[[4, 5, 6, 7]] = 1 # Apply the function to test mask_largest = largest_island(islands_labels) # Now the degenerate case of only one island surviving # Same process as before clean_mask_one = np.zeros(camera.n_pixels).astype("bool") clean_mask_one[[0, 1]] = 1 _, islands_labels_one = number_of_islands(camera, clean_mask_one) true_mask_largest_one = np.zeros(camera.n_pixels).astype("bool") true_mask_largest_one[[0, 1]] = 1 mask_largest_one = largest_island(islands_labels_one) # Last the case of no islands surviving clean_mask_0 = np.zeros(camera.n_pixels).astype("bool") _, islands_labels_0 = number_of_islands(camera, clean_mask_0) true_mask_largest_0 = np.zeros(camera.n_pixels).astype("bool") mask_largest_0 = largest_island(islands_labels_0) # Check if the function recovers the ground truth in all of the three cases assert (mask_largest_one == true_mask_largest_one).all() assert (mask_largest_0 == true_mask_largest_0).all() assert_allclose(mask_largest, true_mask_largest)
def mask_from_biggest_island(charge: np.array, geometry: CameraGeometry, mask): n_islands, labels = number_of_islands(geometry, mask) if n_islands == 1: camera_biggest = geometry[mask] charge_biggest = charge[mask] elif n_islands > 1: mask_biggest = largest_island(labels) camera_biggest = geometry[mask_biggest] charge_biggest = charge[mask_biggest] else: camera_biggest = geometry charge_biggest = charge return charge_biggest, camera_biggest, n_islands
def test_number_of_islands(): from ctapipe.image import number_of_islands # test with LST geometry (1855 pixels) geom = CameraGeometry.from_name("LSTCam") # create 18 triggered pixels grouped to 5 clusters island_mask_true = np.zeros(geom.n_pixels) mask = np.zeros(geom.n_pixels).astype("bool") triggered_pixels = np.array( [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 37, 38, 111, 222]) island_mask_true[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]] = 1 island_mask_true[14] = 2 island_mask_true[[37, 38]] = 3 island_mask_true[111] = 4 island_mask_true[222] = 5 mask[triggered_pixels] = 1 n_islands, island_mask = number_of_islands(geom, mask) n_islands_true = 5 assert n_islands == n_islands_true assert_allclose(island_mask, island_mask_true) assert island_mask.dtype == np.int32
def get_dl1( calibrated_event, subarray, telescope_id, dl1_container=None, custom_config={}, ): """ Return a DL1ParametersContainer of extracted features from a calibrated event. The DL1ParametersContainer can be passed to be filled if created outside the function (faster for multiple event processing) Parameters ---------- calibrated_event: ctapipe event container subarray: `ctapipe.instrument.subarray.SubarrayDescription` telescope_id: `int` dl1_container: DL1ParametersContainer custom_config: path to a configuration file configuration used for tailcut cleaning supersedes the standard configuration Returns ------- DL1ParametersContainer """ config = replace_config(standard_config, custom_config) # pop delta_time and use_main_island, so we can pass cleaning_parameters to # tailcuts cleaning_parameters = config["tailcut"].copy() delta_time = cleaning_parameters.pop("delta_time", None) use_main_island = cleaning_parameters.pop("use_only_main_island", True) use_dynamic_cleaning = False if "apply" in config["dynamic_cleaning"]: use_dynamic_cleaning = config["dynamic_cleaning"]["apply"] dl1_container = DL1ParametersContainer( ) if dl1_container is None else dl1_container dl1 = calibrated_event.dl1.tel[telescope_id] telescope = subarray.tel[telescope_id] camera_geometry = telescope.camera.geometry optics = telescope.optics image = dl1.image peak_time = dl1.peak_time signal_pixels = cleaning_method(camera_geometry, image, **cleaning_parameters) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: if delta_time is not None: signal_pixels = apply_time_delta_cleaning(camera_geometry, signal_pixels, peak_time, min_number_neighbors=1, time_limit=delta_time) if use_dynamic_cleaning: threshold_dynamic = config['dynamic_cleaning']['threshold'] fraction_dynamic = config['dynamic_cleaning'][ 'fraction_cleaning_intensity'] signal_pixels = apply_dynamic_cleaning(image, signal_pixels, threshold_dynamic, fraction_dynamic) # check the number of islands num_islands, island_labels = number_of_islands(camera_geometry, signal_pixels) dl1_container.n_islands = num_islands if use_main_island: n_pixels_on_island = np.bincount(island_labels) n_pixels_on_island[ 0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) signal_pixels[island_labels != max_island_label] = False # count surviving pixels n_pixels = np.count_nonzero(signal_pixels) dl1_container.n_pixels = n_pixels if n_pixels > 0: parametrize_image( image=image, peak_time=peak_time, signal_pixels=signal_pixels, camera_geometry=camera_geometry, focal_length=optics.equivalent_focal_length, dl1_container=dl1_container, ) # We set other fields which still make sense for a non-parametrized # image: dl1_container.set_telescope_info(subarray, telescope_id) # save the applied cleaning mask calibrated_event.dl1.tel[telescope_id].image_mask = signal_pixels return dl1_container
def prepare_event(self, source, return_stub=True, save_images=False, debug=False): """ Calibrate, clean and reconstruct the direction of an event. Parameters ---------- source : ctapipe.io.EventSource A container of selected showers from a simtel file. geom_cam_tel: dict Dictionary of of MyCameraGeometry objects for each camera in the file return_stub : bool If True, yield also images from events that won't be reconstructed. This feature is not currently available. save_images : bool If True, save photoelectron images from reconstructed events. debug : bool If True, print some debugging information (to be expanded). Yields ------ PreparedEvent: dict Dictionary containing event-image information to be written. """ # ============================================================= # TRANSFORMED CAMERA GEOMETRIES # ============================================================= # These are the camera geometries were the Hillas parametrization will # be performed. # They are transformed to TelescopeFrame using the effective focal # lengths # These geometries could be used also to performe the image cleaning, # but for the moment we still do that in the CameraFrame geom_cam_tel = {} for camera in source.subarray.camera_types: # Original geometry of each camera geom = camera.geometry # Same but with focal length as an attribute # This is planned to disappear and be imported by ctapipe focal_length = effective_focal_lengths(camera.camera_name) geom_cam_tel[camera.camera_name] = MyCameraGeometry( camera_name=camera.camera_name, pix_type=geom.pix_type, pix_id=geom.pix_id, pix_x=geom.pix_x, pix_y=geom.pix_y, pix_area=geom.pix_area, cam_rotation=geom.cam_rotation, pix_rotation=geom.pix_rotation, frame=CameraFrame(focal_length=focal_length), ).transform_to(TelescopeFrame()) # ============================================================= ievt = 0 for event in source: # Display event counts if debug: print( bcolors.BOLD + f"EVENT #{event.count}, EVENT_ID #{event.index.event_id}" + bcolors.ENDC) print(bcolors.BOLD + f"has triggered telescopes {event.r1.tel.keys()}" + bcolors.ENDC) ievt += 1 # if (ievt < 10) or (ievt % 10 == 0): # print(ievt) self.event_cutflow.count("noCuts") # LST stereo condition # whenever there is only 1 LST in an event, we remove that telescope # if the remaining telescopes are less than min_tel we remove the event lst_tel_ids = set( source.subarray.get_tel_ids_for_type("LST_LST_LSTCam")) triggered_LSTs = set(event.r0.tel.keys()).intersection(lst_tel_ids) n_triggered_LSTs = len(triggered_LSTs) n_triggered_non_LSTs = len(event.r0.tel.keys()) - n_triggered_LSTs bad_LST_stereo = False if self.LST_stereo and self.event_cutflow.cut( "no-LST-stereo + <2 other types", n_triggered_LSTs, n_triggered_non_LSTs): bad_LST_stereo = True if return_stub: print( bcolors.WARNING + "WARNING: LST_stereo is set to 'True'\n" + f"This event has < {self.min_ntel_LST} triggered LSTs\n" + "and < 2 triggered telescopes from other telescope types.\n" + "The event will be processed up to DL1b." + bcolors.ENDC) # we show this, but we proceed to analyze the event up to # DL1a/b for the associated benchmarks # this checks for < 2 triggered telescopes of ANY type if self.event_cutflow.cut("min2Tels trig", len(event.r1.tel.keys())): if return_stub: print( bcolors.WARNING + f"WARNING : < {self.min_ntel} triggered telescopes!" + bcolors.ENDC) # we show this, but we proceed to analyze it # ============================================================= # CALIBRATION # ============================================================= if debug: print(bcolors.OKBLUE + "Extracting all calibrated images..." + bcolors.ENDC) self.calib(event) # Calibrate the event # ============================================================= # BEGINNING OF LOOP OVER TELESCOPES # ============================================================= dl1_phe_image = {} dl1_phe_image_mask_reco = {} dl1_phe_image_mask_clusters = {} mc_phe_image = {} max_signals = {} n_pixel_dict = {} hillas_dict_reco = {} # for direction reconstruction hillas_dict = {} # for discrimination leakage_dict = {} concentration_dict = {} n_tels = { "Triggered": len(event.r1.tel.keys()), "LST_LST_LSTCam": 0, "MST_MST_NectarCam": 0, "MST_MST_FlashCam": 0, "MST_SCT_SCTCam": 0, "SST_1M_DigiCam": 0, "SST_ASTRI_ASTRICam": 0, "SST_GCT_CHEC": 0, } n_cluster_dict = {} impact_dict_reco = {} # impact distance measured in tilt system point_azimuth_dict = {} point_altitude_dict = {} good_for_reco = {} # 1 = success, 0 = fail # Array pointing in AltAz frame az = event.pointing.array_azimuth alt = event.pointing.array_altitude array_pointing = SkyCoord(az, alt, frame=AltAz()) ground_frame = GroundFrame() tilted_frame = TiltedGroundFrame(pointing_direction=array_pointing) for tel_id in event.r1.tel.keys(): point_azimuth_dict[tel_id] = event.pointing.tel[tel_id].azimuth point_altitude_dict[tel_id] = event.pointing.tel[ tel_id].altitude if debug: print(bcolors.OKBLUE + f"Working on telescope #{tel_id}..." + bcolors.ENDC) self.image_cutflow.count("noCuts") camera = source.subarray.tel[tel_id].camera.geometry # count the current telescope according to its type tel_type = str(source.subarray.tel[tel_id]) n_tels[tel_type] += 1 # use ctapipe's functionality to get the calibrated image # and scale the reconstructed values if required pmt_signal = event.dl1.tel[tel_id].image / self.calibscale # If required... if save_images is True: # Save the simulated and reconstructed image of the event dl1_phe_image[tel_id] = pmt_signal mc_phe_image[tel_id] = event.simulation.tel[ tel_id].true_image # We now ASSUME that the event will be good good_for_reco[tel_id] = 1 # later we change to 0 if any condition is NOT satisfied if self.cleaner_reco.mode == "tail": # tail uses only ctapipe # Cleaning used for direction reconstruction image_biggest, mask_reco = self.cleaner_reco.clean_image( pmt_signal, camera) # calculate the leakage (before filtering) leakages = {} # this is needed by both cleanings # The check on SIZE shouldn't be here, but for the moment # I prefer to sacrifice elegancy... if np.sum(image_biggest[mask_reco]) != 0.0: leakage_biggest = leakage_parameters( camera, image_biggest, mask_reco) leakages["leak1_reco"] = leakage_biggest[ "intensity_width_1"] leakages["leak2_reco"] = leakage_biggest[ "intensity_width_2"] else: leakages["leak1_reco"] = 0.0 leakages["leak2_reco"] = 0.0 # find all islands using this cleaning num_islands, labels = number_of_islands(camera, mask_reco) if num_islands == 1: # if only ONE islands is left ... # ...use directly the old mask and reduce dimensions # to make Hillas parametrization faster camera_biggest = camera[mask_reco] image_biggest = image_biggest[mask_reco] if save_images is True: dl1_phe_image_mask_reco[tel_id] = mask_reco elif num_islands > 1: # if more islands survived.. # ...find the biggest one mask_biggest = largest_island(labels) # and also reduce dimensions camera_biggest = camera[mask_biggest] image_biggest = image_biggest[mask_biggest] if save_images is True: dl1_phe_image_mask_reco[tel_id] = mask_biggest else: # if no islands survived use old camera and image camera_biggest = camera dl1_phe_image_mask_reco[tel_id] = mask_reco # Cleaning used for score/energy estimation image_extended, mask_extended = self.cleaner_extended.clean_image( pmt_signal, camera) dl1_phe_image_mask_clusters[tel_id] = mask_extended # calculate the leakage (before filtering) # this part is not well coded, but for the moment it works if np.sum(image_extended[mask_extended]) != 0.0: leakage_extended = leakage_parameters( camera, image_extended, mask_extended) leakages["leak1"] = leakage_extended[ "intensity_width_1"] leakages["leak2"] = leakage_extended[ "intensity_width_2"] else: leakages["leak1"] = 0.0 leakages["leak2"] = 0.0 # find all islands with this cleaning # we will also register how many have been found n_cluster_dict[tel_id], labels = number_of_islands( camera, mask_extended) # NOTE: the next check shouldn't be necessary if we keep # all the isolated pixel clusters, but for now the # two cleanings are set the same in analysis.yml because # the performance of the extended one has never been really # studied in model estimation. # if some islands survived if n_cluster_dict[tel_id] > 0: # keep all of them and reduce dimensions camera_extended = camera[mask_extended] image_extended = image_extended[mask_extended] else: # otherwise continue with the old camera and image camera_extended = camera # could this go into `hillas_parameters` ...? # this is basically the charge of ALL islands # not calculated later by the Hillas parametrization! max_signals[tel_id] = np.max(image_extended) else: # for wavelets we stick to old pywi-cta code try: # "try except FileNotFoundError" not clear to me, but for now it stays... with warnings.catch_warnings(): # Image with biggest cluster (reco cleaning) image_biggest, mask_reco = self.cleaner_reco.clean_image( pmt_signal, camera) image_biggest2d = geometry_converter.image_1d_to_2d( image_biggest, camera.camera_name) image_biggest2d = filter_pixels_clusters( image_biggest2d) image_biggest = geometry_converter.image_2d_to_1d( image_biggest2d, camera.camera_name) # Image for score/energy estimation (with clusters) ( image_extended, mask_extended, ) = self.cleaner_extended.clean_image( pmt_signal, camera) # This last part was outside the pywi-cta block # before, but is indeed part of it because it uses # pywi-cta functions in the "extended" case # For cluster counts image_2d = geometry_converter.image_1d_to_2d( image_extended, camera.camera_name) n_cluster_dict[ tel_id] = pixel_clusters.number_of_pixels_clusters( array=image_2d, threshold=0) # could this go into `hillas_parameters` ...? max_signals[tel_id] = np.max(image_extended) except FileNotFoundError as e: print(e) continue # ============================================================= # PRELIMINARY IMAGE SELECTION # ============================================================= cleaned_image_is_good = True # we assume this if self.image_selection_source == "extended": cleaned_image_to_use = image_extended elif self.image_selection_source == "biggest": cleaned_image_to_use = image_biggest else: raise ValueError( "Only supported cleanings are 'biggest' or 'extended'." ) # Apply some selection if self.image_cutflow.cut("min pixel", cleaned_image_to_use): if debug: print(bcolors.WARNING + "WARNING : not enough pixels!" + bcolors.ENDC) good_for_reco[tel_id] = 0 # we record it as BAD cleaned_image_is_good = False if self.image_cutflow.cut("min charge", np.sum(cleaned_image_to_use)): if debug: print(bcolors.WARNING + "WARNING : not enough charge!" + bcolors.ENDC) good_for_reco[tel_id] = 0 # we record it as BAD cleaned_image_is_good = False if debug and (not cleaned_image_is_good): # BAD image quality print(bcolors.WARNING + "WARNING : The cleaned image didn't pass" + " preliminary cuts.\n" + "An attempt to parametrize it will be made," + " but the image will NOT be used for" + " direction reconstruction." + bcolors.ENDC) # ============================================================= # IMAGE PARAMETRIZATION # ============================================================= with np.errstate(invalid="raise", divide="raise"): try: # Filter the cameras in TelescopeFrame with the same # cleaning masks camera_biggest_tel = geom_cam_tel[camera.camera_name][ camera_biggest.pix_id] camera_extended_tel = geom_cam_tel[camera.camera_name][ camera_extended.pix_id] # Parametrize the image in the TelescopeFrame moments_reco = hillas_parameters( camera_biggest_tel, image_biggest) # for geometry (eg direction) moments = hillas_parameters( camera_extended_tel, image_extended ) # for discrimination and energy reconstruction if debug: print( "Image parameters from main cluster cleaning:") print(moments_reco) print( "Image parameters from all-clusters cleaning:") print(moments) # Add concentration parameters concentrations = {} concentrations_extended = concentration_parameters( camera_extended_tel, image_extended, moments) concentrations[ "concentration_cog"] = concentrations_extended[ "cog"] concentrations[ "concentration_core"] = concentrations_extended[ "core"] concentrations[ "concentration_pixel"] = concentrations_extended[ "pixel"] # =================================================== # PARAMETRIZED IMAGE SELECTION # =================================================== if self.image_selection_source == "extended": moments_to_use = moments else: moments_to_use = moments_reco # if width and/or length are zero (e.g. when there is # only only one pixel or when all pixel are exactly # in one row), the parametrisation # won't be very useful: skip if self.image_cutflow.cut("poor moments", moments_to_use): if debug: print(bcolors.WARNING + "WARNING : poor moments!" + bcolors.ENDC) good_for_reco[tel_id] = 0 # we record it as BAD if self.image_cutflow.cut("close to the edge", moments_to_use, camera.camera_name): if debug: print( bcolors.WARNING + "WARNING : out of containment radius!\n" + f"Camera radius = {self.camera_radius[camera.camera_name]}\n" + f"COG radius = {moments_to_use.r}" + bcolors.ENDC) good_for_reco[tel_id] = 0 if self.image_cutflow.cut("bad ellipticity", moments_to_use): if debug: print(bcolors.WARNING + "WARNING : bad ellipticity" + bcolors.ENDC) good_for_reco[tel_id] = 0 if debug and good_for_reco[tel_id] == 1: print( bcolors.OKGREEN + "Image survived and correctly parametrized." # + "\nIt will be used for direction reconstruction!" + bcolors.ENDC) elif debug and good_for_reco[tel_id] == 0: print( bcolors.WARNING + "Image not survived or " + "not good enough for parametrization." # + "\nIt will be NOT used for direction reconstruction, " # + "BUT it's information will be recorded." + bcolors.ENDC) hillas_dict[tel_id] = moments hillas_dict_reco[tel_id] = moments_reco n_pixel_dict[tel_id] = len( np.where(image_extended > 0)[0]) leakage_dict[tel_id] = leakages concentration_dict[tel_id] = concentrations except ( FloatingPointError, HillasParameterizationError, ValueError, ) as e: if debug: print(bcolors.FAIL + "Parametrization error: " + f"{e}\n" + "Dummy parameters recorded." + bcolors.ENDC) good_for_reco[tel_id] = 0 hillas_dict[ tel_id] = HillasParametersTelescopeFrameContainer( ) hillas_dict_reco[ tel_id] = HillasParametersTelescopeFrameContainer( ) n_pixel_dict[tel_id] = len( np.where(image_extended > 0)[0]) leakage_dict[tel_id] = leakages concentration_dict[tel_id] = concentrations # END OF THE CYCLE OVER THE TELESCOPES # ============================================================= # DIRECTION RECONSTRUCTION # ============================================================= if bad_LST_stereo: if debug: print( bcolors.WARNING + "WARNING: This event was triggered with 1 LST image and <2 images from other telescope types." + "\nWARNING : direction reconstruction will not be performed." + bcolors.ENDC) # Set all the involved images as NOT good for recosntruction # even though they might have been # but this is because of the LST stereo trigger.... for tel_id in event.r0.tel.keys(): good_for_reco[tel_id] = 0 # and set the number of good and bad images accordingly n_tels["GOOD images"] = 0 n_tels[ "BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"] # create a dummy container for direction reconstruction reco_result = ReconstructedShowerContainer() if return_stub: # if saving all events (default) if debug: print(bcolors.OKBLUE + "Recording event..." + bcolors.ENDC) print( bcolors.WARNING + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, concentration_dict) continue else: continue # Now in case the only triggered telescopes were # - < self.min_ntel_LST LST, # - >=2 any other telescope type, # we remove the single-LST image and continue reconstruction with # the images from the other telescope types if self.LST_stereo and (n_triggered_LSTs < self.min_ntel_LST) and ( n_triggered_LSTs != 0) and (n_triggered_non_LSTs >= 2): if debug: print( bcolors.WARNING + f"WARNING: LST stereo trigger condition is active.\n" + f"This event triggered < {self.min_ntel_LST} LSTs " + f"and {n_triggered_non_LSTs} images from other telescope types.\n" + bcolors.ENDC) for tel_id in triggered_LSTs: # in case we test for min_ntel_LST>2 if good_for_reco[tel_id]: # we don't use it for reconstruction good_for_reco[tel_id] = 0 print( bcolors.WARNING + f"WARNING: LST image #{tel_id} removed, even though it passed quality cuts." + bcolors.ENDC) # TODO: book-keeping of this kind of events doesn't seem easy # convert dictionary in numpy array to get a "mask" images_status = np.asarray(list(good_for_reco.values())) # record how many images will be used for reconstruction n_tels["GOOD images"] = len( np.extract(images_status == 1, images_status)) n_tels["BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"] if self.event_cutflow.cut("min2Tels reco", n_tels["GOOD images"]): if debug: print( bcolors.FAIL + f"WARNING: < {self.min_ntel} ({n_tels['GOOD images']}) images remaining!" + "\nWARNING : direction reconstruction is not possible!" + bcolors.ENDC) # create a dummy container for direction reconstruction reco_result = ReconstructedShowerContainer() if return_stub: # if saving all events (default) if debug: print(bcolors.OKBLUE + "Recording event..." + bcolors.ENDC) print( bcolors.WARNING + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, concentration_dict) continue else: continue if debug: print(bcolors.OKBLUE + "Starting direction reconstruction..." + bcolors.ENDC) try: with warnings.catch_warnings(): warnings.simplefilter("ignore") if self.image_selection_source == "extended": hillas_dict_to_use = hillas_dict else: hillas_dict_to_use = hillas_dict_reco # use only the successfully parametrized images # to reconstruct the direction of this event successfull_hillas = np.where(images_status == 1)[0] all_images = np.asarray(list(good_for_reco.keys())) good_images = set(all_images[successfull_hillas]) good_hillas_dict = { k: v for k, v in hillas_dict_to_use.items() if k in good_images } if debug: print( bcolors.PURPLE + f"{len(good_hillas_dict)} images " + f"(from telescopes #{list(good_hillas_dict.keys())}) will be " + "used to recover the shower's direction..." + bcolors.ENDC) # Reconstruction results reco_result = self.shower_reco.predict( good_hillas_dict, source.subarray, SkyCoord(alt=alt, az=az, frame="altaz"), None, # use the array direction ) # Impact parameter for telescope-wise energy estimation subarray = source.subarray for tel_id in hillas_dict_to_use.keys(): pos = subarray.positions[tel_id] tel_ground = SkyCoord(pos[0], pos[1], pos[2], frame=ground_frame) core_ground = SkyCoord( reco_result.core_x, reco_result.core_y, 0 * u.m, frame=ground_frame, ) # Go back to the tilted frame # this should be the same... tel_tilted = tel_ground.transform_to(tilted_frame) # but this not core_tilted = SkyCoord(x=core_ground.x, y=core_ground.y, frame=tilted_frame) impact_dict_reco[tel_id] = np.sqrt( (core_tilted.x - tel_tilted.x)**2 + (core_tilted.y - tel_tilted.y)**2) except (Exception, TooFewTelescopesException, InvalidWidthException) as e: if debug: print("exception in reconstruction:", e) raise if return_stub: if debug: print( bcolors.FAIL + "Shower could NOT be correctly reconstructed! " + "Recording event..." + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, concentration_dict) else: continue if self.event_cutflow.cut("direction nan", reco_result): if debug: print(bcolors.WARNING + "WARNING: undefined direction!" + bcolors.ENDC) if return_stub: if debug: print( bcolors.FAIL + "Shower could NOT be correctly reconstructed! " + "Recording event..." + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, concentration_dict) else: continue if debug: print(bcolors.BOLDGREEN + "Shower correctly reconstructed! " + "Recording event..." + bcolors.ENDC) yield PreparedEvent( event=event, dl1_phe_image=dl1_phe_image, dl1_phe_image_mask_reco=dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters=dl1_phe_image_mask_clusters, mc_phe_image=mc_phe_image, n_pixel_dict=n_pixel_dict, hillas_dict=hillas_dict, hillas_dict_reco=hillas_dict_reco, leakage_dict=leakage_dict, concentration_dict=concentration_dict, n_tels=n_tels, max_signals=max_signals, n_cluster_dict=n_cluster_dict, reco_result=reco_result, impact_dict=impact_dict_reco, good_event=True, good_for_reco=good_for_reco, )
def main(): args = parser.parse_args() log.setLevel(logging.INFO) handler = logging.StreamHandler() logging.getLogger().addHandler(handler) if Path(args.output_file).exists(): log.critical(f'Output file {args.output_file} already exists') sys.exit(1) std_config = get_standard_config() if args.config_file is not None: config = replace_config(std_config, read_configuration_file(args.config_file)) else: config = std_config with tables.open_file(args.input_file, 'r') as f: is_simulation = 'simulation' in f.root increase_nsb = False increase_psf = False if "image_modifier" in config: imconfig = config["image_modifier"] increase_nsb = imconfig["increase_nsb"] increase_psf = imconfig["increase_psf"] if increase_nsb or increase_psf: log.info(f"image_modifier configuration: {imconfig}") extra_noise_in_dim_pixels = imconfig["extra_noise_in_dim_pixels"] extra_bias_in_dim_pixels = imconfig["extra_bias_in_dim_pixels"] transition_charge = imconfig["transition_charge"] extra_noise_in_bright_pixels = imconfig["extra_noise_in_bright_pixels"] smeared_light_fraction = imconfig["smeared_light_fraction"] if (increase_nsb or increase_psf): log.info( "NOTE: Using the image_modifier options means images will " "not be saved.") args.no_image = True if is_simulation: args.pedestal_cleaning = False if args.pedestal_cleaning: log.info("Pedestal cleaning") clean_method_name = 'tailcuts_clean_with_pedestal_threshold' sigma = config[clean_method_name]['sigma'] pedestal_thresh = get_threshold_from_dl1_file(args.input_file, sigma) cleaning_params = get_cleaning_parameters(config, clean_method_name) pic_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params log.info( f"Fraction of pixel cleaning thresholds above picture thr.:" f"{np.sum(pedestal_thresh > pic_th) / len(pedestal_thresh):.3f}") picture_th = np.clip(pedestal_thresh, pic_th, None) log.info(f"Tailcut clean with pedestal threshold config used:" f"{config['tailcuts_clean_with_pedestal_threshold']}") else: clean_method_name = 'tailcut' cleaning_params = get_cleaning_parameters(config, clean_method_name) picture_th, boundary_th, isolated_pixels, min_n_neighbors = cleaning_params log.info(f"Tailcut config used: {config['tailcut']}") use_dynamic_cleaning = False if 'apply' in config['dynamic_cleaning']: use_dynamic_cleaning = config['dynamic_cleaning']['apply'] if use_dynamic_cleaning: THRESHOLD_DYNAMIC_CLEANING = config['dynamic_cleaning']['threshold'] FRACTION_CLEANING_SIZE = config['dynamic_cleaning'][ 'fraction_cleaning_intensity'] log.info( "Using dynamic cleaning for events with average size of the " f"3 most brighest pixels > {config['dynamic_cleaning']['threshold']} p.e" ) log.info( "Remove from image pixels which have charge below " f"= {config['dynamic_cleaning']['fraction_cleaning_intensity']} * average size" ) use_only_main_island = True if "use_only_main_island" in config[clean_method_name]: use_only_main_island = config[clean_method_name][ "use_only_main_island"] delta_time = None if "delta_time" in config[clean_method_name]: delta_time = config[clean_method_name]["delta_time"] subarray_info = SubarrayDescription.from_hdf(args.input_file) tel_id = config["allowed_tels"][0] if "allowed_tels" in config else 1 optics = subarray_info.tel[tel_id].optics camera_geom = subarray_info.tel[tel_id].camera.geometry dl1_container = DL1ParametersContainer() parameters_to_update = [ 'intensity', 'x', 'y', 'r', 'phi', 'length', 'width', 'psi', 'skewness', 'kurtosis', 'concentration_cog', 'concentration_core', 'concentration_pixel', 'leakage_intensity_width_1', 'leakage_intensity_width_2', 'leakage_pixels_width_1', 'leakage_pixels_width_2', 'n_islands', 'intercept', 'time_gradient', 'n_pixels', 'wl', 'log_intensity' ] nodes_keys = get_dataset_keys(args.input_file) if args.no_image: nodes_keys.remove(dl1_images_lstcam_key) metadata = global_metadata() with tables.open_file(args.input_file, mode='r') as infile: image_table = infile.root[dl1_images_lstcam_key] dl1_params_input = infile.root[dl1_params_lstcam_key].colnames disp_params = { 'disp_dx', 'disp_dy', 'disp_norm', 'disp_angle', 'disp_sign' } if set(dl1_params_input).intersection(disp_params): parameters_to_update.extend(disp_params) uncertainty_params = {'width_uncertainty', 'length_uncertainty'} if set(dl1_params_input).intersection(uncertainty_params): parameters_to_update.extend(uncertainty_params) if increase_nsb: rng = np.random.default_rng( infile.root.dl1.event.subarray.trigger.col('obs_id')[0]) if increase_psf: set_numba_seed( infile.root.dl1.event.subarray.trigger.col('obs_id')[0]) image_mask_save = not args.no_image and 'image_mask' in infile.root[ dl1_images_lstcam_key].colnames with tables.open_file(args.output_file, mode='a', filters=HDF5_ZSTD_FILTERS) as outfile: copy_h5_nodes(infile, outfile, nodes=nodes_keys) add_source_filenames(outfile, [args.input_file]) params = outfile.root[dl1_params_lstcam_key].read() if image_mask_save: image_mask = outfile.root[dl1_images_lstcam_key].col( 'image_mask') # need container to use lstchain.io.add_global_metadata and lstchain.io.add_config_metadata for k, item in metadata.as_dict().items(): outfile.root[dl1_params_lstcam_key].attrs[k] = item outfile.root[dl1_params_lstcam_key].attrs["config"] = str(config) for ii, row in enumerate(image_table): dl1_container.reset() image = row['image'] peak_time = row['peak_time'] if increase_nsb: # Add noise in pixels, to adjust MC to data noise levels. # TO BE DONE: in case of "pedestal cleaning" (not used now # in MC) we should recalculate picture_th above! image = add_noise_in_pixels(rng, image, extra_noise_in_dim_pixels, extra_bias_in_dim_pixels, transition_charge, extra_noise_in_bright_pixels) if increase_psf: image = random_psf_smearer( image, smeared_light_fraction, camera_geom.neighbor_matrix_sparse.indices, camera_geom.neighbor_matrix_sparse.indptr) signal_pixels = tailcuts_clean(camera_geom, image, picture_th, boundary_th, isolated_pixels, min_n_neighbors) n_pixels = np.count_nonzero(signal_pixels) if n_pixels > 0: # if delta_time has been set, we require at least one # neighbor within delta_time to accept a pixel in the image: if delta_time is not None: cleaned_pixel_times = peak_time # makes sure only signal pixels are used in the time # check: cleaned_pixel_times[~signal_pixels] = np.nan new_mask = apply_time_delta_cleaning( camera_geom, signal_pixels, cleaned_pixel_times, 1, delta_time) signal_pixels = new_mask if use_dynamic_cleaning: new_mask = apply_dynamic_cleaning( image, signal_pixels, THRESHOLD_DYNAMIC_CLEANING, FRACTION_CLEANING_SIZE) signal_pixels = new_mask # count a number of islands after all of the image cleaning steps num_islands, island_labels = number_of_islands( camera_geom, signal_pixels) dl1_container.n_islands = num_islands n_pixels_on_island = np.bincount( island_labels.astype(np.int64)) n_pixels_on_island[ 0] = 0 # first island is no-island and should not be considered max_island_label = np.argmax(n_pixels_on_island) if use_only_main_island: signal_pixels[ island_labels != max_island_label] = False # count the surviving pixels n_pixels = np.count_nonzero(signal_pixels) dl1_container.n_pixels = n_pixels if n_pixels > 0: parametrize_image( image=image, peak_time=peak_time, signal_pixels=signal_pixels, camera_geometry=camera_geom, focal_length=optics.equivalent_focal_length, dl1_container=dl1_container, ) if set(dl1_params_input).intersection(disp_params): disp_dx, disp_dy, disp_norm, disp_angle, disp_sign = disp( dl1_container['x'].to_value(u.m), dl1_container['y'].to_value(u.m), params['src_x'][ii], params['src_y'][ii]) dl1_container['disp_dx'] = disp_dx dl1_container['disp_dy'] = disp_dy dl1_container['disp_norm'] = disp_norm dl1_container['disp_angle'] = disp_angle dl1_container['disp_sign'] = disp_sign for p in parameters_to_update: params[ii][p] = u.Quantity(dl1_container[p]).value if image_mask_save: image_mask[ii] = signal_pixels outfile.root[dl1_params_lstcam_key][:] = params if image_mask_save: outfile.root[dl1_images_lstcam_key].modify_column( colname='image_mask', column=image_mask) write_metadata(metadata, args.output_file)
def process_telescope(tel, dl1, stereo): geom = tel.camera.geometry camera_name = tel.camera.camera_name image = dl1.image peak_time = dl1.peak_time # Cleaning using CHEC method clean = obtain_cleaning_mask(geom, image, peak_time, camera_name) # Skipping inadequate events if clean.sum() == 0: return None, None, None if stereo and clean.sum() < 5: return None, None, None # Get hillas parameters hillas_c = hillas_parameters(geom[clean], image[clean]) if hillas_c.width == 0 or np.isnan(hillas_c.width.value): return None, None, None # Get leakage and islands leakage_c = leakage(geom, image, clean) n_islands, island_ids = number_of_islands(geom, clean) # Get time gradient tgrad = np.nan try: timing_c = timing_parameters(geom, image, peak_time, hillas_c, clean) tgrad = timing_c.slope.value except BaseException: print("Timing parameters didn't work. clean.sum() = " + str(clean.sum()), "\n") # Grab info for telescope_events tel_data_dict = { 'nislands': n_islands, 'telescope_type': tel.type, 'camera_type': tel.camera.camera_name, 'focal_length': tel.optics.equivalent_focal_length.value, 'n_survived_pixels': clean.sum(), 'tgradient': tgrad, 'x': hillas_c.x.value, 'y': hillas_c.y.value, 'r': hillas_c.r.value, 'phi': hillas_c.phi.value, 'intensity': hillas_c.intensity, 'length': hillas_c.length.value, 'width': hillas_c.width.value, 'psi': hillas_c.psi.value, 'skewness': hillas_c.skewness, 'kurtosis': hillas_c.kurtosis, 'pixels_width_1': leakage_c.pixels_width_1, 'pixels_width_2': leakage_c.pixels_width_2, 'intensity_width_1': leakage_c.intensity_width_1, 'intensity_width_2': leakage_c.intensity_width_2} return tel.type, tel_data_dict, hillas_c