def clean_image(self, img, output_data_dict=None, **kwargs): # Add (in the output dictionary) information about islands (pixels clusters) for the reference image if output_data_dict is not None: cleaned_image = np.copy(img) kill_islands = filter_pixels_clusters_stats(cleaned_image) img_cleaned_islands_delta_pe, img_cleaned_islands_delta_abs_pe, img_cleaned_islands_delta_num_pixels = kill_islands img_cleaned_num_islands = number_of_pixels_clusters(cleaned_image) output_data_dict["img_cleaned_islands_delta_pe"] = img_cleaned_islands_delta_pe output_data_dict["img_cleaned_islands_delta_abs_pe"] = img_cleaned_islands_delta_abs_pe output_data_dict["img_cleaned_islands_delta_num_pixels"] = img_cleaned_islands_delta_num_pixels output_data_dict["img_cleaned_num_islands"] = img_cleaned_num_islands return copy.deepcopy(img)
def clean_image(self, input_image, type_of_filtering=hard_filter.DEFAULT_TYPE_OF_FILTERING, filter_thresholds=hard_filter.DEFAULT_FILTER_THRESHOLDS, last_scale_treatment=starlet.DEFAULT_LAST_SCALE_TREATMENT, detect_only_positive_structures=False, kill_isolated_pixels=False, noise_distribution=None, output_data_dict=None, clusters_threshold=0, **kwargs): """Clean the `input_image` image. Apply the wavelet transform, filter planes and return the reverse transformed image. Parameters ---------- input_image : array_like The image to clean. type_of_filtering : str Type of filtering: 'hard_filtering' or 'ksigma_hard_filtering'. filter_thresholds : list of float Thresholds used for the plane filtering. last_scale_treatment : str Last plane treatment: 'keep', 'drop' or 'mask'. detect_only_positive_structures : bool Detect only positive structures. kill_isolated_pixels : bool Suppress isolated pixels in the support. noise_distribution : bool The JSON file containing the Cumulated Distribution Function of the noise model used to inject artificial noise in blank pixels (those with a NaN value). output_data_dict : dict A dictionary used to return results and intermediate results. clusters_threshold : float The threshold used to define clusters of pixels (the "level of the sea") Returns ------- Return the cleaned image. """ if DEBUG: print("Filter thresholds:", filter_thresholds) number_of_scales = len(filter_thresholds) + 1 if not (1 < number_of_scales <= 10): # This range ]1,10] is a hard constraint from the starlet transform raise ValueError( "bad number of scales: {}. Should be 1 < Nbr Scales <= 10. Check that filter_thresholds is a list of number and not a string." .format(number_of_scales)) if DEBUG: print("Number of scales:", number_of_scales) # COMPUTE THE WAVELET TRANSFORM ####################################### wavelet_planes = wavelet_transform( input_image, number_of_scales=number_of_scales, noise_distribution=noise_distribution) if DEBUG: for index, plane in enumerate(wavelet_planes): images.plot(plane, "Plane " + str(index)) # FILTER WAVELET PLANES ############################################### filtered_wavelet_planes = filter_planes( wavelet_planes, method=type_of_filtering, thresholds=filter_thresholds, detect_only_positive_structures=detect_only_positive_structures) #if DEBUG: # for index, plane in enumerate(filtered_wavelet_planes): # images.plot(plane, "Filtered plane " + str(index)) # COMPUTE THE INVERSE TRANSFORM ####################################### cleaned_image = inverse_wavelet_transform( filtered_wavelet_planes, last_plane=last_scale_treatment) if DEBUG: images.plot(cleaned_image, "Cleaned image") # KILL ISOLATED PIXELS ################################################ kill_islands = filter_pixels_clusters_stats(cleaned_image) img_cleaned_islands_delta_pe, img_cleaned_islands_delta_abs_pe, img_cleaned_islands_delta_num_pixels = kill_islands img_cleaned_num_islands = number_of_pixels_clusters(cleaned_image) if output_data_dict is not None: output_data_dict[ "img_cleaned_islands_delta_pe"] = img_cleaned_islands_delta_pe output_data_dict[ "img_cleaned_islands_delta_abs_pe"] = img_cleaned_islands_delta_abs_pe output_data_dict[ "img_cleaned_islands_delta_num_pixels"] = img_cleaned_islands_delta_num_pixels output_data_dict[ "img_cleaned_num_islands"] = img_cleaned_num_islands if kill_isolated_pixels: cleaned_image = scipy_kill_isolated_pixels( cleaned_image, threshold=clusters_threshold) if DEBUG: images.plot(cleaned_image, "Cleaned image after island kill") return cleaned_image
def run( self, cleaning_function_params, input_file_or_dir_path_list, benchmark_method, output_file_path, plot=False, saveplot=None, ref_img_as_input=False, # A hack to easily produce CSV files... max_num_img=None, debug=False): """A convenient optional wrapper to simplify the image cleaning analysis. Apply the image cleaning analysis on `input_file_or_dir_path_list`, apply some pre-processing and post-processing procedures, collect and return results, intermediate values and metadata. Parameters ---------- cleaning_function_params A dictionary containing the parameters required for the image cleaning method. input_file_or_dir_path_list A list of file to clean. Can be a list of simtel files, fits files or directories containing such files. benchmark_method The list of estimators to use to assess the image cleaning. If `None`, images are cleaned but nothing is returned (can be used with e.g. the `plot` and/or `saveplot` options). output_file_path The result file path (a JSON file). plot The result of each cleaning is plot if `True`. saveplot The result of each cleaning is saved if `True`. ref_img_as_input This option is a hack to easily produce a "flatten" CSV results files. max_num_img The number of images to process among the input set (`input_file_or_dir_path_list`). debug Stop the execution and print the full traceback when an exception is encountered if this parameter is `True`. Report exceptions and continue with the next input image if this parameter is `False`. Returns ------- dict Results, intermediate values and metadata. """ launch_time = time.perf_counter() if benchmark_method is not None: io_list = [] # The list of returned dictionaries for image in image_generator(input_file_or_dir_path_list, max_num_images=max_num_img): input_file_path = image.meta['file_path'] if self.verbose: print(input_file_path) # `image_dict` contains metadata (to be returned) on the current image image_dict = {"input_file_path": input_file_path} try: # READ THE INPUT FILE ##################################### reference_img = image.reference_image pixels_position = image.pixels_position if ref_img_as_input: # This option is a hack to easily produce CSV files with # the "null_ref" "cleaning" module... input_img = copy.deepcopy(reference_img) else: input_img = image.input_image image_dict.update(image.meta) if benchmark_method is not None: # FETCH ADDITIONAL IMAGE METADATA ##################### delta_pe, delta_abs_pe, delta_num_pixels = filter_pixels_clusters_stats( reference_img) # TODO: NaN num_islands = number_of_pixels_clusters( reference_img) # TODO: NaN image_dict["img_ref_islands_delta_pe"] = delta_pe image_dict["img_ref_islands_delta_abs_pe"] = delta_abs_pe image_dict[ "img_ref_islands_delta_num_pixels"] = delta_num_pixels image_dict["img_ref_num_islands"] = num_islands image_dict["img_ref_sum_pe"] = float( np.nansum(reference_img)) image_dict["img_ref_min_pe"] = float( np.nanmin(reference_img)) image_dict["img_ref_max_pe"] = float( np.nanmax(reference_img)) image_dict["img_ref_num_pix"] = int( (reference_img[np.isfinite(reference_img)] > 0).sum()) image_dict["img_in_sum_pe"] = float(np.nansum(input_img)) image_dict["img_in_min_pe"] = float(np.nanmin(input_img)) image_dict["img_in_max_pe"] = float(np.nanmax(input_img)) image_dict["img_in_num_pix"] = int( (input_img[np.isfinite(input_img)] > 0).sum()) # CLEAN THE INPUT IMAGE ################################### # Copy the image (otherwise some cleaning functions like Tailcut may change it) #input_img_copy = copy.deepcopy(input_img) input_img_copy = input_img.astype('float64', copy=True) cleaning_function_params["output_data_dict"] = {} initial_time = time.perf_counter() cleaned_img = self.clean_image( input_img_copy, **cleaning_function_params) # TODO: NaN full_clean_execution_time_sec = time.perf_counter( ) - initial_time if benchmark_method is not None: image_dict.update( cleaning_function_params["output_data_dict"]) del cleaning_function_params["output_data_dict"] # ASSESS OR PRINT THE CLEANED IMAGE ####################### if benchmark_method is not None: # ASSESS THE CLEANING ################################# kwargs = {} # TODO GEOM score = mse(cleaned_img, reference_img) image_dict["score"] = [score] image_dict["score_name"] = ["mse"] image_dict[ "full_clean_execution_time_sec"] = full_clean_execution_time_sec image_dict["img_cleaned_sum_pe"] = float( np.nansum(cleaned_img)) image_dict["img_cleaned_min_pe"] = float( np.nanmin(cleaned_img)) image_dict["img_cleaned_max_pe"] = float( np.nanmax(cleaned_img)) image_dict["img_cleaned_num_pix"] = int( (cleaned_img[np.isfinite(cleaned_img)] > 0).sum()) # PLOT IMAGES ######################################################### if plot or (saveplot is not None): image_list = [input_img, reference_img, cleaned_img] title_list = [ "Input image", "Reference image", "Cleaned image" ] if plot: pywi.io.images.plot_list(image_list, title_list=title_list, metadata_dict=image.meta) if saveplot is not None: plot_file_path = saveplot print("Saving {}".format(plot_file_path)) pywi.io.images.mpl_save_list( image_list, output_file_path=plot_file_path, title_list=title_list, metadata_dict=image.meta) except Exception as e: print("Abort image {}: {} ({})".format(input_file_path, e, type(e))) if debug: # The following line print the full trackback traceback.print_tb(e.__traceback__, file=sys.stdout) if benchmark_method is not None: # http://docs.python.org/2/library/sys.html#sys.exc_info exc_type, exc_value, exc_traceback = sys.exc_info( ) # most recent (if any) by default ''' Reason this _can_ be bad: If an (unhandled) exception happens AFTER this, or if we do not delete the labels on (not much) older versions of Py, the reference we created can linger. traceback.format_exc/print_exc do this very thing, BUT note this creates a temp scope within the function. ''' error_dict = { 'filename': exc_traceback.tb_frame.f_code.co_filename, 'lineno': exc_traceback.tb_lineno, 'name': exc_traceback.tb_frame.f_code.co_name, 'type': exc_type.__name__, #'message' : exc_value.message 'message': str(e) } del ( exc_type, exc_value, exc_traceback ) # So we don't leave our local labels/objects dangling # This still isn't "completely safe", though! #error_dict = {"type": str(type(e)), # "message": str(e)} image_dict["error"] = error_dict finally: if benchmark_method is not None: io_list.append(image_dict) if benchmark_method is not None: error_list = [ image_dict["error"] for image_dict in io_list if "error" in image_dict ] print("{} images aborted".format(len(error_list))) # GENERAL EXPERIMENT METADATA output_dict = {} output_dict["benchmark_execution_time_sec"] = str( time.perf_counter() - launch_time) output_dict["date_time"] = str(datetime.datetime.now()) output_dict["class_name"] = self.__class__.__name__ output_dict["algo_code_ref"] = str( self.__class__.clean_image.__code__) output_dict["label"] = self.label output_dict["cmd"] = " ".join(sys.argv) output_dict["algo_params"] = cleaning_function_params if "noise_distribution" in output_dict["algo_params"]: del output_dict["algo_params"][ "noise_distribution"] # not JSON serializable... output_dict["benchmark_method"] = benchmark_method output_dict["system"] = " ".join(os.uname()) output_dict["io"] = io_list with open(output_file_path, "w") as fd: json.dump(output_dict, fd, sort_keys=True, indent=4) # pretty print format return output_dict
def prepare_event(self, source, return_stub=True, save_images=False, debug=False): """ Calibrate, clean and reconstruct the direction of an event. Parameters ---------- source : ctapipe.io.EventSource A container of selected showers from a simtel file. geom_cam_tel: dict Dictionary of of MyCameraGeometry objects for each camera in the file return_stub : bool If True, yield also images from events that won't be reconstructed. This feature is not currently available. save_images : bool If True, save photoelectron images from reconstructed events. debug : bool If True, print some debugging information (to be expanded). Yields ------ PreparedEvent: dict Dictionary containing event-image information to be written. """ # ============================================================= # TRANSFORMED CAMERA GEOMETRIES # ============================================================= # These are the camera geometries were the Hillas parametrization will # be performed. # They are transformed to TelescopeFrame using the effective focal # lengths # These geometries could be used also to performe the image cleaning, # but for the moment we still do that in the CameraFrame geom_cam_tel = {} for camera in source.subarray.camera_types: # Original geometry of each camera geom = camera.geometry # Same but with focal length as an attribute # This is planned to disappear and be imported by ctapipe focal_length = effective_focal_lengths(camera.camera_name) geom_cam_tel[camera.camera_name] = MyCameraGeometry( camera_name=camera.camera_name, pix_type=geom.pix_type, pix_id=geom.pix_id, pix_x=geom.pix_x, pix_y=geom.pix_y, pix_area=geom.pix_area, cam_rotation=geom.cam_rotation, pix_rotation=geom.pix_rotation, frame=CameraFrame(focal_length=focal_length), ).transform_to(TelescopeFrame()) # ============================================================= ievt = 0 for event in source: # Display event counts if debug: print( bcolors.BOLD + f"EVENT #{event.count}, EVENT_ID #{event.index.event_id}" + bcolors.ENDC) print(bcolors.BOLD + f"has triggered telescopes {event.r1.tel.keys()}" + bcolors.ENDC) ievt += 1 # if (ievt < 10) or (ievt % 10 == 0): # print(ievt) self.event_cutflow.count("noCuts") # LST stereo condition # whenever there is only 1 LST in an event, we remove that telescope # if the remaining telescopes are less than min_tel we remove the event lst_tel_ids = set( source.subarray.get_tel_ids_for_type("LST_LST_LSTCam")) triggered_LSTs = set(event.r0.tel.keys()).intersection(lst_tel_ids) n_triggered_LSTs = len(triggered_LSTs) n_triggered_non_LSTs = len(event.r0.tel.keys()) - n_triggered_LSTs bad_LST_stereo = False if self.LST_stereo and self.event_cutflow.cut( "no-LST-stereo + <2 other types", n_triggered_LSTs, n_triggered_non_LSTs): bad_LST_stereo = True if return_stub: print( bcolors.WARNING + "WARNING: LST_stereo is set to 'True'\n" + f"This event has < {self.min_ntel_LST} triggered LSTs\n" + "and < 2 triggered telescopes from other telescope types.\n" + "The event will be processed up to DL1b." + bcolors.ENDC) # we show this, but we proceed to analyze the event up to # DL1a/b for the associated benchmarks # this checks for < 2 triggered telescopes of ANY type if self.event_cutflow.cut("min2Tels trig", len(event.r1.tel.keys())): if return_stub: print( bcolors.WARNING + f"WARNING : < {self.min_ntel} triggered telescopes!" + bcolors.ENDC) # we show this, but we proceed to analyze it # ============================================================= # CALIBRATION # ============================================================= if debug: print(bcolors.OKBLUE + "Extracting all calibrated images..." + bcolors.ENDC) self.calib(event) # Calibrate the event # ============================================================= # BEGINNING OF LOOP OVER TELESCOPES # ============================================================= dl1_phe_image = {} dl1_phe_image_mask_reco = {} dl1_phe_image_mask_clusters = {} mc_phe_image = {} max_signals = {} n_pixel_dict = {} hillas_dict_reco = {} # for direction reconstruction hillas_dict = {} # for discrimination leakage_dict = {} concentration_dict = {} n_tels = { "Triggered": len(event.r1.tel.keys()), "LST_LST_LSTCam": 0, "MST_MST_NectarCam": 0, "MST_MST_FlashCam": 0, "MST_SCT_SCTCam": 0, "SST_1M_DigiCam": 0, "SST_ASTRI_ASTRICam": 0, "SST_GCT_CHEC": 0, } n_cluster_dict = {} impact_dict_reco = {} # impact distance measured in tilt system point_azimuth_dict = {} point_altitude_dict = {} good_for_reco = {} # 1 = success, 0 = fail # Array pointing in AltAz frame az = event.pointing.array_azimuth alt = event.pointing.array_altitude array_pointing = SkyCoord(az, alt, frame=AltAz()) ground_frame = GroundFrame() tilted_frame = TiltedGroundFrame(pointing_direction=array_pointing) for tel_id in event.r1.tel.keys(): point_azimuth_dict[tel_id] = event.pointing.tel[tel_id].azimuth point_altitude_dict[tel_id] = event.pointing.tel[ tel_id].altitude if debug: print(bcolors.OKBLUE + f"Working on telescope #{tel_id}..." + bcolors.ENDC) self.image_cutflow.count("noCuts") camera = source.subarray.tel[tel_id].camera.geometry # count the current telescope according to its type tel_type = str(source.subarray.tel[tel_id]) n_tels[tel_type] += 1 # use ctapipe's functionality to get the calibrated image # and scale the reconstructed values if required pmt_signal = event.dl1.tel[tel_id].image / self.calibscale # If required... if save_images is True: # Save the simulated and reconstructed image of the event dl1_phe_image[tel_id] = pmt_signal mc_phe_image[tel_id] = event.simulation.tel[ tel_id].true_image # We now ASSUME that the event will be good good_for_reco[tel_id] = 1 # later we change to 0 if any condition is NOT satisfied if self.cleaner_reco.mode == "tail": # tail uses only ctapipe # Cleaning used for direction reconstruction image_biggest, mask_reco = self.cleaner_reco.clean_image( pmt_signal, camera) # calculate the leakage (before filtering) leakages = {} # this is needed by both cleanings # The check on SIZE shouldn't be here, but for the moment # I prefer to sacrifice elegancy... if np.sum(image_biggest[mask_reco]) != 0.0: leakage_biggest = leakage_parameters( camera, image_biggest, mask_reco) leakages["leak1_reco"] = leakage_biggest[ "intensity_width_1"] leakages["leak2_reco"] = leakage_biggest[ "intensity_width_2"] else: leakages["leak1_reco"] = 0.0 leakages["leak2_reco"] = 0.0 # find all islands using this cleaning num_islands, labels = number_of_islands(camera, mask_reco) if num_islands == 1: # if only ONE islands is left ... # ...use directly the old mask and reduce dimensions # to make Hillas parametrization faster camera_biggest = camera[mask_reco] image_biggest = image_biggest[mask_reco] if save_images is True: dl1_phe_image_mask_reco[tel_id] = mask_reco elif num_islands > 1: # if more islands survived.. # ...find the biggest one mask_biggest = largest_island(labels) # and also reduce dimensions camera_biggest = camera[mask_biggest] image_biggest = image_biggest[mask_biggest] if save_images is True: dl1_phe_image_mask_reco[tel_id] = mask_biggest else: # if no islands survived use old camera and image camera_biggest = camera dl1_phe_image_mask_reco[tel_id] = mask_reco # Cleaning used for score/energy estimation image_extended, mask_extended = self.cleaner_extended.clean_image( pmt_signal, camera) dl1_phe_image_mask_clusters[tel_id] = mask_extended # calculate the leakage (before filtering) # this part is not well coded, but for the moment it works if np.sum(image_extended[mask_extended]) != 0.0: leakage_extended = leakage_parameters( camera, image_extended, mask_extended) leakages["leak1"] = leakage_extended[ "intensity_width_1"] leakages["leak2"] = leakage_extended[ "intensity_width_2"] else: leakages["leak1"] = 0.0 leakages["leak2"] = 0.0 # find all islands with this cleaning # we will also register how many have been found n_cluster_dict[tel_id], labels = number_of_islands( camera, mask_extended) # NOTE: the next check shouldn't be necessary if we keep # all the isolated pixel clusters, but for now the # two cleanings are set the same in analysis.yml because # the performance of the extended one has never been really # studied in model estimation. # if some islands survived if n_cluster_dict[tel_id] > 0: # keep all of them and reduce dimensions camera_extended = camera[mask_extended] image_extended = image_extended[mask_extended] else: # otherwise continue with the old camera and image camera_extended = camera # could this go into `hillas_parameters` ...? # this is basically the charge of ALL islands # not calculated later by the Hillas parametrization! max_signals[tel_id] = np.max(image_extended) else: # for wavelets we stick to old pywi-cta code try: # "try except FileNotFoundError" not clear to me, but for now it stays... with warnings.catch_warnings(): # Image with biggest cluster (reco cleaning) image_biggest, mask_reco = self.cleaner_reco.clean_image( pmt_signal, camera) image_biggest2d = geometry_converter.image_1d_to_2d( image_biggest, camera.camera_name) image_biggest2d = filter_pixels_clusters( image_biggest2d) image_biggest = geometry_converter.image_2d_to_1d( image_biggest2d, camera.camera_name) # Image for score/energy estimation (with clusters) ( image_extended, mask_extended, ) = self.cleaner_extended.clean_image( pmt_signal, camera) # This last part was outside the pywi-cta block # before, but is indeed part of it because it uses # pywi-cta functions in the "extended" case # For cluster counts image_2d = geometry_converter.image_1d_to_2d( image_extended, camera.camera_name) n_cluster_dict[ tel_id] = pixel_clusters.number_of_pixels_clusters( array=image_2d, threshold=0) # could this go into `hillas_parameters` ...? max_signals[tel_id] = np.max(image_extended) except FileNotFoundError as e: print(e) continue # ============================================================= # PRELIMINARY IMAGE SELECTION # ============================================================= cleaned_image_is_good = True # we assume this if self.image_selection_source == "extended": cleaned_image_to_use = image_extended elif self.image_selection_source == "biggest": cleaned_image_to_use = image_biggest else: raise ValueError( "Only supported cleanings are 'biggest' or 'extended'." ) # Apply some selection if self.image_cutflow.cut("min pixel", cleaned_image_to_use): if debug: print(bcolors.WARNING + "WARNING : not enough pixels!" + bcolors.ENDC) good_for_reco[tel_id] = 0 # we record it as BAD cleaned_image_is_good = False if self.image_cutflow.cut("min charge", np.sum(cleaned_image_to_use)): if debug: print(bcolors.WARNING + "WARNING : not enough charge!" + bcolors.ENDC) good_for_reco[tel_id] = 0 # we record it as BAD cleaned_image_is_good = False if debug and (not cleaned_image_is_good): # BAD image quality print(bcolors.WARNING + "WARNING : The cleaned image didn't pass" + " preliminary cuts.\n" + "An attempt to parametrize it will be made," + " but the image will NOT be used for" + " direction reconstruction." + bcolors.ENDC) # ============================================================= # IMAGE PARAMETRIZATION # ============================================================= with np.errstate(invalid="raise", divide="raise"): try: # Filter the cameras in TelescopeFrame with the same # cleaning masks camera_biggest_tel = geom_cam_tel[camera.camera_name][ camera_biggest.pix_id] camera_extended_tel = geom_cam_tel[camera.camera_name][ camera_extended.pix_id] # Parametrize the image in the TelescopeFrame moments_reco = hillas_parameters( camera_biggest_tel, image_biggest) # for geometry (eg direction) moments = hillas_parameters( camera_extended_tel, image_extended ) # for discrimination and energy reconstruction if debug: print( "Image parameters from main cluster cleaning:") print(moments_reco) print( "Image parameters from all-clusters cleaning:") print(moments) # Add concentration parameters concentrations = {} concentrations_extended = concentration_parameters( camera_extended_tel, image_extended, moments) concentrations[ "concentration_cog"] = concentrations_extended[ "cog"] concentrations[ "concentration_core"] = concentrations_extended[ "core"] concentrations[ "concentration_pixel"] = concentrations_extended[ "pixel"] # =================================================== # PARAMETRIZED IMAGE SELECTION # =================================================== if self.image_selection_source == "extended": moments_to_use = moments else: moments_to_use = moments_reco # if width and/or length are zero (e.g. when there is # only only one pixel or when all pixel are exactly # in one row), the parametrisation # won't be very useful: skip if self.image_cutflow.cut("poor moments", moments_to_use): if debug: print(bcolors.WARNING + "WARNING : poor moments!" + bcolors.ENDC) good_for_reco[tel_id] = 0 # we record it as BAD if self.image_cutflow.cut("close to the edge", moments_to_use, camera.camera_name): if debug: print( bcolors.WARNING + "WARNING : out of containment radius!\n" + f"Camera radius = {self.camera_radius[camera.camera_name]}\n" + f"COG radius = {moments_to_use.r}" + bcolors.ENDC) good_for_reco[tel_id] = 0 if self.image_cutflow.cut("bad ellipticity", moments_to_use): if debug: print(bcolors.WARNING + "WARNING : bad ellipticity" + bcolors.ENDC) good_for_reco[tel_id] = 0 if debug and good_for_reco[tel_id] == 1: print( bcolors.OKGREEN + "Image survived and correctly parametrized." # + "\nIt will be used for direction reconstruction!" + bcolors.ENDC) elif debug and good_for_reco[tel_id] == 0: print( bcolors.WARNING + "Image not survived or " + "not good enough for parametrization." # + "\nIt will be NOT used for direction reconstruction, " # + "BUT it's information will be recorded." + bcolors.ENDC) hillas_dict[tel_id] = moments hillas_dict_reco[tel_id] = moments_reco n_pixel_dict[tel_id] = len( np.where(image_extended > 0)[0]) leakage_dict[tel_id] = leakages concentration_dict[tel_id] = concentrations except ( FloatingPointError, HillasParameterizationError, ValueError, ) as e: if debug: print(bcolors.FAIL + "Parametrization error: " + f"{e}\n" + "Dummy parameters recorded." + bcolors.ENDC) good_for_reco[tel_id] = 0 hillas_dict[ tel_id] = HillasParametersTelescopeFrameContainer( ) hillas_dict_reco[ tel_id] = HillasParametersTelescopeFrameContainer( ) n_pixel_dict[tel_id] = len( np.where(image_extended > 0)[0]) leakage_dict[tel_id] = leakages concentration_dict[tel_id] = concentrations # END OF THE CYCLE OVER THE TELESCOPES # ============================================================= # DIRECTION RECONSTRUCTION # ============================================================= if bad_LST_stereo: if debug: print( bcolors.WARNING + "WARNING: This event was triggered with 1 LST image and <2 images from other telescope types." + "\nWARNING : direction reconstruction will not be performed." + bcolors.ENDC) # Set all the involved images as NOT good for recosntruction # even though they might have been # but this is because of the LST stereo trigger.... for tel_id in event.r0.tel.keys(): good_for_reco[tel_id] = 0 # and set the number of good and bad images accordingly n_tels["GOOD images"] = 0 n_tels[ "BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"] # create a dummy container for direction reconstruction reco_result = ReconstructedShowerContainer() if return_stub: # if saving all events (default) if debug: print(bcolors.OKBLUE + "Recording event..." + bcolors.ENDC) print( bcolors.WARNING + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, concentration_dict) continue else: continue # Now in case the only triggered telescopes were # - < self.min_ntel_LST LST, # - >=2 any other telescope type, # we remove the single-LST image and continue reconstruction with # the images from the other telescope types if self.LST_stereo and (n_triggered_LSTs < self.min_ntel_LST) and ( n_triggered_LSTs != 0) and (n_triggered_non_LSTs >= 2): if debug: print( bcolors.WARNING + f"WARNING: LST stereo trigger condition is active.\n" + f"This event triggered < {self.min_ntel_LST} LSTs " + f"and {n_triggered_non_LSTs} images from other telescope types.\n" + bcolors.ENDC) for tel_id in triggered_LSTs: # in case we test for min_ntel_LST>2 if good_for_reco[tel_id]: # we don't use it for reconstruction good_for_reco[tel_id] = 0 print( bcolors.WARNING + f"WARNING: LST image #{tel_id} removed, even though it passed quality cuts." + bcolors.ENDC) # TODO: book-keeping of this kind of events doesn't seem easy # convert dictionary in numpy array to get a "mask" images_status = np.asarray(list(good_for_reco.values())) # record how many images will be used for reconstruction n_tels["GOOD images"] = len( np.extract(images_status == 1, images_status)) n_tels["BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"] if self.event_cutflow.cut("min2Tels reco", n_tels["GOOD images"]): if debug: print( bcolors.FAIL + f"WARNING: < {self.min_ntel} ({n_tels['GOOD images']}) images remaining!" + "\nWARNING : direction reconstruction is not possible!" + bcolors.ENDC) # create a dummy container for direction reconstruction reco_result = ReconstructedShowerContainer() if return_stub: # if saving all events (default) if debug: print(bcolors.OKBLUE + "Recording event..." + bcolors.ENDC) print( bcolors.WARNING + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, concentration_dict) continue else: continue if debug: print(bcolors.OKBLUE + "Starting direction reconstruction..." + bcolors.ENDC) try: with warnings.catch_warnings(): warnings.simplefilter("ignore") if self.image_selection_source == "extended": hillas_dict_to_use = hillas_dict else: hillas_dict_to_use = hillas_dict_reco # use only the successfully parametrized images # to reconstruct the direction of this event successfull_hillas = np.where(images_status == 1)[0] all_images = np.asarray(list(good_for_reco.keys())) good_images = set(all_images[successfull_hillas]) good_hillas_dict = { k: v for k, v in hillas_dict_to_use.items() if k in good_images } if debug: print( bcolors.PURPLE + f"{len(good_hillas_dict)} images " + f"(from telescopes #{list(good_hillas_dict.keys())}) will be " + "used to recover the shower's direction..." + bcolors.ENDC) # Reconstruction results reco_result = self.shower_reco.predict( good_hillas_dict, source.subarray, SkyCoord(alt=alt, az=az, frame="altaz"), None, # use the array direction ) # Impact parameter for telescope-wise energy estimation subarray = source.subarray for tel_id in hillas_dict_to_use.keys(): pos = subarray.positions[tel_id] tel_ground = SkyCoord(pos[0], pos[1], pos[2], frame=ground_frame) core_ground = SkyCoord( reco_result.core_x, reco_result.core_y, 0 * u.m, frame=ground_frame, ) # Go back to the tilted frame # this should be the same... tel_tilted = tel_ground.transform_to(tilted_frame) # but this not core_tilted = SkyCoord(x=core_ground.x, y=core_ground.y, frame=tilted_frame) impact_dict_reco[tel_id] = np.sqrt( (core_tilted.x - tel_tilted.x)**2 + (core_tilted.y - tel_tilted.y)**2) except (Exception, TooFewTelescopesException, InvalidWidthException) as e: if debug: print("exception in reconstruction:", e) raise if return_stub: if debug: print( bcolors.FAIL + "Shower could NOT be correctly reconstructed! " + "Recording event..." + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, concentration_dict) else: continue if self.event_cutflow.cut("direction nan", reco_result): if debug: print(bcolors.WARNING + "WARNING: undefined direction!" + bcolors.ENDC) if return_stub: if debug: print( bcolors.FAIL + "Shower could NOT be correctly reconstructed! " + "Recording event..." + "WARNING: This is event shall NOT be used further along the pipeline." + bcolors.ENDC) yield stub( # record event with dummy info event, mc_phe_image, dl1_phe_image, dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters, good_for_reco, hillas_dict, hillas_dict_reco, n_tels, leakage_dict, concentration_dict) else: continue if debug: print(bcolors.BOLDGREEN + "Shower correctly reconstructed! " + "Recording event..." + bcolors.ENDC) yield PreparedEvent( event=event, dl1_phe_image=dl1_phe_image, dl1_phe_image_mask_reco=dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters=dl1_phe_image_mask_clusters, mc_phe_image=mc_phe_image, n_pixel_dict=n_pixel_dict, hillas_dict=hillas_dict, hillas_dict_reco=hillas_dict_reco, leakage_dict=leakage_dict, concentration_dict=concentration_dict, n_tels=n_tels, max_signals=max_signals, n_cluster_dict=n_cluster_dict, reco_result=reco_result, impact_dict=impact_dict_reco, good_event=True, good_for_reco=good_for_reco, )
integration_correction=integration_correction) image = next(it) input_image = image.input_image # ## Plot the image example # In[5]: geom1d = geometry_converter.get_geom1d(cam_id) plot_ctapipe_image(input_image, geom=geom1d, plot_axis=False) # ## Plot the image example after thresholding # In[6]: THRESHOLD = 6 filtered_image = np.copy(input_image) filtered_image[filtered_image < THRESHOLD] = 0 plot_ctapipe_image(filtered_image, geom=geom1d, plot_axis=False) # ## Get the number of clusters # In[7]: input_image_2d = geometry_converter.image_1d_to_2d(input_image, cam_id) num_clusters = pixel_clusters.number_of_pixels_clusters(array=input_image_2d, threshold=THRESHOLD) print("Number of clusters:", num_clusters)
def clean_image( input_image, type_of_filtering=hard_filter.DEFAULT_TYPE_OF_FILTERING, filter_thresholds=hard_filter.DEFAULT_FILTER_THRESHOLDS, last_scale_treatment=mrtransform_wrapper.DEFAULT_LAST_SCALE_TREATMENT, detect_only_positive_structures=False, kill_isolated_pixels=False, noise_distribution=None, tmp_files_directory=".", output_data_dict=None, **kwargs): """Clean the `input_image` image. Apply the wavelet transform, filter planes and return the reverse transformed image. Parameters ---------- input_image : array_like The image to clean. type_of_filtering : str Type of filtering: 'hard_filtering' or 'ksigma_hard_filtering'. filter_thresholds : list of float Thresholds used for the plane filtering. last_scale_treatment : str Last plane treatment: 'keep', 'drop' or 'mask'. detect_only_positive_structures : bool Detect only positive structures. kill_isolated_pixels : bool Suppress isolated pixels in the support. noise_distribution : bool The JSON file containing the Cumulated Distribution Function of the noise model used to inject artificial noise in blank pixels (those with a NaN value). tmp_files_directory : str The path of the directory where temporary files are written. output_data_dict : dict A dictionary used to return results and intermediate results. Returns ------- Return the cleaned image. """ if DEBUG: print("Filter thresholds:", filter_thresholds) number_of_scales = len(filter_thresholds) + 1 if DEBUG: print("Number of scales:", number_of_scales) # COMPUTE THE WAVELET TRANSFORM ####################################### wavelet_planes = wavelet_transform(input_image, number_of_scales=number_of_scales, tmp_files_directory=tmp_files_directory, noise_distribution=noise_distribution) if DEBUG: for index, plane in enumerate(wavelet_planes): images.plot(plane, "Plane " + str(index)) # FILTER WAVELET PLANES ############################################### filtered_wavelet_planes = filter_planes( wavelet_planes, method=type_of_filtering, thresholds=filter_thresholds, detect_only_positive_structures=detect_only_positive_structures) #if DEBUG: # for index, plane in enumerate(filtered_wavelet_planes): # images.plot(plane, "Filtered plane " + str(index)) # COMPUTE THE INVERSE TRANSFORM ####################################### cleaned_image = inverse_wavelet_transform(filtered_wavelet_planes, last_plane=last_scale_treatment) if DEBUG: images.plot(cleaned_image, "Cleaned image") # REMOVE ISOLATED PIXELS ############################################## if output_data_dict is not None: img_cleaned_clusters_delta_intensity, img_cleaned_clusters_delta_abs_intensity, img_cleaned_clusters_delta_num_pixels = filter_pixels_clusters_stats( cleaned_image) img_cleaned_num_islands = number_of_pixels_clusters(cleaned_image) output_data_dict[ "img_cleaned_clusters_delta_intensity"] = img_cleaned_clusters_delta_intensity output_data_dict[ "img_cleaned_clusters_delta_abs_intensity"] = img_cleaned_clusters_delta_abs_intensity output_data_dict[ "img_cleaned_clusters_delta_num_pixels"] = img_cleaned_clusters_delta_num_pixels output_data_dict["img_cleaned_num_islands"] = img_cleaned_num_islands if kill_isolated_pixels: cleaned_image = filter_pixels_clusters(cleaned_image) if DEBUG: images.plot(cleaned_image, "Cleaned image after cluster filtering") return cleaned_image
def run( self, cleaning_function_params, input_file_or_dir_path_list, benchmark_method, output_file_path, plot=False, saveplot=None, ref_img_as_input=False, # A hack to easily produce CSV files... max_num_img=None, tel_id=None, event_id=None, cam_id=None, debug=False, rejection_criteria=None): """A convenient optional wrapper to simplify the image cleaning analysis. Apply the image cleaning analysis on `input_file_or_dir_path_list`, apply some pre-processing and post-processing procedures, collect and return results, intermediate values and metadata. Parameters ---------- cleaning_function_params A dictionary containing the parameters required for the image cleaning method. input_file_or_dir_path_list A list of file to clean. Can be a list of simtel files, fits files or directories containing such files. benchmark_method The list of estimators to use to assess the image cleaning. If `None`, images are cleaned but nothing is returned (can be used with e.g. the `plot` and/or `saveplot` options). output_file_path The result file path (a JSON file). plot The result of each cleaning is plot if `True`. saveplot The result of each cleaning is saved if `True`. ref_img_as_input This option is a hack to easily produce a "flatten" CSV results files. max_num_img The number of images to process among the input set (`input_file_or_dir_path_list`). debug Stop the execution and print the full traceback when an exception is encountered if this parameter is `True`. Report exceptions and continue with the next input image if this parameter is `False`. Returns ------- dict Results, intermediate values and metadata. """ launch_time = time.perf_counter() if benchmark_method is not None: io_list = [] # The list of returned dictionaries if tel_id is not None: tel_id = [tel_id] if event_id is not None: event_id = [event_id] if cam_id is None: raise ValueError('cam_id is now mandatory.') if cam_id == "ASTRICam": integrator_window_width = 1 integrator_window_shift = 1 elif cam_id == "CHEC": integrator_window_width = 10 integrator_window_shift = 5 elif cam_id == "DigiCam": integrator_window_width = 5 integrator_window_shift = 2 elif cam_id == "FlashCam": integrator_window_width = 6 integrator_window_shift = 3 elif cam_id == "NectarCam": integrator_window_width = 5 integrator_window_shift = 2 elif cam_id == "LSTCam": integrator_window_width = 5 integrator_window_shift = 2 else: raise ValueError('Unknown cam_id "{}"'.format(cam_id)) for image in image_generator( input_file_or_dir_path_list, max_num_images=max_num_img, tel_filter_list=tel_id, ev_filter_list=event_id, cam_filter_list=[cam_id], mc_rejection_criteria=rejection_criteria, ctapipe_format=False, integrator='LocalPeakIntegrator', integrator_window_width=integrator_window_width, integrator_window_shift=integrator_window_shift, integration_correction=False, mix_channels=True): input_file_path = image.meta['file_path'] if self.verbose: print(input_file_path) # `image_dict` contains metadata (to be returned) on the current image image_dict = {"input_file_path": input_file_path} try: # READ THE INPUT FILE ##################################### if self.verbose: print("TEL{}_EV{}".format(image.meta["tel_id"], image.meta["event_id"])) reference_img = image.reference_image pixels_position = image.pixels_position if ref_img_as_input: # This option is a hack to easily produce CSV files with # the "null_ref" "cleaning" module... input_img = copy.deepcopy(reference_img) else: input_img = image.input_image image_dict.update(image.meta) # Make the original 1D geom (required for the 2D to 1D geometry # conversion, for Tailcut and for Hillas) cam_id = image.meta['cam_id'] cleaning_function_params["cam_id"] = cam_id geom1d = geometry_converter.get_geom1d(cam_id) if benchmark_method is not None: # FETCH ADDITIONAL IMAGE METADATA ##################### image_dict["img_ref_signal_to_border"] = signal_to_border( reference_img) # TODO: NaN image_dict[ "img_ref_signal_to_border_distance"] = signal_to_border_distance( reference_img) # TODO: NaN image_dict["img_ref_pemax_on_border"] = pemax_on_border( reference_img) # TODO: NaN delta_pe, delta_abs_pe, delta_num_pixels = filter_pixels_clusters_stats( reference_img) # TODO: NaN num_islands = number_of_pixels_clusters( reference_img) # TODO: NaN image_dict["img_ref_islands_delta_pe"] = delta_pe image_dict["img_ref_islands_delta_abs_pe"] = delta_abs_pe image_dict[ "img_ref_islands_delta_num_pixels"] = delta_num_pixels image_dict["img_ref_num_islands"] = num_islands image_dict["img_ref_sum_pe"] = float( np.nansum(reference_img)) image_dict["img_ref_min_pe"] = float( np.nanmin(reference_img)) image_dict["img_ref_max_pe"] = float( np.nanmax(reference_img)) image_dict["img_ref_num_pix"] = int( (reference_img[np.isfinite(reference_img)] > 0).sum()) image_dict["img_in_sum_pe"] = float(np.nansum(input_img)) image_dict["img_in_min_pe"] = float(np.nanmin(input_img)) image_dict["img_in_max_pe"] = float(np.nanmax(input_img)) image_dict["img_in_num_pix"] = int( (input_img[np.isfinite(input_img)] > 0).sum()) reference_img1d = geometry_converter.image_2d_to_1d( reference_img, cam_id) try: hillas_params_2_ref_img = get_hillas_parameters( geom1d, reference_img1d, HILLAS_IMPLEMENTATION) # TODO GEOM except Exception as e: hillas_params_2_ref_img = float('nan') print(e) #traceback.print_tb(e.__traceback__, file=sys.stdout) image_dict["img_ref_hillas_2_size"] = float( hillas_params_2_ref_img.intensity) image_dict[ "img_ref_hillas_2_cen_x"] = hillas_params_2_ref_img.x.value image_dict[ "img_ref_hillas_2_cen_y"] = hillas_params_2_ref_img.y.value image_dict[ "img_ref_hillas_2_length"] = hillas_params_2_ref_img.length.value image_dict[ "img_ref_hillas_2_width"] = hillas_params_2_ref_img.width.value image_dict[ "img_ref_hillas_2_r"] = hillas_params_2_ref_img.r.value image_dict[ "img_ref_hillas_2_phi"] = hillas_params_2_ref_img.phi.to( u.rad).value image_dict[ "img_ref_hillas_2_psi"] = hillas_params_2_ref_img.psi.to( u.rad).value try: image_dict["img_ref_hillas_2_miss"] = float( hillas_params_2_ref_img.miss.value) except: image_dict["img_ref_hillas_2_miss"] = None image_dict[ "img_ref_hillas_2_kurtosis"] = hillas_params_2_ref_img.kurtosis image_dict[ "img_ref_hillas_2_skewness"] = hillas_params_2_ref_img.skewness # CLEAN THE INPUT IMAGE ################################### # Copy the image (otherwise some cleaning functions like Tailcut may change it) #input_img_copy = copy.deepcopy(input_img) input_img_copy = input_img.astype('float64', copy=True) cleaning_function_params["output_data_dict"] = {} initial_time = time.perf_counter() cleaned_img = self.clean_image( input_img_copy, **cleaning_function_params) # TODO: NaN full_clean_execution_time_sec = time.perf_counter( ) - initial_time if benchmark_method is not None: image_dict.update( cleaning_function_params["output_data_dict"]) del cleaning_function_params["output_data_dict"] # ASSESS OR PRINT THE CLEANED IMAGE ####################### if benchmark_method is not None: # ASSESS THE CLEANING ################################# kwargs = { 'geom': geom1d, 'hillas_implementation': HILLAS_IMPLEMENTATION } # TODO GEOM score_tuple, score_name_tuple = assess.assess_image_cleaning( input_img, cleaned_img, reference_img, benchmark_method, **kwargs) image_dict[ "img_cleaned_signal_to_border"] = signal_to_border( cleaned_img) image_dict[ "img_cleaned_signal_to_border_distance"] = signal_to_border_distance( cleaned_img) image_dict[ "img_cleaned_pemax_on_border"] = pemax_on_border( cleaned_img) image_dict["score"] = score_tuple image_dict["score_name"] = score_name_tuple image_dict[ "full_clean_execution_time_sec"] = full_clean_execution_time_sec image_dict["img_cleaned_sum_pe"] = float( np.nansum(cleaned_img)) image_dict["img_cleaned_min_pe"] = float( np.nanmin(cleaned_img)) image_dict["img_cleaned_max_pe"] = float( np.nanmax(cleaned_img)) image_dict["img_cleaned_num_pix"] = int( (cleaned_img[np.isfinite(cleaned_img)] > 0).sum()) cleaned_img1d = geometry_converter.image_2d_to_1d( cleaned_img, cam_id) try: hillas_params_2_cleaned_img = get_hillas_parameters( geom1d, cleaned_img1d, HILLAS_IMPLEMENTATION) # GEOM except Exception as e: hillas_params_2_cleaned_img = float('nan') print(e) #traceback.print_tb(e.__traceback__, file=sys.stdout) try: image_dict["img_cleaned_hillas_2_size"] = float( hillas_params_2_cleaned_img.intensity) except AttributeError as e: print(e) image_dict["img_cleaned_hillas_2_size"] = float('nan') try: image_dict[ "img_cleaned_hillas_2_cen_x"] = hillas_params_2_cleaned_img.x.value except AttributeError as e: print(e) image_dict["img_cleaned_hillas_2_cen_x"] = float('nan') try: image_dict[ "img_cleaned_hillas_2_cen_y"] = hillas_params_2_cleaned_img.y.value except AttributeError as e: print(e) image_dict["img_cleaned_hillas_2_cen_y"] = float('nan') try: image_dict[ "img_cleaned_hillas_2_length"] = hillas_params_2_cleaned_img.length.value except AttributeError as e: print(e) image_dict["img_cleaned_hillas_2_length"] = float( 'nan') try: image_dict[ "img_cleaned_hillas_2_width"] = hillas_params_2_cleaned_img.width.value except AttributeError as e: print(e) image_dict["img_cleaned_hillas_2_width"] = float('nan') try: image_dict[ "img_cleaned_hillas_2_r"] = hillas_params_2_cleaned_img.r.value except AttributeError as e: print(e) image_dict["img_cleaned_hillas_2_r"] = float('nan') try: image_dict[ "img_cleaned_hillas_2_phi"] = hillas_params_2_cleaned_img.phi.to( u.rad).value except AttributeError as e: print(e) image_dict["img_cleaned_hillas_2_phi"] = float('nan') try: image_dict[ "img_cleaned_hillas_2_psi"] = hillas_params_2_cleaned_img.psi.to( u.rad).value except AttributeError as e: print(e) image_dict["img_cleaned_hillas_2_psi"] = float('nan') try: image_dict["img_cleaned_hillas_2_miss"] = float( hillas_params_2_cleaned_img.miss.value) except: image_dict["img_cleaned_hillas_2_miss"] = float('nan') try: image_dict[ "img_cleaned_hillas_2_kurtosis"] = hillas_params_2_cleaned_img.kurtosis except AttributeError as e: print(e) image_dict["img_cleaned_hillas_2_kurtosis"] = float( 'nan') try: image_dict[ "img_cleaned_hillas_2_skewness"] = hillas_params_2_cleaned_img.skewness except AttributeError as e: print(e) image_dict["img_cleaned_hillas_2_skewness"] = float( 'nan') # PLOT IMAGES ######################################################### if plot or (saveplot is not None): image_list = [ geometry_converter.image_2d_to_1d(input_img, cam_id), geometry_converter.image_2d_to_1d( reference_img, cam_id), geometry_converter.image_2d_to_1d(cleaned_img, cam_id) ] title_list = [ "Input image", "Reference image", "Cleaned image" ] geom_list = [geom1d, geom1d, geom1d] hillas_list = [False, True, True] if plot: pywicta.io.images.plot_list(image_list, geom_list=geom_list, title_list=title_list, hillas_list=hillas_list, metadata_dict=image.meta) if saveplot is not None: basename, extension = os.path.splitext(saveplot) plot_file_path = "{}_E{}_T{}{}".format( basename, image.meta["event_id"], image.meta["tel_id"], extension) print("Saving {}".format(plot_file_path)) pywicta.io.images.mpl_save_list( image_list, geom_list=geom_list, output_file_path=plot_file_path, title_list=title_list, hillas_list=hillas_list, metadata_dict=image.meta) except Exception as e: print("Abort image {}: {} ({})".format(input_file_path, e, type(e))) if debug: # The following line print the full trackback traceback.print_tb(e.__traceback__, file=sys.stdout) if benchmark_method is not None: # http://docs.python.org/2/library/sys.html#sys.exc_info exc_type, exc_value, exc_traceback = sys.exc_info( ) # most recent (if any) by default ''' Reason this _can_ be bad: If an (unhandled) exception happens AFTER this, or if we do not delete the labels on (not much) older versions of Py, the reference we created can linger. traceback.format_exc/print_exc do this very thing, BUT note this creates a temp scope within the function. ''' error_dict = { 'filename': exc_traceback.tb_frame.f_code.co_filename, 'lineno': exc_traceback.tb_lineno, 'name': exc_traceback.tb_frame.f_code.co_name, 'type': exc_type.__name__, #'message' : exc_value.message 'message': str(e) } del ( exc_type, exc_value, exc_traceback ) # So we don't leave our local labels/objects dangling # This still isn't "completely safe", though! #error_dict = {"type": str(type(e)), # "message": str(e)} image_dict["error"] = error_dict finally: if benchmark_method is not None: io_list.append(image_dict) if benchmark_method is not None: error_list = [ image_dict["error"] for image_dict in io_list if "error" in image_dict ] print("{} images aborted".format(len(error_list))) # GENERAL EXPERIMENT METADATA output_dict = {} output_dict["benchmark_execution_time_sec"] = str( time.perf_counter() - launch_time) output_dict["date_time"] = str(datetime.datetime.now()) output_dict["class_name"] = self.__class__.__name__ output_dict["algo_code_ref"] = str( self.__class__.clean_image.__code__) output_dict["label"] = self.label output_dict["cmd"] = " ".join(sys.argv) output_dict["algo_params"] = cleaning_function_params if "noise_distribution" in output_dict["algo_params"]: del output_dict["algo_params"][ "noise_distribution"] # not JSON serializable... output_dict["benchmark_method"] = benchmark_method output_dict["system"] = " ".join(os.uname()) output_dict["io"] = io_list if output_file_path is not None: with open(output_file_path, "w") as fd: json.dump(output_dict, fd, sort_keys=True, indent=4) # pretty print format return output_dict
def clean_image(self, input_img, high_threshold=10., low_threshold=8., pixels_clusters_filtering="off", verbose=False, cam_id=None, output_data_dict=None, **kwargs): """Apply ctapipe's tail-cut image cleaning on ``input_img``. Note ---- The main difference with :mod:`ctapipe.image.cleaning.tailcuts_clean` is that here the cleaning function takes 2D Numpy arrays. Parameters ---------- input_img : array_like The image to clean. Should be a **2D** Numpy array. high_threshold : float The *core threshold* (a.k.a. *picture threshold*). low_threshold : float The *boundary threshold*. pixels_clusters_filtering : str Defines the method used to remove isolated pixels after the tail-cut image cleaning. Accepted values are: "off", "scipy" or "mars". - "off": don't apply any filtering after the tail-cut image cleaning. - "scipy": keep only the largest cluster of pixels after the tail-cut image cleaning. See :mod:`pywi.processing.filtering.pixel_clusters` for more information. - "mars": apply the same filtering than in CTA-Mars analysis, keep only *significant core pixels* that have at least two others *significant core pixels* among its neighbors (a *significant core pixels* is a pixel above the *core threshold*). verbose : bool Print additional messages if ``True``. cam_id : str The camera ID from which ``input_img`` came from: "ASTRICam", "CHEC", "DigiCam", "FlashCam", "NectarCam" or "LSTCam". output_data_dict : dict An optional dictionary used to transmit internal information to the caller. Returns ------- array_like The ``input_img`` after tail-cut image cleaning. This is a **2D** Numpy array (i.e. it should be converted with :func:`pywicta.io.geometry_converter.image_2d_to_1d` before any usage in ctapipe). """ if cam_id is None: raise Exception("cam_id have to be defined") # TODO if not (pixels_clusters_filtering.lower() in ("off", "scipy", "mars")): raise ValueError( 'pixels_clusters_filtering = {}. Accepted values are: "off", "scipy" or "mars".' .format(pixels_clusters_filtering)) # If low_threshold > high_threshold then low_threshold = high_threshold low_threshold = min(high_threshold, low_threshold) # 2D ARRAY (FITS IMAGE) TO CTAPIPE IMAGE ############### geom_1d = geometry_converter.get_geom1d(cam_id) img_1d = geometry_converter.image_2d_to_1d(input_img, cam_id) # APPLY TAILCUT CLEANING ############################## if pixels_clusters_filtering.lower() == "mars": if verbose: print("Mars pixels clusters filtering") mask = tailcuts_clean(geom_1d, img_1d, picture_thresh=high_threshold, boundary_thresh=low_threshold, keep_isolated_pixels=False, min_number_picture_neighbors=2) else: mask = tailcuts_clean(geom_1d, img_1d, picture_thresh=high_threshold, boundary_thresh=low_threshold, keep_isolated_pixels=True) img_1d[mask == False] = 0 # CTAPIPE IMAGE TO 2D ARRAY (FITS IMAGE) ############### cleaned_img_2d = geometry_converter.image_1d_to_2d(img_1d, cam_id) # KILL ISOLATED PIXELS ################################# img_cleaned_islands_delta_pe, img_cleaned_islands_delta_abs_pe, img_cleaned_islands_delta_num_pixels = filter_pixels_clusters_stats( cleaned_img_2d) img_cleaned_num_islands = number_of_pixels_clusters(cleaned_img_2d) if output_data_dict is not None: output_data_dict[ "img_cleaned_islands_delta_pe"] = img_cleaned_islands_delta_pe output_data_dict[ "img_cleaned_islands_delta_abs_pe"] = img_cleaned_islands_delta_abs_pe output_data_dict[ "img_cleaned_islands_delta_num_pixels"] = img_cleaned_islands_delta_num_pixels output_data_dict[ "img_cleaned_num_islands"] = img_cleaned_num_islands if pixels_clusters_filtering.lower() == "scipy": if verbose: print("Scipy pixels clusters filtering") cleaned_img_2d = scipy_pixels_clusters_filtering(cleaned_img_2d) return cleaned_img_2d
def prepare_event(self, source, return_stub=False, save_images=False): for event in source: self.event_cutflow.count("noCuts") if self.event_cutflow.cut("min2Tels trig", len(event.dl0.tels_with_data)): if return_stub: yield stub(event) else: continue self.calib(event) # telescope loop tot_signal = 0 dl1_phe_image = None mc_phe_image = None max_signals = {} n_pixel_dict = {} hillas_dict_reco = {} # for direction reconstruction hillas_dict = {} # for discrimination n_tels = { "tot": len(event.dl0.tels_with_data), "LST_LST_LSTCam": 0, "MST_MST_NectarCam": 0, "SST": 0, # add later correct names when testing on Paranal } n_cluster_dict = {} impact_dict_reco = {} # impact distance measured in tilt system point_azimuth_dict = {} point_altitude_dict = {} # Compute impact parameter in tilt system run_array_direction = event.mcheader.run_array_direction az, alt = run_array_direction[0], run_array_direction[1] ground_frame = GroundFrame() for tel_id in event.dl0.tels_with_data: self.image_cutflow.count("noCuts") camera = event.inst.subarray.tel[tel_id].camera # count the current telescope according to its size tel_type = str(event.inst.subarray.tel[tel_id]) # use ctapipe's functionality to get the calibrated image pmt_signal = event.dl1.tel[tel_id].image # Save the calibrated image after the gain has been chosen # automatically by ctapipe, together with the simulated one if save_images is True: dl1_phe_image = pmt_signal mc_phe_image = event.mc.tel[tel_id].photo_electron_image if self.cleaner_reco.mode == "tail": # tail uses only ctapipe # Cleaning used for direction reconstruction image_biggest, mask_reco = self.cleaner_reco.clean_image( pmt_signal, camera) # find all islands using this cleaning num_islands, labels = number_of_islands(camera, mask_reco) if num_islands == 1: # if only ONE islands is left ... # ...use directly the old mask and reduce dimensions # to make Hillas parametrization faster camera_biggest = camera[mask_reco] image_biggest = image_biggest[mask_reco] elif num_islands > 1: # if more islands survived.. # ...find the biggest one mask_biggest = largest_island(labels) # and also reduce dimensions camera_biggest = camera[mask_biggest] image_biggest = image_biggest[mask_biggest] else: # if no islands survived use old camera and image camera_biggest = camera # Cleaning used for score/energy estimation image_extended, mask_extended = self.cleaner_extended.clean_image( pmt_signal, camera) # find all islands with this cleaning # we will also register how many have been found n_cluster_dict[tel_id], labels = number_of_islands( camera, mask_extended) # NOTE: the next check shouldn't be necessary if we keep # all the isolated pixel clusters, but for now the # two cleanings are set the same in analysis.yml because # the performance of the extended one has never been really # studied in model estimation. # (This is a nice way to ask for volunteers :P) # if some islands survived if num_islands > 0: # keep all of them and reduce dimensions camera_extended = camera[mask_extended] image_extended = image_extended[mask_extended] else: # otherwise continue with the old camera and image camera_extended = camera # could this go into `hillas_parameters` ...? # this is basically the charge of ALL islands # not calculated later by the Hillas parametrization! max_signals[tel_id] = np.max(image_extended) else: # for wavelets we stick to old pywi-cta code try: # "try except FileNotFoundError" not clear to me, but for now it stays... with warnings.catch_warnings(): # Image with biggest cluster (reco cleaning) image_biggest, mask_reco = self.cleaner_reco.clean_image( pmt_signal, camera) image_biggest2d = geometry_converter.image_1d_to_2d( image_biggest, camera.cam_id) image_biggest2d = filter_pixels_clusters( image_biggest2d) image_biggest = geometry_converter.image_2d_to_1d( image_biggest2d, camera.cam_id) # Image for score/energy estimation (with clusters) image_extended, mask_extended = self.cleaner_extended.clean_image( pmt_signal, camera) # This last part was outside the pywi-cta block # before, but is indeed part of it because it uses # pywi-cta functions in the "extended" case # For cluster counts image_2d = geometry_converter.image_1d_to_2d( image_extended, camera.cam_id) n_cluster_dict[ tel_id] = pixel_clusters.number_of_pixels_clusters( array=image_2d, threshold=0) # could this go into `hillas_parameters` ...? max_signals[tel_id] = np.max(image_extended) except FileNotFoundError as e: # JLK, WHAT? print(e) continue # ============================================================== # Apply some selection if self.image_cutflow.cut("min pixel", image_biggest): continue if self.image_cutflow.cut("min charge", np.sum(image_biggest)): continue # do the hillas reconstruction of the images # QUESTION should this change in numpy behaviour be done here # or within `hillas_parameters` itself? # JLK: make selection on biggest cluster with np.errstate(invalid="raise", divide="raise"): try: moments_reco = hillas_parameters( camera_biggest, image_biggest) # for geometry (eg direction) moments = hillas_parameters( camera_extended, image_extended ) # for discrimination and energy reconstruction # if width and/or length are zero (e.g. when there is # only only one pixel or when all pixel are exactly # in one row), the parametrisation # won't be very useful: skip if self.image_cutflow.cut("poor moments", moments_reco): continue if self.image_cutflow.cut("close to the edge", moments_reco, camera.cam_id): continue if self.image_cutflow.cut("bad ellipticity", moments_reco): continue except (FloatingPointError, hillas.HillasParameterizationError): continue point_azimuth_dict[ tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad point_altitude_dict[ tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad n_tels[tel_type] += 1 hillas_dict[tel_id] = moments hillas_dict_reco[tel_id] = moments_reco n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0]) tot_signal += moments.intensity n_tels["reco"] = len(hillas_dict_reco) n_tels["discri"] = len(hillas_dict) if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]): if return_stub: yield stub(event) else: continue try: with warnings.catch_warnings(): warnings.simplefilter("ignore") # Reconstruction results reco_result = self.shower_reco.predict( hillas_dict_reco, event.inst, SkyCoord(alt=alt, az=az, frame="altaz"), { tel_id: SkyCoord( alt=point_altitude_dict[tel_id], az=point_azimuth_dict[tel_id], frame="altaz", ) # cycle only on tels which still have an image for tel_id in point_altitude_dict.keys() }, ) # Impact parameter for energy estimation (/ tel) subarray = event.inst.subarray for tel_id in hillas_dict.keys(): pos = subarray.positions[tel_id] tel_ground = SkyCoord(pos[0], pos[1], pos[2], frame=ground_frame) core_ground = SkyCoord( reco_result.core_x, reco_result.core_y, 0 * u.m, frame=ground_frame, ) # Should be better handled (tilted frame) impact_dict_reco[tel_id] = np.sqrt( (core_ground.x - tel_ground.x)**2 + (core_ground.y - tel_ground.y)**2) except Exception as e: print("exception in reconstruction:", e) raise if return_stub: yield stub(event) else: continue if self.event_cutflow.cut("direction nan", reco_result): if return_stub: yield stub(event) else: continue yield PreparedEvent( event=event, dl1_phe_image=dl1_phe_image, mc_phe_image=mc_phe_image, n_pixel_dict=n_pixel_dict, hillas_dict=hillas_dict, hillas_dict_reco=hillas_dict_reco, n_tels=n_tels, tot_signal=tot_signal, max_signals=max_signals, n_cluster_dict=n_cluster_dict, reco_result=reco_result, impact_dict=impact_dict_reco, )
def prepare_event(self, source, return_stub=False): # configuration for the camera calibrator # modifies the integration window to be more like in MARS # JLK, only for LST!!!! # Option for integration correction is done above cfg = Config() cfg["ChargeExtractorFactory"]["window_width"] = 5 cfg["ChargeExtractorFactory"]["window_shift"] = 2 self.calib = CameraCalibrator( config=cfg, extractor_product="LocalPeakIntegrator", eventsource=source, tool=None, ) for event in source: self.event_cutflow.count("noCuts") if self.event_cutflow.cut("min2Tels trig", len(event.dl0.tels_with_data)): if return_stub: yield stub(event) else: continue # calibrate the event self.calib.calibrate(event) # telescope loop tot_signal = 0 max_signals = {} n_pixel_dict = {} hillas_dict_reco = {} # for geometry hillas_dict = {} # for discrimination n_tels = { "tot": len(event.dl0.tels_with_data), "LST": 0, "MST": 0, "SST": 0, } n_cluster_dict = {} impact_dict_reco = {} # impact distance measured in tilt system point_azimuth_dict = {} point_altitude_dict = {} # To compute impact parameter in tilt system run_array_direction = event.mcheader.run_array_direction az, alt = run_array_direction[0], run_array_direction[1] ground_frame = GroundFrame() for tel_id in event.dl0.tels_with_data: self.image_cutflow.count("noCuts") camera = event.inst.subarray.tel[tel_id].camera # count the current telescope according to its size tel_type = event.inst.subarray.tel[tel_id].optics.tel_type # JLK, N telescopes before cut selection are not really interesting for # discrimination, too much fluctuations # n_tels[tel_type] += 1 # the camera image as a 1D array and stuff needed for calibration # Choose gain according to pywicta's procedure image_1d = simtel_event_to_images(event=event, tel_id=tel_id, ctapipe_format=True) pmt_signal = image_1d.input_image # calibrated image # clean the image try: with warnings.catch_warnings(): # Image with biggest cluster (reco cleaning) image_biggest = self.cleaner_reco.clean_image( pmt_signal, camera) image_biggest2d = geometry_converter.image_1d_to_2d( image_biggest, camera.cam_id) image_biggest2d = filter_pixels_clusters( image_biggest2d) image_biggest = geometry_converter.image_2d_to_1d( image_biggest2d, camera.cam_id) # Image for score/energy estimation (with clusters) image_extended = self.cleaner_extended.clean_image( pmt_signal, camera) except FileNotFoundError as e: # JLK, WHAT? print(e) continue # Apply some selection if self.image_cutflow.cut("min pixel", image_biggest): continue if self.image_cutflow.cut("min charge", np.sum(image_biggest)): continue # For cluster counts image_2d = geometry_converter.image_1d_to_2d( image_extended, camera.cam_id) n_cluster_dict[ tel_id] = pixel_clusters.number_of_pixels_clusters( array=image_2d, threshold=0) # could this go into `hillas_parameters` ...? max_signals[tel_id] = np.max(image_extended) # do the hillas reconstruction of the images # QUESTION should this change in numpy behaviour be done here # or within `hillas_parameters` itself? # JLK: make selection on biggest cluster with np.errstate(invalid="raise", divide="raise"): try: moments_reco = hillas_parameters( camera, image_biggest) # for geometry (eg direction) moments = hillas_parameters( camera, image_extended ) # for discrimination and energy reconstruction # if width and/or length are zero (e.g. when there is only only one # pixel or when all pixel are exactly in one row), the # parametrisation won't be very useful: skip if self.image_cutflow.cut("poor moments", moments_reco): # print('poor moments') continue if self.image_cutflow.cut("close to the edge", moments_reco, camera.cam_id): # print('close to the edge') continue if self.image_cutflow.cut("bad ellipticity", moments_reco): # print('bad ellipticity: w={}, l={}'.format(moments_reco.width, moments_reco.length)) continue except (FloatingPointError, hillas.HillasParameterizationError): continue point_azimuth_dict[ tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad point_altitude_dict[ tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad n_tels[tel_type] += 1 hillas_dict[tel_id] = moments hillas_dict_reco[tel_id] = moments_reco n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0]) tot_signal += moments.intensity n_tels["reco"] = len(hillas_dict_reco) n_tels["discri"] = len(hillas_dict) if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]): if return_stub: yield stub(event) else: continue try: with warnings.catch_warnings(): warnings.simplefilter("ignore") # Reconstruction results reco_result = self.shower_reco.predict( hillas_dict_reco, event.inst, point_altitude_dict, point_azimuth_dict, ) # shower_sys = TiltedGroundFrame(pointing_direction=HorizonFrame( # az=reco_result.az, # alt=reco_result.alt # )) # Impact parameter for energy estimation (/ tel) subarray = event.inst.subarray for tel_id in hillas_dict.keys(): pos = subarray.positions[tel_id] tel_ground = SkyCoord(pos[0], pos[1], pos[2], frame=ground_frame) # tel_tilt = tel_ground.transform_to(shower_sys) core_ground = SkyCoord( reco_result.core_x, reco_result.core_y, 0 * u.m, frame=ground_frame, ) # core_tilt = core_ground.transform_to(shower_sys) # Should be better handled (tilted frame) impact_dict_reco[tel_id] = np.sqrt( (core_ground.x - tel_ground.x)**2 + (core_ground.y - tel_ground.y)**2) except Exception as e: print("exception in reconstruction:", e) raise if return_stub: yield stub(event) else: continue if self.event_cutflow.cut("direction nan", reco_result): if return_stub: yield stub(event) else: continue yield PreparedEvent( event=event, n_pixel_dict=n_pixel_dict, hillas_dict=hillas_dict, hillas_dict_reco=hillas_dict_reco, n_tels=n_tels, tot_signal=tot_signal, max_signals=max_signals, n_cluster_dict=n_cluster_dict, reco_result=reco_result, impact_dict=impact_dict_reco, )
def clean_image( self, input_img, type_of_multiresolution_transform=None, type_of_filters=None, type_of_non_orthog_filters=None, number_of_scales=None, suppress_last_scale=False, suppress_isolated_pixels=False, kill_isolated_pixels=False, coef_detection_method=None, k_sigma_noise_threshold=None, noise_model=None, detect_only_positive_structure=False, suppress_positivity_constraint=False, type_of_filtering=None, first_detection_scale=None, number_of_iterations=None, epsilon=None, support_file_name=None, precision=None, mask_file_path=None, offset_after_calibration=None, correction_offset=False, input_image_scale='linear', noise_distribution=None, verbose=False, raw_option_string=None, tmp_files_directory=".", # "/Volumes/ramdisk" mrfilter_directory=None, # "/Volumes/ramdisk" output_data_dict=None, **kwargs): """Clean the `input_img` image. Apply the wavelet transform, filter planes and return the reverse transformed image. Parameters ---------- input_img : array_like The input image to transform. number_of_scales : int The number of scales used to transform `input_image` or in other words the number of wavelet planes returned. tmp_files_directory : str The path of the directory used to store mr_transform temporary data. The default is the current directory, but it may be more appropriate to specify here the path of a directory mounted in a ramdisk to speedup I/Os ("/Volumes/ramdisk" on MacOSX or "/dev/shm" on Linux). noise_distribution : EmpiricalDistribution The noise distribution used to fill 'empty' NaN pixels with the appropriate random noise distribution. If none, NaN pixels are fill with zeros (which may add unwanted harmonics in wavelet planes). Returns ------- array_like Return the cleaned image. Raises ------ WrongDimensionError If `cleaned_img` is not a 2D array. """ input_img = input_img.copy() if input_img.ndim != 2: raise WrongDimensionError() input_file_path = os.path.join( tmp_files_directory, ".tmp_{}_{}_in.fits".format(os.getpid(), time.time())) mr_output_file_path = os.path.join( tmp_files_directory, ".tmp_{}_{}_out.fits".format(os.getpid(), time.time())) if output_data_dict is not None: output_data_dict["mr_input_tmp_file_path"] = input_file_path output_data_dict["mr_output_tmp_file_path"] = mr_output_file_path if (output_data_dict is not None) and (mask_file_path is not None): output_data_dict["mr_mask_file_path"] = mask_file_path # INJECT NOISE IN NAN ################################## # See https://stackoverflow.com/questions/29365194/replacing-missing-values-with-random-in-a-numpy-array nan_mask = images.fill_nan_pixels(input_img, noise_distribution) # APPLY AN OFFSET ###################################### if offset_after_calibration is not None: if verbose: print("Apply an offset after calibration:", offset_after_calibration) input_img = input_img + offset_after_calibration # CHANGE THE SCALE ##################################### if input_image_scale == 'log': if verbose: print("Apply log scale") #images.plot(input_img) input_img = np.log10( input_img) # TODO: it creates NaN values where pixels <= 0 #images.plot(input_img) elif input_image_scale == 'sqrt': if verbose: print("Apply sqrt scale") #images.plot(input_img) input_img = np.sqrt( input_img) # TODO: it creates NaN values where pixels < 0 #images.plot(input_img) # WRITE THE INPUT FILE (FITS) ########################## try: initial_time = time.perf_counter() images.save_fits(input_img, input_file_path) exec_time_sec = time.perf_counter() - initial_time if output_data_dict is not None: output_data_dict["save_tmp_file_time_sec"] = exec_time_sec except: print("Error on input FITS file:", input_file_path) raise # EXECUTE MR_FILTER #################################### # TODO: improve the following lines if mrfilter_directory is None: cmd = 'mr_filter' else: cmd = os.path.join(mrfilter_directory, 'mr_filter') if raw_option_string is None: cmd += ' -t{}'.format( type_of_multiresolution_transform ) if type_of_multiresolution_transform is not None else '' cmd += ' -T{}'.format( type_of_filters) if type_of_filters is not None else '' cmd += ' -U{}'.format( type_of_non_orthog_filters ) if type_of_non_orthog_filters is not None else '' cmd += ' -n{}'.format( number_of_scales) if number_of_scales is not None else '' cmd += ' -K' if suppress_last_scale else '' cmd += ' -k' if suppress_isolated_pixels else '' # You should use scipy implementation instead (pywi.processing.filtering.pixel_clusters.filter_pixels_clusters); it's much more efficient cmd += ' -C{}'.format( coef_detection_method ) if coef_detection_method is not None else '' cmd += ' -s{}'.format( k_sigma_noise_threshold ) if k_sigma_noise_threshold is not None else '' cmd += ' -m{}'.format( noise_model) if noise_model is not None else '' cmd += ' -p' if detect_only_positive_structure else '' cmd += ' -P' if suppress_positivity_constraint else '' cmd += ' -f{}'.format( type_of_filtering) if type_of_filtering is not None else '' cmd += ' -F{}'.format( first_detection_scale ) if first_detection_scale is not None else '' cmd += ' -i{}'.format(number_of_iterations ) if number_of_iterations is not None else '' cmd += ' -e{}'.format(epsilon) if epsilon is not None else '' cmd += ' -w{}'.format( support_file_name) if support_file_name is not None else '' cmd += ' -E{}'.format(precision) if precision is not None else '' cmd += ' -I {}'.format( mask_file_path) if mask_file_path is not None else '' cmd += ' -v' if verbose else '' else: cmd += ' ' + raw_option_string #self.label = "WT ({})".format(cmd) # Name to show in plots cmd += ' "{}" "{}"'.format(input_file_path, mr_output_file_path) #cmd = 'mr_filter -K -k -C1 -s3 -m3 -n{} "{}" {}'.format(number_of_scales, input_file_path, mr_output_file_path) #cmd = 'mr_filter -K -k -C1 -s3 -m2 -p -P -n{} "{}" {}'.format(number_of_scales, input_file_path, mr_output_file_path) if verbose: print() print(cmd) else: cmd += ' > /dev/null' try: initial_time = time.perf_counter() os.system(cmd) exec_time_sec = time.perf_counter() - initial_time if output_data_dict is not None: output_data_dict["mrfilter_cmd_exec_time_sec"] = exec_time_sec except: print("Error on command:", cmd) raise # READ THE MR_FILTER OUTPUT FILE ####################### try: initial_time = time.perf_counter() cleaned_img = images.load_fits(mr_output_file_path, 0) exec_time_sec = time.perf_counter() - initial_time if output_data_dict is not None: output_data_dict["load_tmp_file_time_sec"] = exec_time_sec except: print("Error on output FITS file:", mr_output_file_path) raise # REMOVE FITS FILES #################################### os.remove(input_file_path) os.remove(mr_output_file_path) # CHECK RESULT ######################################### if cleaned_img.ndim != 2: raise WrongDimensionError() # INJECT NOISE IN NAN: PUT BACK NAN VALUES ############# cleaned_img[nan_mask] = np.nan # CHANGE THE SCALE ##################################### if input_image_scale == 'log': if verbose: print("Invert log scale") cleaned_img = np.power(10., cleaned_img) elif input_image_scale == 'sqrt': if verbose: print("Invert sqrt scale") cleaned_img = np.power(2., cleaned_img) # INVERT THE OFFSET #################################### if (offset_after_calibration is not None) and (not suppress_last_scale): cleaned_img = cleaned_img - offset_after_calibration # CORRECTION OFFSET #################################### if correction_offset: if verbose: print("Apply a correction offset after cleaning") cleaned_img = cleaned_img - np.nanmin(cleaned_img) cleaned_img[np.isfinite(cleaned_img) & (cleaned_img < 1.0 )] = 0. # May genereate warnings on NaN values # KILL ISOLATED PIXELS ################################# img_cleaned_islands_delta_pe, img_cleaned_islands_delta_abs_pe, img_cleaned_islands_delta_num_pixels = filter_pixels_clusters_stats( cleaned_img) img_cleaned_num_islands = number_of_pixels_clusters(cleaned_img) if output_data_dict is not None: output_data_dict[ "img_cleaned_islands_delta_pe"] = img_cleaned_islands_delta_pe output_data_dict[ "img_cleaned_islands_delta_abs_pe"] = img_cleaned_islands_delta_abs_pe output_data_dict[ "img_cleaned_islands_delta_num_pixels"] = img_cleaned_islands_delta_num_pixels output_data_dict[ "img_cleaned_num_islands"] = img_cleaned_num_islands if kill_isolated_pixels: if verbose: print("Kill isolated pixels") initial_time = time.perf_counter() cleaned_img = scipy_kill_isolated_pixels(cleaned_img) exec_time_sec = time.perf_counter() - initial_time if output_data_dict is not None: output_data_dict[ "scipy_kill_isolated_pixels_time_sec"] = exec_time_sec #print(cleaned_img) #images.plot_hist(cleaned_img) #images.plot_hist(cleaned_img, num_bins=500, x_max=5) return cleaned_img