def clean_image(self, img, output_data_dict=None, **kwargs):
# Add (in the output dictionary) information about islands (pixels clusters) for the reference image
if output_data_dict is not None:
cleaned_image = np.copy(img)
kill_islands = filter_pixels_clusters_stats(cleaned_image)
img_cleaned_islands_delta_pe, img_cleaned_islands_delta_abs_pe, img_cleaned_islands_delta_num_pixels = kill_islands
img_cleaned_num_islands = number_of_pixels_clusters(cleaned_image)
output_data_dict["img_cleaned_islands_delta_pe"] = img_cleaned_islands_delta_pe
output_data_dict["img_cleaned_islands_delta_abs_pe"] = img_cleaned_islands_delta_abs_pe
output_data_dict["img_cleaned_islands_delta_num_pixels"] = img_cleaned_islands_delta_num_pixels
output_data_dict["img_cleaned_num_islands"] = img_cleaned_num_islands
return copy.deepcopy(img)
def clean_image(self,
input_image,
type_of_filtering=hard_filter.DEFAULT_TYPE_OF_FILTERING,
filter_thresholds=hard_filter.DEFAULT_FILTER_THRESHOLDS,
last_scale_treatment=starlet.DEFAULT_LAST_SCALE_TREATMENT,
detect_only_positive_structures=False,
kill_isolated_pixels=False,
noise_distribution=None,
output_data_dict=None,
clusters_threshold=0,
**kwargs):
"""Clean the `input_image` image.
Apply the wavelet transform, filter planes and return the reverse
transformed image.
Parameters
----------
input_image : array_like
The image to clean.
type_of_filtering : str
Type of filtering: 'hard_filtering' or 'ksigma_hard_filtering'.
filter_thresholds : list of float
Thresholds used for the plane filtering.
last_scale_treatment : str
Last plane treatment: 'keep', 'drop' or 'mask'.
detect_only_positive_structures : bool
Detect only positive structures.
kill_isolated_pixels : bool
Suppress isolated pixels in the support.
noise_distribution : bool
The JSON file containing the Cumulated Distribution Function of the
noise model used to inject artificial noise in blank pixels (those
with a NaN value).
output_data_dict : dict
A dictionary used to return results and intermediate results.
clusters_threshold : float
The threshold used to define clusters of pixels (the "level of the sea")
Returns
-------
Return the cleaned image.
"""
if DEBUG:
print("Filter thresholds:", filter_thresholds)
number_of_scales = len(filter_thresholds) + 1
if not (1 < number_of_scales <= 10):
# This range ]1,10] is a hard constraint from the starlet transform
raise ValueError(
"bad number of scales: {}. Should be 1 < Nbr Scales <= 10. Check that filter_thresholds is a list of number and not a string."
.format(number_of_scales))
if DEBUG:
print("Number of scales:", number_of_scales)
# COMPUTE THE WAVELET TRANSFORM #######################################
wavelet_planes = wavelet_transform(
input_image,
number_of_scales=number_of_scales,
noise_distribution=noise_distribution)
if DEBUG:
for index, plane in enumerate(wavelet_planes):
images.plot(plane, "Plane " + str(index))
# FILTER WAVELET PLANES ###############################################
filtered_wavelet_planes = filter_planes(
wavelet_planes,
method=type_of_filtering,
thresholds=filter_thresholds,
detect_only_positive_structures=detect_only_positive_structures)
#if DEBUG:
# for index, plane in enumerate(filtered_wavelet_planes):
# images.plot(plane, "Filtered plane " + str(index))
# COMPUTE THE INVERSE TRANSFORM #######################################
cleaned_image = inverse_wavelet_transform(
filtered_wavelet_planes, last_plane=last_scale_treatment)
if DEBUG:
images.plot(cleaned_image, "Cleaned image")
# KILL ISOLATED PIXELS ################################################
kill_islands = filter_pixels_clusters_stats(cleaned_image)
img_cleaned_islands_delta_pe, img_cleaned_islands_delta_abs_pe, img_cleaned_islands_delta_num_pixels = kill_islands
img_cleaned_num_islands = number_of_pixels_clusters(cleaned_image)
if output_data_dict is not None:
output_data_dict[
"img_cleaned_islands_delta_pe"] = img_cleaned_islands_delta_pe
output_data_dict[
"img_cleaned_islands_delta_abs_pe"] = img_cleaned_islands_delta_abs_pe
output_data_dict[
"img_cleaned_islands_delta_num_pixels"] = img_cleaned_islands_delta_num_pixels
output_data_dict[
"img_cleaned_num_islands"] = img_cleaned_num_islands
if kill_isolated_pixels:
cleaned_image = scipy_kill_isolated_pixels(
cleaned_image, threshold=clusters_threshold)
if DEBUG:
images.plot(cleaned_image, "Cleaned image after island kill")
return cleaned_image
def run(
self,
cleaning_function_params,
input_file_or_dir_path_list,
benchmark_method,
output_file_path,
plot=False,
saveplot=None,
ref_img_as_input=False, # A hack to easily produce CSV files...
max_num_img=None,
debug=False):
"""A convenient optional wrapper to simplify the image cleaning analysis.
Apply the image cleaning analysis on `input_file_or_dir_path_list`,
apply some pre-processing and post-processing procedures, collect and
return results, intermediate values and metadata.
Parameters
----------
cleaning_function_params
A dictionary containing the parameters required for the image
cleaning method.
input_file_or_dir_path_list
A list of file to clean. Can be a list of simtel files, fits files
or directories containing such files.
benchmark_method
The list of estimators to use to assess the image cleaning. If
`None`, images are cleaned but nothing is returned (can be used
with e.g. the `plot` and/or `saveplot` options).
output_file_path
The result file path (a JSON file).
plot
The result of each cleaning is plot if `True`.
saveplot
The result of each cleaning is saved if `True`.
ref_img_as_input
This option is a hack to easily produce a "flatten" CSV results
files.
max_num_img
The number of images to process among the input set
(`input_file_or_dir_path_list`).
debug
Stop the execution and print the full traceback when an exception
is encountered if this parameter is `True`. Report exceptions and
continue with the next input image if this parameter is `False`.
Returns
-------
dict
Results, intermediate values and metadata.
"""
launch_time = time.perf_counter()
if benchmark_method is not None:
io_list = [] # The list of returned dictionaries
for image in image_generator(input_file_or_dir_path_list,
max_num_images=max_num_img):
input_file_path = image.meta['file_path']
if self.verbose:
print(input_file_path)
# `image_dict` contains metadata (to be returned) on the current image
image_dict = {"input_file_path": input_file_path}
try:
# READ THE INPUT FILE #####################################
reference_img = image.reference_image
pixels_position = image.pixels_position
if ref_img_as_input:
# This option is a hack to easily produce CSV files with
# the "null_ref" "cleaning" module...
input_img = copy.deepcopy(reference_img)
else:
input_img = image.input_image
image_dict.update(image.meta)
if benchmark_method is not None:
# FETCH ADDITIONAL IMAGE METADATA #####################
delta_pe, delta_abs_pe, delta_num_pixels = filter_pixels_clusters_stats(
reference_img) # TODO: NaN
num_islands = number_of_pixels_clusters(
reference_img) # TODO: NaN
image_dict["img_ref_islands_delta_pe"] = delta_pe
image_dict["img_ref_islands_delta_abs_pe"] = delta_abs_pe
image_dict[
"img_ref_islands_delta_num_pixels"] = delta_num_pixels
image_dict["img_ref_num_islands"] = num_islands
image_dict["img_ref_sum_pe"] = float(
np.nansum(reference_img))
image_dict["img_ref_min_pe"] = float(
np.nanmin(reference_img))
image_dict["img_ref_max_pe"] = float(
np.nanmax(reference_img))
image_dict["img_ref_num_pix"] = int(
(reference_img[np.isfinite(reference_img)] > 0).sum())
image_dict["img_in_sum_pe"] = float(np.nansum(input_img))
image_dict["img_in_min_pe"] = float(np.nanmin(input_img))
image_dict["img_in_max_pe"] = float(np.nanmax(input_img))
image_dict["img_in_num_pix"] = int(
(input_img[np.isfinite(input_img)] > 0).sum())
# CLEAN THE INPUT IMAGE ###################################
# Copy the image (otherwise some cleaning functions like Tailcut may change it)
#input_img_copy = copy.deepcopy(input_img)
input_img_copy = input_img.astype('float64', copy=True)
cleaning_function_params["output_data_dict"] = {}
initial_time = time.perf_counter()
cleaned_img = self.clean_image(
input_img_copy, **cleaning_function_params) # TODO: NaN
full_clean_execution_time_sec = time.perf_counter(
) - initial_time
if benchmark_method is not None:
image_dict.update(
cleaning_function_params["output_data_dict"])
del cleaning_function_params["output_data_dict"]
# ASSESS OR PRINT THE CLEANED IMAGE #######################
if benchmark_method is not None:
# ASSESS THE CLEANING #################################
kwargs = {} # TODO GEOM
score = mse(cleaned_img, reference_img)
image_dict["score"] = [score]
image_dict["score_name"] = ["mse"]
image_dict[
"full_clean_execution_time_sec"] = full_clean_execution_time_sec
image_dict["img_cleaned_sum_pe"] = float(
np.nansum(cleaned_img))
image_dict["img_cleaned_min_pe"] = float(
np.nanmin(cleaned_img))
image_dict["img_cleaned_max_pe"] = float(
np.nanmax(cleaned_img))
image_dict["img_cleaned_num_pix"] = int(
(cleaned_img[np.isfinite(cleaned_img)] > 0).sum())
# PLOT IMAGES #########################################################
if plot or (saveplot is not None):
image_list = [input_img, reference_img, cleaned_img]
title_list = [
"Input image", "Reference image", "Cleaned image"
]
if plot:
pywi.io.images.plot_list(image_list,
title_list=title_list,
metadata_dict=image.meta)
if saveplot is not None:
plot_file_path = saveplot
print("Saving {}".format(plot_file_path))
pywi.io.images.mpl_save_list(
image_list,
output_file_path=plot_file_path,
title_list=title_list,
metadata_dict=image.meta)
except Exception as e:
print("Abort image {}: {} ({})".format(input_file_path, e,
type(e)))
if debug:
# The following line print the full trackback
traceback.print_tb(e.__traceback__, file=sys.stdout)
if benchmark_method is not None:
# http://docs.python.org/2/library/sys.html#sys.exc_info
exc_type, exc_value, exc_traceback = sys.exc_info(
) # most recent (if any) by default
'''
Reason this _can_ be bad: If an (unhandled) exception happens AFTER this,
or if we do not delete the labels on (not much) older versions of Py, the
reference we created can linger.
traceback.format_exc/print_exc do this very thing, BUT note this creates a
temp scope within the function.
'''
error_dict = {
'filename': exc_traceback.tb_frame.f_code.co_filename,
'lineno': exc_traceback.tb_lineno,
'name': exc_traceback.tb_frame.f_code.co_name,
'type': exc_type.__name__,
#'message' : exc_value.message
'message': str(e)
}
del (
exc_type, exc_value, exc_traceback
) # So we don't leave our local labels/objects dangling
# This still isn't "completely safe", though!
#error_dict = {"type": str(type(e)),
# "message": str(e)}
image_dict["error"] = error_dict
finally:
if benchmark_method is not None:
io_list.append(image_dict)
if benchmark_method is not None:
error_list = [
image_dict["error"] for image_dict in io_list
if "error" in image_dict
]
print("{} images aborted".format(len(error_list)))
# GENERAL EXPERIMENT METADATA
output_dict = {}
output_dict["benchmark_execution_time_sec"] = str(
time.perf_counter() - launch_time)
output_dict["date_time"] = str(datetime.datetime.now())
output_dict["class_name"] = self.__class__.__name__
output_dict["algo_code_ref"] = str(
self.__class__.clean_image.__code__)
output_dict["label"] = self.label
output_dict["cmd"] = " ".join(sys.argv)
output_dict["algo_params"] = cleaning_function_params
if "noise_distribution" in output_dict["algo_params"]:
del output_dict["algo_params"][
"noise_distribution"] # not JSON serializable...
output_dict["benchmark_method"] = benchmark_method
output_dict["system"] = " ".join(os.uname())
output_dict["io"] = io_list
with open(output_file_path, "w") as fd:
json.dump(output_dict, fd, sort_keys=True,
indent=4) # pretty print format
return output_dict
def prepare_event(self,
source,
return_stub=True,
save_images=False,
debug=False):
"""
Calibrate, clean and reconstruct the direction of an event.
Parameters
----------
source : ctapipe.io.EventSource
A container of selected showers from a simtel file.
geom_cam_tel: dict
Dictionary of of MyCameraGeometry objects for each camera in the file
return_stub : bool
If True, yield also images from events that won't be reconstructed.
This feature is not currently available.
save_images : bool
If True, save photoelectron images from reconstructed events.
debug : bool
If True, print some debugging information (to be expanded).
Yields
------
PreparedEvent: dict
Dictionary containing event-image information to be written.
"""
# =============================================================
# TRANSFORMED CAMERA GEOMETRIES
# =============================================================
# These are the camera geometries were the Hillas parametrization will
# be performed.
# They are transformed to TelescopeFrame using the effective focal
# lengths
# These geometries could be used also to performe the image cleaning,
# but for the moment we still do that in the CameraFrame
geom_cam_tel = {}
for camera in source.subarray.camera_types:
# Original geometry of each camera
geom = camera.geometry
# Same but with focal length as an attribute
# This is planned to disappear and be imported by ctapipe
focal_length = effective_focal_lengths(camera.camera_name)
geom_cam_tel[camera.camera_name] = MyCameraGeometry(
camera_name=camera.camera_name,
pix_type=geom.pix_type,
pix_id=geom.pix_id,
pix_x=geom.pix_x,
pix_y=geom.pix_y,
pix_area=geom.pix_area,
cam_rotation=geom.cam_rotation,
pix_rotation=geom.pix_rotation,
frame=CameraFrame(focal_length=focal_length),
).transform_to(TelescopeFrame())
# =============================================================
ievt = 0
for event in source:
# Display event counts
if debug:
print(
bcolors.BOLD +
f"EVENT #{event.count}, EVENT_ID #{event.index.event_id}" +
bcolors.ENDC)
print(bcolors.BOLD +
f"has triggered telescopes {event.r1.tel.keys()}" +
bcolors.ENDC)
ievt += 1
# if (ievt < 10) or (ievt % 10 == 0):
# print(ievt)
self.event_cutflow.count("noCuts")
# LST stereo condition
# whenever there is only 1 LST in an event, we remove that telescope
# if the remaining telescopes are less than min_tel we remove the event
lst_tel_ids = set(
source.subarray.get_tel_ids_for_type("LST_LST_LSTCam"))
triggered_LSTs = set(event.r0.tel.keys()).intersection(lst_tel_ids)
n_triggered_LSTs = len(triggered_LSTs)
n_triggered_non_LSTs = len(event.r0.tel.keys()) - n_triggered_LSTs
bad_LST_stereo = False
if self.LST_stereo and self.event_cutflow.cut(
"no-LST-stereo + <2 other types", n_triggered_LSTs,
n_triggered_non_LSTs):
bad_LST_stereo = True
if return_stub:
print(
bcolors.WARNING +
"WARNING: LST_stereo is set to 'True'\n" +
f"This event has < {self.min_ntel_LST} triggered LSTs\n"
+
"and < 2 triggered telescopes from other telescope types.\n"
+ "The event will be processed up to DL1b." +
bcolors.ENDC)
# we show this, but we proceed to analyze the event up to
# DL1a/b for the associated benchmarks
# this checks for < 2 triggered telescopes of ANY type
if self.event_cutflow.cut("min2Tels trig",
len(event.r1.tel.keys())):
if return_stub:
print(
bcolors.WARNING +
f"WARNING : < {self.min_ntel} triggered telescopes!" +
bcolors.ENDC)
# we show this, but we proceed to analyze it
# =============================================================
# CALIBRATION
# =============================================================
if debug:
print(bcolors.OKBLUE + "Extracting all calibrated images..." +
bcolors.ENDC)
self.calib(event) # Calibrate the event
# =============================================================
# BEGINNING OF LOOP OVER TELESCOPES
# =============================================================
dl1_phe_image = {}
dl1_phe_image_mask_reco = {}
dl1_phe_image_mask_clusters = {}
mc_phe_image = {}
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for direction reconstruction
hillas_dict = {} # for discrimination
leakage_dict = {}
concentration_dict = {}
n_tels = {
"Triggered": len(event.r1.tel.keys()),
"LST_LST_LSTCam": 0,
"MST_MST_NectarCam": 0,
"MST_MST_FlashCam": 0,
"MST_SCT_SCTCam": 0,
"SST_1M_DigiCam": 0,
"SST_ASTRI_ASTRICam": 0,
"SST_GCT_CHEC": 0,
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
good_for_reco = {} # 1 = success, 0 = fail
# Array pointing in AltAz frame
az = event.pointing.array_azimuth
alt = event.pointing.array_altitude
array_pointing = SkyCoord(az, alt, frame=AltAz())
ground_frame = GroundFrame()
tilted_frame = TiltedGroundFrame(pointing_direction=array_pointing)
for tel_id in event.r1.tel.keys():
point_azimuth_dict[tel_id] = event.pointing.tel[tel_id].azimuth
point_altitude_dict[tel_id] = event.pointing.tel[
tel_id].altitude
if debug:
print(bcolors.OKBLUE +
f"Working on telescope #{tel_id}..." + bcolors.ENDC)
self.image_cutflow.count("noCuts")
camera = source.subarray.tel[tel_id].camera.geometry
# count the current telescope according to its type
tel_type = str(source.subarray.tel[tel_id])
n_tels[tel_type] += 1
# use ctapipe's functionality to get the calibrated image
# and scale the reconstructed values if required
pmt_signal = event.dl1.tel[tel_id].image / self.calibscale
# If required...
if save_images is True:
# Save the simulated and reconstructed image of the event
dl1_phe_image[tel_id] = pmt_signal
mc_phe_image[tel_id] = event.simulation.tel[
tel_id].true_image
# We now ASSUME that the event will be good
good_for_reco[tel_id] = 1
# later we change to 0 if any condition is NOT satisfied
if self.cleaner_reco.mode == "tail": # tail uses only ctapipe
# Cleaning used for direction reconstruction
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
# calculate the leakage (before filtering)
leakages = {} # this is needed by both cleanings
# The check on SIZE shouldn't be here, but for the moment
# I prefer to sacrifice elegancy...
if np.sum(image_biggest[mask_reco]) != 0.0:
leakage_biggest = leakage_parameters(
camera, image_biggest, mask_reco)
leakages["leak1_reco"] = leakage_biggest[
"intensity_width_1"]
leakages["leak2_reco"] = leakage_biggest[
"intensity_width_2"]
else:
leakages["leak1_reco"] = 0.0
leakages["leak2_reco"] = 0.0
# find all islands using this cleaning
num_islands, labels = number_of_islands(camera, mask_reco)
if num_islands == 1: # if only ONE islands is left ...
# ...use directly the old mask and reduce dimensions
# to make Hillas parametrization faster
camera_biggest = camera[mask_reco]
image_biggest = image_biggest[mask_reco]
if save_images is True:
dl1_phe_image_mask_reco[tel_id] = mask_reco
elif num_islands > 1: # if more islands survived..
# ...find the biggest one
mask_biggest = largest_island(labels)
# and also reduce dimensions
camera_biggest = camera[mask_biggest]
image_biggest = image_biggest[mask_biggest]
if save_images is True:
dl1_phe_image_mask_reco[tel_id] = mask_biggest
else: # if no islands survived use old camera and image
camera_biggest = camera
dl1_phe_image_mask_reco[tel_id] = mask_reco
# Cleaning used for score/energy estimation
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
dl1_phe_image_mask_clusters[tel_id] = mask_extended
# calculate the leakage (before filtering)
# this part is not well coded, but for the moment it works
if np.sum(image_extended[mask_extended]) != 0.0:
leakage_extended = leakage_parameters(
camera, image_extended, mask_extended)
leakages["leak1"] = leakage_extended[
"intensity_width_1"]
leakages["leak2"] = leakage_extended[
"intensity_width_2"]
else:
leakages["leak1"] = 0.0
leakages["leak2"] = 0.0
# find all islands with this cleaning
# we will also register how many have been found
n_cluster_dict[tel_id], labels = number_of_islands(
camera, mask_extended)
# NOTE: the next check shouldn't be necessary if we keep
# all the isolated pixel clusters, but for now the
# two cleanings are set the same in analysis.yml because
# the performance of the extended one has never been really
# studied in model estimation.
# if some islands survived
if n_cluster_dict[tel_id] > 0:
# keep all of them and reduce dimensions
camera_extended = camera[mask_extended]
image_extended = image_extended[mask_extended]
else: # otherwise continue with the old camera and image
camera_extended = camera
# could this go into `hillas_parameters` ...?
# this is basically the charge of ALL islands
# not calculated later by the Hillas parametrization!
max_signals[tel_id] = np.max(image_extended)
else: # for wavelets we stick to old pywi-cta code
try: # "try except FileNotFoundError" not clear to me, but for now it stays...
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.camera_name)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.camera_name)
# Image for score/energy estimation (with clusters)
(
image_extended,
mask_extended,
) = self.cleaner_extended.clean_image(
pmt_signal, camera)
# This last part was outside the pywi-cta block
# before, but is indeed part of it because it uses
# pywi-cta functions in the "extended" case
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.camera_name)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
except FileNotFoundError as e:
print(e)
continue
# =============================================================
# PRELIMINARY IMAGE SELECTION
# =============================================================
cleaned_image_is_good = True # we assume this
if self.image_selection_source == "extended":
cleaned_image_to_use = image_extended
elif self.image_selection_source == "biggest":
cleaned_image_to_use = image_biggest
else:
raise ValueError(
"Only supported cleanings are 'biggest' or 'extended'."
)
# Apply some selection
if self.image_cutflow.cut("min pixel", cleaned_image_to_use):
if debug:
print(bcolors.WARNING +
"WARNING : not enough pixels!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
cleaned_image_is_good = False
if self.image_cutflow.cut("min charge",
np.sum(cleaned_image_to_use)):
if debug:
print(bcolors.WARNING +
"WARNING : not enough charge!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
cleaned_image_is_good = False
if debug and (not cleaned_image_is_good): # BAD image quality
print(bcolors.WARNING +
"WARNING : The cleaned image didn't pass" +
" preliminary cuts.\n" +
"An attempt to parametrize it will be made," +
" but the image will NOT be used for" +
" direction reconstruction." + bcolors.ENDC)
# =============================================================
# IMAGE PARAMETRIZATION
# =============================================================
with np.errstate(invalid="raise", divide="raise"):
try:
# Filter the cameras in TelescopeFrame with the same
# cleaning masks
camera_biggest_tel = geom_cam_tel[camera.camera_name][
camera_biggest.pix_id]
camera_extended_tel = geom_cam_tel[camera.camera_name][
camera_extended.pix_id]
# Parametrize the image in the TelescopeFrame
moments_reco = hillas_parameters(
camera_biggest_tel,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera_extended_tel, image_extended
) # for discrimination and energy reconstruction
if debug:
print(
"Image parameters from main cluster cleaning:")
print(moments_reco)
print(
"Image parameters from all-clusters cleaning:")
print(moments)
# Add concentration parameters
concentrations = {}
concentrations_extended = concentration_parameters(
camera_extended_tel, image_extended, moments)
concentrations[
"concentration_cog"] = concentrations_extended[
"cog"]
concentrations[
"concentration_core"] = concentrations_extended[
"core"]
concentrations[
"concentration_pixel"] = concentrations_extended[
"pixel"]
# ===================================================
# PARAMETRIZED IMAGE SELECTION
# ===================================================
if self.image_selection_source == "extended":
moments_to_use = moments
else:
moments_to_use = moments_reco
# if width and/or length are zero (e.g. when there is
# only only one pixel or when all pixel are exactly
# in one row), the parametrisation
# won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_to_use):
if debug:
print(bcolors.WARNING +
"WARNING : poor moments!" + bcolors.ENDC)
good_for_reco[tel_id] = 0 # we record it as BAD
if self.image_cutflow.cut("close to the edge",
moments_to_use,
camera.camera_name):
if debug:
print(
bcolors.WARNING +
"WARNING : out of containment radius!\n" +
f"Camera radius = {self.camera_radius[camera.camera_name]}\n"
+ f"COG radius = {moments_to_use.r}" +
bcolors.ENDC)
good_for_reco[tel_id] = 0
if self.image_cutflow.cut("bad ellipticity",
moments_to_use):
if debug:
print(bcolors.WARNING +
"WARNING : bad ellipticity" +
bcolors.ENDC)
good_for_reco[tel_id] = 0
if debug and good_for_reco[tel_id] == 1:
print(
bcolors.OKGREEN +
"Image survived and correctly parametrized."
# + "\nIt will be used for direction reconstruction!"
+ bcolors.ENDC)
elif debug and good_for_reco[tel_id] == 0:
print(
bcolors.WARNING + "Image not survived or " +
"not good enough for parametrization."
# + "\nIt will be NOT used for direction reconstruction, "
# + "BUT it's information will be recorded."
+ bcolors.ENDC)
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(
np.where(image_extended > 0)[0])
leakage_dict[tel_id] = leakages
concentration_dict[tel_id] = concentrations
except (
FloatingPointError,
HillasParameterizationError,
ValueError,
) as e:
if debug:
print(bcolors.FAIL + "Parametrization error: " +
f"{e}\n" + "Dummy parameters recorded." +
bcolors.ENDC)
good_for_reco[tel_id] = 0
hillas_dict[
tel_id] = HillasParametersTelescopeFrameContainer(
)
hillas_dict_reco[
tel_id] = HillasParametersTelescopeFrameContainer(
)
n_pixel_dict[tel_id] = len(
np.where(image_extended > 0)[0])
leakage_dict[tel_id] = leakages
concentration_dict[tel_id] = concentrations
# END OF THE CYCLE OVER THE TELESCOPES
# =============================================================
# DIRECTION RECONSTRUCTION
# =============================================================
if bad_LST_stereo:
if debug:
print(
bcolors.WARNING +
"WARNING: This event was triggered with 1 LST image and <2 images from other telescope types."
+
"\nWARNING : direction reconstruction will not be performed."
+ bcolors.ENDC)
# Set all the involved images as NOT good for recosntruction
# even though they might have been
# but this is because of the LST stereo trigger....
for tel_id in event.r0.tel.keys():
good_for_reco[tel_id] = 0
# and set the number of good and bad images accordingly
n_tels["GOOD images"] = 0
n_tels[
"BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"]
# create a dummy container for direction reconstruction
reco_result = ReconstructedShowerContainer()
if return_stub: # if saving all events (default)
if debug:
print(bcolors.OKBLUE + "Recording event..." +
bcolors.ENDC)
print(
bcolors.WARNING +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
continue
else:
continue
# Now in case the only triggered telescopes were
# - < self.min_ntel_LST LST,
# - >=2 any other telescope type,
# we remove the single-LST image and continue reconstruction with
# the images from the other telescope types
if self.LST_stereo and (n_triggered_LSTs < self.min_ntel_LST) and (
n_triggered_LSTs != 0) and (n_triggered_non_LSTs >= 2):
if debug:
print(
bcolors.WARNING +
f"WARNING: LST stereo trigger condition is active.\n" +
f"This event triggered < {self.min_ntel_LST} LSTs " +
f"and {n_triggered_non_LSTs} images from other telescope types.\n"
+ bcolors.ENDC)
for tel_id in triggered_LSTs: # in case we test for min_ntel_LST>2
if good_for_reco[tel_id]:
# we don't use it for reconstruction
good_for_reco[tel_id] = 0
print(
bcolors.WARNING +
f"WARNING: LST image #{tel_id} removed, even though it passed quality cuts."
+ bcolors.ENDC)
# TODO: book-keeping of this kind of events doesn't seem easy
# convert dictionary in numpy array to get a "mask"
images_status = np.asarray(list(good_for_reco.values()))
# record how many images will be used for reconstruction
n_tels["GOOD images"] = len(
np.extract(images_status == 1, images_status))
n_tels["BAD images"] = n_tels["Triggered"] - n_tels["GOOD images"]
if self.event_cutflow.cut("min2Tels reco", n_tels["GOOD images"]):
if debug:
print(
bcolors.FAIL +
f"WARNING: < {self.min_ntel} ({n_tels['GOOD images']}) images remaining!"
+
"\nWARNING : direction reconstruction is not possible!"
+ bcolors.ENDC)
# create a dummy container for direction reconstruction
reco_result = ReconstructedShowerContainer()
if return_stub: # if saving all events (default)
if debug:
print(bcolors.OKBLUE + "Recording event..." +
bcolors.ENDC)
print(
bcolors.WARNING +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
continue
else:
continue
if debug:
print(bcolors.OKBLUE + "Starting direction reconstruction..." +
bcolors.ENDC)
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
if self.image_selection_source == "extended":
hillas_dict_to_use = hillas_dict
else:
hillas_dict_to_use = hillas_dict_reco
# use only the successfully parametrized images
# to reconstruct the direction of this event
successfull_hillas = np.where(images_status == 1)[0]
all_images = np.asarray(list(good_for_reco.keys()))
good_images = set(all_images[successfull_hillas])
good_hillas_dict = {
k: v
for k, v in hillas_dict_to_use.items()
if k in good_images
}
if debug:
print(
bcolors.PURPLE +
f"{len(good_hillas_dict)} images " +
f"(from telescopes #{list(good_hillas_dict.keys())}) will be "
+ "used to recover the shower's direction..." +
bcolors.ENDC)
# Reconstruction results
reco_result = self.shower_reco.predict(
good_hillas_dict,
source.subarray,
SkyCoord(alt=alt, az=az, frame="altaz"),
None, # use the array direction
)
# Impact parameter for telescope-wise energy estimation
subarray = source.subarray
for tel_id in hillas_dict_to_use.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# Go back to the tilted frame
# this should be the same...
tel_tilted = tel_ground.transform_to(tilted_frame)
# but this not
core_tilted = SkyCoord(x=core_ground.x,
y=core_ground.y,
frame=tilted_frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_tilted.x - tel_tilted.x)**2 +
(core_tilted.y - tel_tilted.y)**2)
except (Exception, TooFewTelescopesException,
InvalidWidthException) as e:
if debug:
print("exception in reconstruction:", e)
raise
if return_stub:
if debug:
print(
bcolors.FAIL +
"Shower could NOT be correctly reconstructed! " +
"Recording event..." +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if debug:
print(bcolors.WARNING + "WARNING: undefined direction!" +
bcolors.ENDC)
if return_stub:
if debug:
print(
bcolors.FAIL +
"Shower could NOT be correctly reconstructed! " +
"Recording event..." +
"WARNING: This is event shall NOT be used further along the pipeline."
+ bcolors.ENDC)
yield stub( # record event with dummy info
event, mc_phe_image, dl1_phe_image,
dl1_phe_image_mask_reco, dl1_phe_image_mask_clusters,
good_for_reco, hillas_dict, hillas_dict_reco, n_tels,
leakage_dict, concentration_dict)
else:
continue
if debug:
print(bcolors.BOLDGREEN + "Shower correctly reconstructed! " +
"Recording event..." + bcolors.ENDC)
yield PreparedEvent(
event=event,
dl1_phe_image=dl1_phe_image,
dl1_phe_image_mask_reco=dl1_phe_image_mask_reco,
dl1_phe_image_mask_clusters=dl1_phe_image_mask_clusters,
mc_phe_image=mc_phe_image,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
leakage_dict=leakage_dict,
concentration_dict=concentration_dict,
n_tels=n_tels,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
good_event=True,
good_for_reco=good_for_reco,
)
integration_correction=integration_correction)
image = next(it)
input_image = image.input_image
# ## Plot the image example
# In[5]:
geom1d = geometry_converter.get_geom1d(cam_id)
plot_ctapipe_image(input_image, geom=geom1d, plot_axis=False)
# ## Plot the image example after thresholding
# In[6]:
THRESHOLD = 6
filtered_image = np.copy(input_image)
filtered_image[filtered_image < THRESHOLD] = 0
plot_ctapipe_image(filtered_image, geom=geom1d, plot_axis=False)
# ## Get the number of clusters
# In[7]:
input_image_2d = geometry_converter.image_1d_to_2d(input_image, cam_id)
num_clusters = pixel_clusters.number_of_pixels_clusters(array=input_image_2d,
threshold=THRESHOLD)
print("Number of clusters:", num_clusters)
def clean_image(
input_image,
type_of_filtering=hard_filter.DEFAULT_TYPE_OF_FILTERING,
filter_thresholds=hard_filter.DEFAULT_FILTER_THRESHOLDS,
last_scale_treatment=mrtransform_wrapper.DEFAULT_LAST_SCALE_TREATMENT,
detect_only_positive_structures=False,
kill_isolated_pixels=False,
noise_distribution=None,
tmp_files_directory=".",
output_data_dict=None,
**kwargs):
"""Clean the `input_image` image.
Apply the wavelet transform, filter planes and return the reverse
transformed image.
Parameters
----------
input_image : array_like
The image to clean.
type_of_filtering : str
Type of filtering: 'hard_filtering' or 'ksigma_hard_filtering'.
filter_thresholds : list of float
Thresholds used for the plane filtering.
last_scale_treatment : str
Last plane treatment: 'keep', 'drop' or 'mask'.
detect_only_positive_structures : bool
Detect only positive structures.
kill_isolated_pixels : bool
Suppress isolated pixels in the support.
noise_distribution : bool
The JSON file containing the Cumulated Distribution Function of the
noise model used to inject artificial noise in blank pixels (those
with a NaN value).
tmp_files_directory : str
The path of the directory where temporary files are written.
output_data_dict : dict
A dictionary used to return results and intermediate results.
Returns
-------
Return the cleaned image.
"""
if DEBUG:
print("Filter thresholds:", filter_thresholds)
number_of_scales = len(filter_thresholds) + 1
if DEBUG:
print("Number of scales:", number_of_scales)
# COMPUTE THE WAVELET TRANSFORM #######################################
wavelet_planes = wavelet_transform(input_image,
number_of_scales=number_of_scales,
tmp_files_directory=tmp_files_directory,
noise_distribution=noise_distribution)
if DEBUG:
for index, plane in enumerate(wavelet_planes):
images.plot(plane, "Plane " + str(index))
# FILTER WAVELET PLANES ###############################################
filtered_wavelet_planes = filter_planes(
wavelet_planes,
method=type_of_filtering,
thresholds=filter_thresholds,
detect_only_positive_structures=detect_only_positive_structures)
#if DEBUG:
# for index, plane in enumerate(filtered_wavelet_planes):
# images.plot(plane, "Filtered plane " + str(index))
# COMPUTE THE INVERSE TRANSFORM #######################################
cleaned_image = inverse_wavelet_transform(filtered_wavelet_planes,
last_plane=last_scale_treatment)
if DEBUG:
images.plot(cleaned_image, "Cleaned image")
# REMOVE ISOLATED PIXELS ##############################################
if output_data_dict is not None:
img_cleaned_clusters_delta_intensity, img_cleaned_clusters_delta_abs_intensity, img_cleaned_clusters_delta_num_pixels = filter_pixels_clusters_stats(
cleaned_image)
img_cleaned_num_islands = number_of_pixels_clusters(cleaned_image)
output_data_dict[
"img_cleaned_clusters_delta_intensity"] = img_cleaned_clusters_delta_intensity
output_data_dict[
"img_cleaned_clusters_delta_abs_intensity"] = img_cleaned_clusters_delta_abs_intensity
output_data_dict[
"img_cleaned_clusters_delta_num_pixels"] = img_cleaned_clusters_delta_num_pixels
output_data_dict["img_cleaned_num_islands"] = img_cleaned_num_islands
if kill_isolated_pixels:
cleaned_image = filter_pixels_clusters(cleaned_image)
if DEBUG:
images.plot(cleaned_image, "Cleaned image after cluster filtering")
return cleaned_image
def run(
self,
cleaning_function_params,
input_file_or_dir_path_list,
benchmark_method,
output_file_path,
plot=False,
saveplot=None,
ref_img_as_input=False, # A hack to easily produce CSV files...
max_num_img=None,
tel_id=None,
event_id=None,
cam_id=None,
debug=False,
rejection_criteria=None):
"""A convenient optional wrapper to simplify the image cleaning analysis.
Apply the image cleaning analysis on `input_file_or_dir_path_list`,
apply some pre-processing and post-processing procedures, collect and
return results, intermediate values and metadata.
Parameters
----------
cleaning_function_params
A dictionary containing the parameters required for the image
cleaning method.
input_file_or_dir_path_list
A list of file to clean. Can be a list of simtel files, fits files
or directories containing such files.
benchmark_method
The list of estimators to use to assess the image cleaning. If
`None`, images are cleaned but nothing is returned (can be used
with e.g. the `plot` and/or `saveplot` options).
output_file_path
The result file path (a JSON file).
plot
The result of each cleaning is plot if `True`.
saveplot
The result of each cleaning is saved if `True`.
ref_img_as_input
This option is a hack to easily produce a "flatten" CSV results
files.
max_num_img
The number of images to process among the input set
(`input_file_or_dir_path_list`).
debug
Stop the execution and print the full traceback when an exception
is encountered if this parameter is `True`. Report exceptions and
continue with the next input image if this parameter is `False`.
Returns
-------
dict
Results, intermediate values and metadata.
"""
launch_time = time.perf_counter()
if benchmark_method is not None:
io_list = [] # The list of returned dictionaries
if tel_id is not None:
tel_id = [tel_id]
if event_id is not None:
event_id = [event_id]
if cam_id is None:
raise ValueError('cam_id is now mandatory.')
if cam_id == "ASTRICam":
integrator_window_width = 1
integrator_window_shift = 1
elif cam_id == "CHEC":
integrator_window_width = 10
integrator_window_shift = 5
elif cam_id == "DigiCam":
integrator_window_width = 5
integrator_window_shift = 2
elif cam_id == "FlashCam":
integrator_window_width = 6
integrator_window_shift = 3
elif cam_id == "NectarCam":
integrator_window_width = 5
integrator_window_shift = 2
elif cam_id == "LSTCam":
integrator_window_width = 5
integrator_window_shift = 2
else:
raise ValueError('Unknown cam_id "{}"'.format(cam_id))
for image in image_generator(
input_file_or_dir_path_list,
max_num_images=max_num_img,
tel_filter_list=tel_id,
ev_filter_list=event_id,
cam_filter_list=[cam_id],
mc_rejection_criteria=rejection_criteria,
ctapipe_format=False,
integrator='LocalPeakIntegrator',
integrator_window_width=integrator_window_width,
integrator_window_shift=integrator_window_shift,
integration_correction=False,
mix_channels=True):
input_file_path = image.meta['file_path']
if self.verbose:
print(input_file_path)
# `image_dict` contains metadata (to be returned) on the current image
image_dict = {"input_file_path": input_file_path}
try:
# READ THE INPUT FILE #####################################
if self.verbose:
print("TEL{}_EV{}".format(image.meta["tel_id"],
image.meta["event_id"]))
reference_img = image.reference_image
pixels_position = image.pixels_position
if ref_img_as_input:
# This option is a hack to easily produce CSV files with
# the "null_ref" "cleaning" module...
input_img = copy.deepcopy(reference_img)
else:
input_img = image.input_image
image_dict.update(image.meta)
# Make the original 1D geom (required for the 2D to 1D geometry
# conversion, for Tailcut and for Hillas)
cam_id = image.meta['cam_id']
cleaning_function_params["cam_id"] = cam_id
geom1d = geometry_converter.get_geom1d(cam_id)
if benchmark_method is not None:
# FETCH ADDITIONAL IMAGE METADATA #####################
image_dict["img_ref_signal_to_border"] = signal_to_border(
reference_img) # TODO: NaN
image_dict[
"img_ref_signal_to_border_distance"] = signal_to_border_distance(
reference_img) # TODO: NaN
image_dict["img_ref_pemax_on_border"] = pemax_on_border(
reference_img) # TODO: NaN
delta_pe, delta_abs_pe, delta_num_pixels = filter_pixels_clusters_stats(
reference_img) # TODO: NaN
num_islands = number_of_pixels_clusters(
reference_img) # TODO: NaN
image_dict["img_ref_islands_delta_pe"] = delta_pe
image_dict["img_ref_islands_delta_abs_pe"] = delta_abs_pe
image_dict[
"img_ref_islands_delta_num_pixels"] = delta_num_pixels
image_dict["img_ref_num_islands"] = num_islands
image_dict["img_ref_sum_pe"] = float(
np.nansum(reference_img))
image_dict["img_ref_min_pe"] = float(
np.nanmin(reference_img))
image_dict["img_ref_max_pe"] = float(
np.nanmax(reference_img))
image_dict["img_ref_num_pix"] = int(
(reference_img[np.isfinite(reference_img)] > 0).sum())
image_dict["img_in_sum_pe"] = float(np.nansum(input_img))
image_dict["img_in_min_pe"] = float(np.nanmin(input_img))
image_dict["img_in_max_pe"] = float(np.nanmax(input_img))
image_dict["img_in_num_pix"] = int(
(input_img[np.isfinite(input_img)] > 0).sum())
reference_img1d = geometry_converter.image_2d_to_1d(
reference_img, cam_id)
try:
hillas_params_2_ref_img = get_hillas_parameters(
geom1d, reference_img1d,
HILLAS_IMPLEMENTATION) # TODO GEOM
except Exception as e:
hillas_params_2_ref_img = float('nan')
print(e)
#traceback.print_tb(e.__traceback__, file=sys.stdout)
image_dict["img_ref_hillas_2_size"] = float(
hillas_params_2_ref_img.intensity)
image_dict[
"img_ref_hillas_2_cen_x"] = hillas_params_2_ref_img.x.value
image_dict[
"img_ref_hillas_2_cen_y"] = hillas_params_2_ref_img.y.value
image_dict[
"img_ref_hillas_2_length"] = hillas_params_2_ref_img.length.value
image_dict[
"img_ref_hillas_2_width"] = hillas_params_2_ref_img.width.value
image_dict[
"img_ref_hillas_2_r"] = hillas_params_2_ref_img.r.value
image_dict[
"img_ref_hillas_2_phi"] = hillas_params_2_ref_img.phi.to(
u.rad).value
image_dict[
"img_ref_hillas_2_psi"] = hillas_params_2_ref_img.psi.to(
u.rad).value
try:
image_dict["img_ref_hillas_2_miss"] = float(
hillas_params_2_ref_img.miss.value)
except:
image_dict["img_ref_hillas_2_miss"] = None
image_dict[
"img_ref_hillas_2_kurtosis"] = hillas_params_2_ref_img.kurtosis
image_dict[
"img_ref_hillas_2_skewness"] = hillas_params_2_ref_img.skewness
# CLEAN THE INPUT IMAGE ###################################
# Copy the image (otherwise some cleaning functions like Tailcut may change it)
#input_img_copy = copy.deepcopy(input_img)
input_img_copy = input_img.astype('float64', copy=True)
cleaning_function_params["output_data_dict"] = {}
initial_time = time.perf_counter()
cleaned_img = self.clean_image(
input_img_copy, **cleaning_function_params) # TODO: NaN
full_clean_execution_time_sec = time.perf_counter(
) - initial_time
if benchmark_method is not None:
image_dict.update(
cleaning_function_params["output_data_dict"])
del cleaning_function_params["output_data_dict"]
# ASSESS OR PRINT THE CLEANED IMAGE #######################
if benchmark_method is not None:
# ASSESS THE CLEANING #################################
kwargs = {
'geom': geom1d,
'hillas_implementation': HILLAS_IMPLEMENTATION
} # TODO GEOM
score_tuple, score_name_tuple = assess.assess_image_cleaning(
input_img, cleaned_img, reference_img,
benchmark_method, **kwargs)
image_dict[
"img_cleaned_signal_to_border"] = signal_to_border(
cleaned_img)
image_dict[
"img_cleaned_signal_to_border_distance"] = signal_to_border_distance(
cleaned_img)
image_dict[
"img_cleaned_pemax_on_border"] = pemax_on_border(
cleaned_img)
image_dict["score"] = score_tuple
image_dict["score_name"] = score_name_tuple
image_dict[
"full_clean_execution_time_sec"] = full_clean_execution_time_sec
image_dict["img_cleaned_sum_pe"] = float(
np.nansum(cleaned_img))
image_dict["img_cleaned_min_pe"] = float(
np.nanmin(cleaned_img))
image_dict["img_cleaned_max_pe"] = float(
np.nanmax(cleaned_img))
image_dict["img_cleaned_num_pix"] = int(
(cleaned_img[np.isfinite(cleaned_img)] > 0).sum())
cleaned_img1d = geometry_converter.image_2d_to_1d(
cleaned_img, cam_id)
try:
hillas_params_2_cleaned_img = get_hillas_parameters(
geom1d, cleaned_img1d,
HILLAS_IMPLEMENTATION) # GEOM
except Exception as e:
hillas_params_2_cleaned_img = float('nan')
print(e)
#traceback.print_tb(e.__traceback__, file=sys.stdout)
try:
image_dict["img_cleaned_hillas_2_size"] = float(
hillas_params_2_cleaned_img.intensity)
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_size"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_cen_x"] = hillas_params_2_cleaned_img.x.value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_cen_x"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_cen_y"] = hillas_params_2_cleaned_img.y.value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_cen_y"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_length"] = hillas_params_2_cleaned_img.length.value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_length"] = float(
'nan')
try:
image_dict[
"img_cleaned_hillas_2_width"] = hillas_params_2_cleaned_img.width.value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_width"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_r"] = hillas_params_2_cleaned_img.r.value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_r"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_phi"] = hillas_params_2_cleaned_img.phi.to(
u.rad).value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_phi"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_psi"] = hillas_params_2_cleaned_img.psi.to(
u.rad).value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_psi"] = float('nan')
try:
image_dict["img_cleaned_hillas_2_miss"] = float(
hillas_params_2_cleaned_img.miss.value)
except:
image_dict["img_cleaned_hillas_2_miss"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_kurtosis"] = hillas_params_2_cleaned_img.kurtosis
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_kurtosis"] = float(
'nan')
try:
image_dict[
"img_cleaned_hillas_2_skewness"] = hillas_params_2_cleaned_img.skewness
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_skewness"] = float(
'nan')
# PLOT IMAGES #########################################################
if plot or (saveplot is not None):
image_list = [
geometry_converter.image_2d_to_1d(input_img, cam_id),
geometry_converter.image_2d_to_1d(
reference_img, cam_id),
geometry_converter.image_2d_to_1d(cleaned_img, cam_id)
]
title_list = [
"Input image", "Reference image", "Cleaned image"
]
geom_list = [geom1d, geom1d, geom1d]
hillas_list = [False, True, True]
if plot:
pywicta.io.images.plot_list(image_list,
geom_list=geom_list,
title_list=title_list,
hillas_list=hillas_list,
metadata_dict=image.meta)
if saveplot is not None:
basename, extension = os.path.splitext(saveplot)
plot_file_path = "{}_E{}_T{}{}".format(
basename, image.meta["event_id"],
image.meta["tel_id"], extension)
print("Saving {}".format(plot_file_path))
pywicta.io.images.mpl_save_list(
image_list,
geom_list=geom_list,
output_file_path=plot_file_path,
title_list=title_list,
hillas_list=hillas_list,
metadata_dict=image.meta)
except Exception as e:
print("Abort image {}: {} ({})".format(input_file_path, e,
type(e)))
if debug:
# The following line print the full trackback
traceback.print_tb(e.__traceback__, file=sys.stdout)
if benchmark_method is not None:
# http://docs.python.org/2/library/sys.html#sys.exc_info
exc_type, exc_value, exc_traceback = sys.exc_info(
) # most recent (if any) by default
'''
Reason this _can_ be bad: If an (unhandled) exception happens AFTER this,
or if we do not delete the labels on (not much) older versions of Py, the
reference we created can linger.
traceback.format_exc/print_exc do this very thing, BUT note this creates a
temp scope within the function.
'''
error_dict = {
'filename': exc_traceback.tb_frame.f_code.co_filename,
'lineno': exc_traceback.tb_lineno,
'name': exc_traceback.tb_frame.f_code.co_name,
'type': exc_type.__name__,
#'message' : exc_value.message
'message': str(e)
}
del (
exc_type, exc_value, exc_traceback
) # So we don't leave our local labels/objects dangling
# This still isn't "completely safe", though!
#error_dict = {"type": str(type(e)),
# "message": str(e)}
image_dict["error"] = error_dict
finally:
if benchmark_method is not None:
io_list.append(image_dict)
if benchmark_method is not None:
error_list = [
image_dict["error"] for image_dict in io_list
if "error" in image_dict
]
print("{} images aborted".format(len(error_list)))
# GENERAL EXPERIMENT METADATA
output_dict = {}
output_dict["benchmark_execution_time_sec"] = str(
time.perf_counter() - launch_time)
output_dict["date_time"] = str(datetime.datetime.now())
output_dict["class_name"] = self.__class__.__name__
output_dict["algo_code_ref"] = str(
self.__class__.clean_image.__code__)
output_dict["label"] = self.label
output_dict["cmd"] = " ".join(sys.argv)
output_dict["algo_params"] = cleaning_function_params
if "noise_distribution" in output_dict["algo_params"]:
del output_dict["algo_params"][
"noise_distribution"] # not JSON serializable...
output_dict["benchmark_method"] = benchmark_method
output_dict["system"] = " ".join(os.uname())
output_dict["io"] = io_list
if output_file_path is not None:
with open(output_file_path, "w") as fd:
json.dump(output_dict, fd, sort_keys=True,
indent=4) # pretty print format
return output_dict
def clean_image(self,
input_img,
high_threshold=10.,
low_threshold=8.,
pixels_clusters_filtering="off",
verbose=False,
cam_id=None,
output_data_dict=None,
**kwargs):
"""Apply ctapipe's tail-cut image cleaning on ``input_img``.
Note
----
The main difference with :mod:`ctapipe.image.cleaning.tailcuts_clean` is that here the cleaning function
takes 2D Numpy arrays.
Parameters
----------
input_img : array_like
The image to clean. Should be a **2D** Numpy array.
high_threshold : float
The *core threshold* (a.k.a. *picture threshold*).
low_threshold : float
The *boundary threshold*.
pixels_clusters_filtering : str
Defines the method used to remove isolated pixels after the tail-cut image cleaning.
Accepted values are: "off", "scipy" or "mars".
- "off": don't apply any filtering after the tail-cut image cleaning.
- "scipy": keep only the largest cluster of pixels after the tail-cut image cleaning.
See :mod:`pywi.processing.filtering.pixel_clusters` for more information.
- "mars": apply the same filtering than in CTA-Mars analysis, keep only *significant core pixels* that have
at least two others *significant core pixels* among its neighbors (a *significant core pixels* is a pixel
above the *core threshold*).
verbose : bool
Print additional messages if ``True``.
cam_id : str
The camera ID from which ``input_img`` came from: "ASTRICam", "CHEC", "DigiCam", "FlashCam", "NectarCam" or
"LSTCam".
output_data_dict : dict
An optional dictionary used to transmit internal information to the caller.
Returns
-------
array_like
The ``input_img`` after tail-cut image cleaning. This is a **2D** Numpy array (i.e. it should be converted
with :func:`pywicta.io.geometry_converter.image_2d_to_1d` before any usage in ctapipe).
"""
if cam_id is None:
raise Exception("cam_id have to be defined") # TODO
if not (pixels_clusters_filtering.lower() in ("off", "scipy", "mars")):
raise ValueError(
'pixels_clusters_filtering = {}. Accepted values are: "off", "scipy" or "mars".'
.format(pixels_clusters_filtering))
# If low_threshold > high_threshold then low_threshold = high_threshold
low_threshold = min(high_threshold, low_threshold)
# 2D ARRAY (FITS IMAGE) TO CTAPIPE IMAGE ###############
geom_1d = geometry_converter.get_geom1d(cam_id)
img_1d = geometry_converter.image_2d_to_1d(input_img, cam_id)
# APPLY TAILCUT CLEANING ##############################
if pixels_clusters_filtering.lower() == "mars":
if verbose:
print("Mars pixels clusters filtering")
mask = tailcuts_clean(geom_1d,
img_1d,
picture_thresh=high_threshold,
boundary_thresh=low_threshold,
keep_isolated_pixels=False,
min_number_picture_neighbors=2)
else:
mask = tailcuts_clean(geom_1d,
img_1d,
picture_thresh=high_threshold,
boundary_thresh=low_threshold,
keep_isolated_pixels=True)
img_1d[mask == False] = 0
# CTAPIPE IMAGE TO 2D ARRAY (FITS IMAGE) ###############
cleaned_img_2d = geometry_converter.image_1d_to_2d(img_1d, cam_id)
# KILL ISOLATED PIXELS #################################
img_cleaned_islands_delta_pe, img_cleaned_islands_delta_abs_pe, img_cleaned_islands_delta_num_pixels = filter_pixels_clusters_stats(
cleaned_img_2d)
img_cleaned_num_islands = number_of_pixels_clusters(cleaned_img_2d)
if output_data_dict is not None:
output_data_dict[
"img_cleaned_islands_delta_pe"] = img_cleaned_islands_delta_pe
output_data_dict[
"img_cleaned_islands_delta_abs_pe"] = img_cleaned_islands_delta_abs_pe
output_data_dict[
"img_cleaned_islands_delta_num_pixels"] = img_cleaned_islands_delta_num_pixels
output_data_dict[
"img_cleaned_num_islands"] = img_cleaned_num_islands
if pixels_clusters_filtering.lower() == "scipy":
if verbose:
print("Scipy pixels clusters filtering")
cleaned_img_2d = scipy_pixels_clusters_filtering(cleaned_img_2d)
return cleaned_img_2d
def prepare_event(self, source, return_stub=False, save_images=False):
for event in source:
self.event_cutflow.count("noCuts")
if self.event_cutflow.cut("min2Tels trig",
len(event.dl0.tels_with_data)):
if return_stub:
yield stub(event)
else:
continue
self.calib(event)
# telescope loop
tot_signal = 0
dl1_phe_image = None
mc_phe_image = None
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for direction reconstruction
hillas_dict = {} # for discrimination
n_tels = {
"tot": len(event.dl0.tels_with_data),
"LST_LST_LSTCam": 0,
"MST_MST_NectarCam": 0,
"SST": 0, # add later correct names when testing on Paranal
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
# Compute impact parameter in tilt system
run_array_direction = event.mcheader.run_array_direction
az, alt = run_array_direction[0], run_array_direction[1]
ground_frame = GroundFrame()
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count("noCuts")
camera = event.inst.subarray.tel[tel_id].camera
# count the current telescope according to its size
tel_type = str(event.inst.subarray.tel[tel_id])
# use ctapipe's functionality to get the calibrated image
pmt_signal = event.dl1.tel[tel_id].image
# Save the calibrated image after the gain has been chosen
# automatically by ctapipe, together with the simulated one
if save_images is True:
dl1_phe_image = pmt_signal
mc_phe_image = event.mc.tel[tel_id].photo_electron_image
if self.cleaner_reco.mode == "tail": # tail uses only ctapipe
# Cleaning used for direction reconstruction
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
# find all islands using this cleaning
num_islands, labels = number_of_islands(camera, mask_reco)
if num_islands == 1: # if only ONE islands is left ...
# ...use directly the old mask and reduce dimensions
# to make Hillas parametrization faster
camera_biggest = camera[mask_reco]
image_biggest = image_biggest[mask_reco]
elif num_islands > 1: # if more islands survived..
# ...find the biggest one
mask_biggest = largest_island(labels)
# and also reduce dimensions
camera_biggest = camera[mask_biggest]
image_biggest = image_biggest[mask_biggest]
else: # if no islands survived use old camera and image
camera_biggest = camera
# Cleaning used for score/energy estimation
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
# find all islands with this cleaning
# we will also register how many have been found
n_cluster_dict[tel_id], labels = number_of_islands(
camera, mask_extended)
# NOTE: the next check shouldn't be necessary if we keep
# all the isolated pixel clusters, but for now the
# two cleanings are set the same in analysis.yml because
# the performance of the extended one has never been really
# studied in model estimation.
# (This is a nice way to ask for volunteers :P)
# if some islands survived
if num_islands > 0:
# keep all of them and reduce dimensions
camera_extended = camera[mask_extended]
image_extended = image_extended[mask_extended]
else: # otherwise continue with the old camera and image
camera_extended = camera
# could this go into `hillas_parameters` ...?
# this is basically the charge of ALL islands
# not calculated later by the Hillas parametrization!
max_signals[tel_id] = np.max(image_extended)
else: # for wavelets we stick to old pywi-cta code
try: # "try except FileNotFoundError" not clear to me, but for now it stays...
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest, mask_reco = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.cam_id)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.cam_id)
# Image for score/energy estimation (with clusters)
image_extended, mask_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
# This last part was outside the pywi-cta block
# before, but is indeed part of it because it uses
# pywi-cta functions in the "extended" case
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.cam_id)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
except FileNotFoundError as e: # JLK, WHAT?
print(e)
continue
# ==============================================================
# Apply some selection
if self.image_cutflow.cut("min pixel", image_biggest):
continue
if self.image_cutflow.cut("min charge", np.sum(image_biggest)):
continue
# do the hillas reconstruction of the images
# QUESTION should this change in numpy behaviour be done here
# or within `hillas_parameters` itself?
# JLK: make selection on biggest cluster
with np.errstate(invalid="raise", divide="raise"):
try:
moments_reco = hillas_parameters(
camera_biggest,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera_extended, image_extended
) # for discrimination and energy reconstruction
# if width and/or length are zero (e.g. when there is
# only only one pixel or when all pixel are exactly
# in one row), the parametrisation
# won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_reco):
continue
if self.image_cutflow.cut("close to the edge",
moments_reco, camera.cam_id):
continue
if self.image_cutflow.cut("bad ellipticity",
moments_reco):
continue
except (FloatingPointError,
hillas.HillasParameterizationError):
continue
point_azimuth_dict[
tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
point_altitude_dict[
tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
n_tels[tel_type] += 1
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0])
tot_signal += moments.intensity
n_tels["reco"] = len(hillas_dict_reco)
n_tels["discri"] = len(hillas_dict)
if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]):
if return_stub:
yield stub(event)
else:
continue
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Reconstruction results
reco_result = self.shower_reco.predict(
hillas_dict_reco,
event.inst,
SkyCoord(alt=alt, az=az, frame="altaz"),
{
tel_id: SkyCoord(
alt=point_altitude_dict[tel_id],
az=point_azimuth_dict[tel_id],
frame="altaz",
) # cycle only on tels which still have an image
for tel_id in point_altitude_dict.keys()
},
)
# Impact parameter for energy estimation (/ tel)
subarray = event.inst.subarray
for tel_id in hillas_dict.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# Should be better handled (tilted frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_ground.x - tel_ground.x)**2 +
(core_ground.y - tel_ground.y)**2)
except Exception as e:
print("exception in reconstruction:", e)
raise
if return_stub:
yield stub(event)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if return_stub:
yield stub(event)
else:
continue
yield PreparedEvent(
event=event,
dl1_phe_image=dl1_phe_image,
mc_phe_image=mc_phe_image,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
n_tels=n_tels,
tot_signal=tot_signal,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
)
def prepare_event(self, source, return_stub=False):
# configuration for the camera calibrator
# modifies the integration window to be more like in MARS
# JLK, only for LST!!!!
# Option for integration correction is done above
cfg = Config()
cfg["ChargeExtractorFactory"]["window_width"] = 5
cfg["ChargeExtractorFactory"]["window_shift"] = 2
self.calib = CameraCalibrator(
config=cfg,
extractor_product="LocalPeakIntegrator",
eventsource=source,
tool=None,
)
for event in source:
self.event_cutflow.count("noCuts")
if self.event_cutflow.cut("min2Tels trig",
len(event.dl0.tels_with_data)):
if return_stub:
yield stub(event)
else:
continue
# calibrate the event
self.calib.calibrate(event)
# telescope loop
tot_signal = 0
max_signals = {}
n_pixel_dict = {}
hillas_dict_reco = {} # for geometry
hillas_dict = {} # for discrimination
n_tels = {
"tot": len(event.dl0.tels_with_data),
"LST": 0,
"MST": 0,
"SST": 0,
}
n_cluster_dict = {}
impact_dict_reco = {} # impact distance measured in tilt system
point_azimuth_dict = {}
point_altitude_dict = {}
# To compute impact parameter in tilt system
run_array_direction = event.mcheader.run_array_direction
az, alt = run_array_direction[0], run_array_direction[1]
ground_frame = GroundFrame()
for tel_id in event.dl0.tels_with_data:
self.image_cutflow.count("noCuts")
camera = event.inst.subarray.tel[tel_id].camera
# count the current telescope according to its size
tel_type = event.inst.subarray.tel[tel_id].optics.tel_type
# JLK, N telescopes before cut selection are not really interesting for
# discrimination, too much fluctuations
# n_tels[tel_type] += 1
# the camera image as a 1D array and stuff needed for calibration
# Choose gain according to pywicta's procedure
image_1d = simtel_event_to_images(event=event,
tel_id=tel_id,
ctapipe_format=True)
pmt_signal = image_1d.input_image # calibrated image
# clean the image
try:
with warnings.catch_warnings():
# Image with biggest cluster (reco cleaning)
image_biggest = self.cleaner_reco.clean_image(
pmt_signal, camera)
image_biggest2d = geometry_converter.image_1d_to_2d(
image_biggest, camera.cam_id)
image_biggest2d = filter_pixels_clusters(
image_biggest2d)
image_biggest = geometry_converter.image_2d_to_1d(
image_biggest2d, camera.cam_id)
# Image for score/energy estimation (with clusters)
image_extended = self.cleaner_extended.clean_image(
pmt_signal, camera)
except FileNotFoundError as e: # JLK, WHAT?
print(e)
continue
# Apply some selection
if self.image_cutflow.cut("min pixel", image_biggest):
continue
if self.image_cutflow.cut("min charge", np.sum(image_biggest)):
continue
# For cluster counts
image_2d = geometry_converter.image_1d_to_2d(
image_extended, camera.cam_id)
n_cluster_dict[
tel_id] = pixel_clusters.number_of_pixels_clusters(
array=image_2d, threshold=0)
# could this go into `hillas_parameters` ...?
max_signals[tel_id] = np.max(image_extended)
# do the hillas reconstruction of the images
# QUESTION should this change in numpy behaviour be done here
# or within `hillas_parameters` itself?
# JLK: make selection on biggest cluster
with np.errstate(invalid="raise", divide="raise"):
try:
moments_reco = hillas_parameters(
camera,
image_biggest) # for geometry (eg direction)
moments = hillas_parameters(
camera, image_extended
) # for discrimination and energy reconstruction
# if width and/or length are zero (e.g. when there is only only one
# pixel or when all pixel are exactly in one row), the
# parametrisation won't be very useful: skip
if self.image_cutflow.cut("poor moments",
moments_reco):
# print('poor moments')
continue
if self.image_cutflow.cut("close to the edge",
moments_reco, camera.cam_id):
# print('close to the edge')
continue
if self.image_cutflow.cut("bad ellipticity",
moments_reco):
# print('bad ellipticity: w={}, l={}'.format(moments_reco.width, moments_reco.length))
continue
except (FloatingPointError,
hillas.HillasParameterizationError):
continue
point_azimuth_dict[
tel_id] = event.mc.tel[tel_id].azimuth_raw * u.rad
point_altitude_dict[
tel_id] = event.mc.tel[tel_id].altitude_raw * u.rad
n_tels[tel_type] += 1
hillas_dict[tel_id] = moments
hillas_dict_reco[tel_id] = moments_reco
n_pixel_dict[tel_id] = len(np.where(image_extended > 0)[0])
tot_signal += moments.intensity
n_tels["reco"] = len(hillas_dict_reco)
n_tels["discri"] = len(hillas_dict)
if self.event_cutflow.cut("min2Tels reco", n_tels["reco"]):
if return_stub:
yield stub(event)
else:
continue
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# Reconstruction results
reco_result = self.shower_reco.predict(
hillas_dict_reco,
event.inst,
point_altitude_dict,
point_azimuth_dict,
)
# shower_sys = TiltedGroundFrame(pointing_direction=HorizonFrame(
# az=reco_result.az,
# alt=reco_result.alt
# ))
# Impact parameter for energy estimation (/ tel)
subarray = event.inst.subarray
for tel_id in hillas_dict.keys():
pos = subarray.positions[tel_id]
tel_ground = SkyCoord(pos[0],
pos[1],
pos[2],
frame=ground_frame)
# tel_tilt = tel_ground.transform_to(shower_sys)
core_ground = SkyCoord(
reco_result.core_x,
reco_result.core_y,
0 * u.m,
frame=ground_frame,
)
# core_tilt = core_ground.transform_to(shower_sys)
# Should be better handled (tilted frame)
impact_dict_reco[tel_id] = np.sqrt(
(core_ground.x - tel_ground.x)**2 +
(core_ground.y - tel_ground.y)**2)
except Exception as e:
print("exception in reconstruction:", e)
raise
if return_stub:
yield stub(event)
else:
continue
if self.event_cutflow.cut("direction nan", reco_result):
if return_stub:
yield stub(event)
else:
continue
yield PreparedEvent(
event=event,
n_pixel_dict=n_pixel_dict,
hillas_dict=hillas_dict,
hillas_dict_reco=hillas_dict_reco,
n_tels=n_tels,
tot_signal=tot_signal,
max_signals=max_signals,
n_cluster_dict=n_cluster_dict,
reco_result=reco_result,
impact_dict=impact_dict_reco,
)
def clean_image(
self,
input_img,
type_of_multiresolution_transform=None,
type_of_filters=None,
type_of_non_orthog_filters=None,
number_of_scales=None,
suppress_last_scale=False,
suppress_isolated_pixels=False,
kill_isolated_pixels=False,
coef_detection_method=None,
k_sigma_noise_threshold=None,
noise_model=None,
detect_only_positive_structure=False,
suppress_positivity_constraint=False,
type_of_filtering=None,
first_detection_scale=None,
number_of_iterations=None,
epsilon=None,
support_file_name=None,
precision=None,
mask_file_path=None,
offset_after_calibration=None,
correction_offset=False,
input_image_scale='linear',
noise_distribution=None,
verbose=False,
raw_option_string=None,
tmp_files_directory=".", # "/Volumes/ramdisk"
mrfilter_directory=None, # "/Volumes/ramdisk"
output_data_dict=None,
**kwargs):
"""Clean the `input_img` image.
Apply the wavelet transform, filter planes and return the reverse
transformed image.
Parameters
----------
input_img : array_like
The input image to transform.
number_of_scales : int
The number of scales used to transform `input_image` or in other words
the number of wavelet planes returned.
tmp_files_directory : str
The path of the directory used to store mr_transform temporary data.
The default is the current directory, but it may be more appropriate to
specify here the path of a directory mounted in a ramdisk to speedup
I/Os ("/Volumes/ramdisk" on MacOSX or "/dev/shm" on Linux).
noise_distribution : EmpiricalDistribution
The noise distribution used to fill 'empty' NaN pixels with the
appropriate random noise distribution. If none, NaN pixels are fill
with zeros (which may add unwanted harmonics in wavelet planes).
Returns
-------
array_like
Return the cleaned image.
Raises
------
WrongDimensionError
If `cleaned_img` is not a 2D array.
"""
input_img = input_img.copy()
if input_img.ndim != 2:
raise WrongDimensionError()
input_file_path = os.path.join(
tmp_files_directory,
".tmp_{}_{}_in.fits".format(os.getpid(), time.time()))
mr_output_file_path = os.path.join(
tmp_files_directory,
".tmp_{}_{}_out.fits".format(os.getpid(), time.time()))
if output_data_dict is not None:
output_data_dict["mr_input_tmp_file_path"] = input_file_path
output_data_dict["mr_output_tmp_file_path"] = mr_output_file_path
if (output_data_dict is not None) and (mask_file_path is not None):
output_data_dict["mr_mask_file_path"] = mask_file_path
# INJECT NOISE IN NAN ##################################
# See https://stackoverflow.com/questions/29365194/replacing-missing-values-with-random-in-a-numpy-array
nan_mask = images.fill_nan_pixels(input_img, noise_distribution)
# APPLY AN OFFSET ######################################
if offset_after_calibration is not None:
if verbose:
print("Apply an offset after calibration:",
offset_after_calibration)
input_img = input_img + offset_after_calibration
# CHANGE THE SCALE #####################################
if input_image_scale == 'log':
if verbose:
print("Apply log scale")
#images.plot(input_img)
input_img = np.log10(
input_img) # TODO: it creates NaN values where pixels <= 0
#images.plot(input_img)
elif input_image_scale == 'sqrt':
if verbose:
print("Apply sqrt scale")
#images.plot(input_img)
input_img = np.sqrt(
input_img) # TODO: it creates NaN values where pixels < 0
#images.plot(input_img)
# WRITE THE INPUT FILE (FITS) ##########################
try:
initial_time = time.perf_counter()
images.save_fits(input_img, input_file_path)
exec_time_sec = time.perf_counter() - initial_time
if output_data_dict is not None:
output_data_dict["save_tmp_file_time_sec"] = exec_time_sec
except:
print("Error on input FITS file:", input_file_path)
raise
# EXECUTE MR_FILTER ####################################
# TODO: improve the following lines
if mrfilter_directory is None:
cmd = 'mr_filter'
else:
cmd = os.path.join(mrfilter_directory, 'mr_filter')
if raw_option_string is None:
cmd += ' -t{}'.format(
type_of_multiresolution_transform
) if type_of_multiresolution_transform is not None else ''
cmd += ' -T{}'.format(
type_of_filters) if type_of_filters is not None else ''
cmd += ' -U{}'.format(
type_of_non_orthog_filters
) if type_of_non_orthog_filters is not None else ''
cmd += ' -n{}'.format(
number_of_scales) if number_of_scales is not None else ''
cmd += ' -K' if suppress_last_scale else ''
cmd += ' -k' if suppress_isolated_pixels else '' # You should use scipy implementation instead (pywi.processing.filtering.pixel_clusters.filter_pixels_clusters); it's much more efficient
cmd += ' -C{}'.format(
coef_detection_method
) if coef_detection_method is not None else ''
cmd += ' -s{}'.format(
k_sigma_noise_threshold
) if k_sigma_noise_threshold is not None else ''
cmd += ' -m{}'.format(
noise_model) if noise_model is not None else ''
cmd += ' -p' if detect_only_positive_structure else ''
cmd += ' -P' if suppress_positivity_constraint else ''
cmd += ' -f{}'.format(
type_of_filtering) if type_of_filtering is not None else ''
cmd += ' -F{}'.format(
first_detection_scale
) if first_detection_scale is not None else ''
cmd += ' -i{}'.format(number_of_iterations
) if number_of_iterations is not None else ''
cmd += ' -e{}'.format(epsilon) if epsilon is not None else ''
cmd += ' -w{}'.format(
support_file_name) if support_file_name is not None else ''
cmd += ' -E{}'.format(precision) if precision is not None else ''
cmd += ' -I {}'.format(
mask_file_path) if mask_file_path is not None else ''
cmd += ' -v' if verbose else ''
else:
cmd += ' ' + raw_option_string
#self.label = "WT ({})".format(cmd) # Name to show in plots
cmd += ' "{}" "{}"'.format(input_file_path, mr_output_file_path)
#cmd = 'mr_filter -K -k -C1 -s3 -m3 -n{} "{}" {}'.format(number_of_scales, input_file_path, mr_output_file_path)
#cmd = 'mr_filter -K -k -C1 -s3 -m2 -p -P -n{} "{}" {}'.format(number_of_scales, input_file_path, mr_output_file_path)
if verbose:
print()
print(cmd)
else:
cmd += ' > /dev/null'
try:
initial_time = time.perf_counter()
os.system(cmd)
exec_time_sec = time.perf_counter() - initial_time
if output_data_dict is not None:
output_data_dict["mrfilter_cmd_exec_time_sec"] = exec_time_sec
except:
print("Error on command:", cmd)
raise
# READ THE MR_FILTER OUTPUT FILE #######################
try:
initial_time = time.perf_counter()
cleaned_img = images.load_fits(mr_output_file_path, 0)
exec_time_sec = time.perf_counter() - initial_time
if output_data_dict is not None:
output_data_dict["load_tmp_file_time_sec"] = exec_time_sec
except:
print("Error on output FITS file:", mr_output_file_path)
raise
# REMOVE FITS FILES ####################################
os.remove(input_file_path)
os.remove(mr_output_file_path)
# CHECK RESULT #########################################
if cleaned_img.ndim != 2:
raise WrongDimensionError()
# INJECT NOISE IN NAN: PUT BACK NAN VALUES #############
cleaned_img[nan_mask] = np.nan
# CHANGE THE SCALE #####################################
if input_image_scale == 'log':
if verbose:
print("Invert log scale")
cleaned_img = np.power(10., cleaned_img)
elif input_image_scale == 'sqrt':
if verbose:
print("Invert sqrt scale")
cleaned_img = np.power(2., cleaned_img)
# INVERT THE OFFSET ####################################
if (offset_after_calibration
is not None) and (not suppress_last_scale):
cleaned_img = cleaned_img - offset_after_calibration
# CORRECTION OFFSET ####################################
if correction_offset:
if verbose:
print("Apply a correction offset after cleaning")
cleaned_img = cleaned_img - np.nanmin(cleaned_img)
cleaned_img[np.isfinite(cleaned_img)
& (cleaned_img < 1.0
)] = 0. # May genereate warnings on NaN values
# KILL ISOLATED PIXELS #################################
img_cleaned_islands_delta_pe, img_cleaned_islands_delta_abs_pe, img_cleaned_islands_delta_num_pixels = filter_pixels_clusters_stats(
cleaned_img)
img_cleaned_num_islands = number_of_pixels_clusters(cleaned_img)
if output_data_dict is not None:
output_data_dict[
"img_cleaned_islands_delta_pe"] = img_cleaned_islands_delta_pe
output_data_dict[
"img_cleaned_islands_delta_abs_pe"] = img_cleaned_islands_delta_abs_pe
output_data_dict[
"img_cleaned_islands_delta_num_pixels"] = img_cleaned_islands_delta_num_pixels
output_data_dict[
"img_cleaned_num_islands"] = img_cleaned_num_islands
if kill_isolated_pixels:
if verbose:
print("Kill isolated pixels")
initial_time = time.perf_counter()
cleaned_img = scipy_kill_isolated_pixels(cleaned_img)
exec_time_sec = time.perf_counter() - initial_time
if output_data_dict is not None:
output_data_dict[
"scipy_kill_isolated_pixels_time_sec"] = exec_time_sec
#print(cleaned_img)
#images.plot_hist(cleaned_img)
#images.plot_hist(cleaned_img, num_bins=500, x_max=5)
return cleaned_img