def __init__(self,
cam_id,
min_npe=50,
max_npe=float('inf'),
min_radius_meters=0,
max_radius_meters=None,
min_ellipticity=0.1,
max_ellipticity=0.6,
min_num_pixels=3):
if max_radius_meters is None:
RADIUS_CONSTRAINT_RATIO = 0.8
if cam_id == "ASTRICam":
average_camera_radius_degree = 4.67
foclen_meters = 2.15
elif cam_id == "CHEC":
average_camera_radius_degree = 3.93
foclen_meters = 2.283
elif cam_id == "DigiCam":
average_camera_radius_degree = 4.56
foclen_meters = 5.59
elif cam_id == "FlashCam":
average_camera_radius_degree = 3.95
foclen_meters = 16.0
elif cam_id == "NectarCam":
average_camera_radius_degree = 4.05
foclen_meters = 16.0
elif cam_id == "LSTCam":
average_camera_radius_degree = 2.31
foclen_meters = 28.0
else:
raise ValueError('Unknown camid', cam_id)
average_camera_radius_meters = math.tan(
math.radians(average_camera_radius_degree)) * foclen_meters
max_radius_meters = RADIUS_CONSTRAINT_RATIO * average_camera_radius_meters
self.cam_id = cam_id
self.geom1d = geometry_converter.get_geom1d(self.cam_id)
self.hillas_implementation = 2
self.min_npe = min_npe
self.max_npe = max_npe
self.min_radius = min_radius_meters
self.max_radius = max_radius_meters
self.min_ellipticity = min_ellipticity
self.max_ellipticity = max_ellipticity
self.min_num_pixels = min_num_pixels
integrator_window_width=integrator_window_width,
integrator_window_shift=integrator_window_shift,
integrator_t0=integrator_t0,
integrator_sig_amp_cut_hg=integrator_sig_amp_cut_hg,
integrator_sig_amp_cut_lg=integrator_sig_amp_cut_lg,
integrator_lwt=integrator_lwt,
integration_correction=integration_correction)
image = next(it)
input_image = image.input_image
# ## Plot the image example
# In[5]:
geom1d = geometry_converter.get_geom1d(cam_id)
plot_ctapipe_image(input_image, geom=geom1d, plot_axis=False)
# ## Plot the image example after thresholding
# In[6]:
THRESHOLD = 6
filtered_image = np.copy(input_image)
filtered_image[filtered_image < THRESHOLD] = 0
plot_ctapipe_image(filtered_image, geom=geom1d, plot_axis=False)
# ## Get the number of clusters
# In[7]:
integrator_window_shift=integrator_window_shift,
integrator_t0=integrator_t0,
integrator_sig_amp_cut_hg=integrator_sig_amp_cut_hg,
integrator_sig_amp_cut_lg=integrator_sig_amp_cut_lg,
integrator_lwt=integrator_lwt,
integration_correction=integration_correction)
image = next(it) # This image is useless...
image = next(it) # This image is useless...
image = next(it) # This image is useless...
image = next(it)
###############################################################################
# Plot the selected image with NSB.
geom1d = geometry_converter.get_geom1d(image.meta['cam_id'])
title_str = "{} (run {}, event {}, tel {}, {:0.2f} {})".format(
image.meta['cam_id'], image.meta['run_id'], image.meta['event_id'],
image.meta['tel_id'], image.meta['mc_energy'][0],
image.meta['mc_energy'][1])
plot_ctapipe_image(image.input_image,
geom=geom1d,
plot_axis=False,
title=title_str)
plt.show()
###############################################################################
# Plot the selected image with NSB after the geometric transformation.
def run(
self,
cleaning_function_params,
input_file_or_dir_path_list,
benchmark_method,
output_file_path,
plot=False,
saveplot=None,
ref_img_as_input=False, # A hack to easily produce CSV files...
max_num_img=None,
tel_id=None,
event_id=None,
cam_id=None,
debug=False,
rejection_criteria=None):
"""A convenient optional wrapper to simplify the image cleaning analysis.
Apply the image cleaning analysis on `input_file_or_dir_path_list`,
apply some pre-processing and post-processing procedures, collect and
return results, intermediate values and metadata.
Parameters
----------
cleaning_function_params
A dictionary containing the parameters required for the image
cleaning method.
input_file_or_dir_path_list
A list of file to clean. Can be a list of simtel files, fits files
or directories containing such files.
benchmark_method
The list of estimators to use to assess the image cleaning. If
`None`, images are cleaned but nothing is returned (can be used
with e.g. the `plot` and/or `saveplot` options).
output_file_path
The result file path (a JSON file).
plot
The result of each cleaning is plot if `True`.
saveplot
The result of each cleaning is saved if `True`.
ref_img_as_input
This option is a hack to easily produce a "flatten" CSV results
files.
max_num_img
The number of images to process among the input set
(`input_file_or_dir_path_list`).
debug
Stop the execution and print the full traceback when an exception
is encountered if this parameter is `True`. Report exceptions and
continue with the next input image if this parameter is `False`.
Returns
-------
dict
Results, intermediate values and metadata.
"""
launch_time = time.perf_counter()
if benchmark_method is not None:
io_list = [] # The list of returned dictionaries
if tel_id is not None:
tel_id = [tel_id]
if event_id is not None:
event_id = [event_id]
if cam_id is None:
raise ValueError('cam_id is now mandatory.')
if cam_id == "ASTRICam":
integrator_window_width = 1
integrator_window_shift = 1
elif cam_id == "CHEC":
integrator_window_width = 10
integrator_window_shift = 5
elif cam_id == "DigiCam":
integrator_window_width = 5
integrator_window_shift = 2
elif cam_id == "FlashCam":
integrator_window_width = 6
integrator_window_shift = 3
elif cam_id == "NectarCam":
integrator_window_width = 5
integrator_window_shift = 2
elif cam_id == "LSTCam":
integrator_window_width = 5
integrator_window_shift = 2
else:
raise ValueError('Unknown cam_id "{}"'.format(cam_id))
for image in image_generator(
input_file_or_dir_path_list,
max_num_images=max_num_img,
tel_filter_list=tel_id,
ev_filter_list=event_id,
cam_filter_list=[cam_id],
mc_rejection_criteria=rejection_criteria,
ctapipe_format=False,
integrator='LocalPeakIntegrator',
integrator_window_width=integrator_window_width,
integrator_window_shift=integrator_window_shift,
integration_correction=False,
mix_channels=True):
input_file_path = image.meta['file_path']
if self.verbose:
print(input_file_path)
# `image_dict` contains metadata (to be returned) on the current image
image_dict = {"input_file_path": input_file_path}
try:
# READ THE INPUT FILE #####################################
if self.verbose:
print("TEL{}_EV{}".format(image.meta["tel_id"],
image.meta["event_id"]))
reference_img = image.reference_image
pixels_position = image.pixels_position
if ref_img_as_input:
# This option is a hack to easily produce CSV files with
# the "null_ref" "cleaning" module...
input_img = copy.deepcopy(reference_img)
else:
input_img = image.input_image
image_dict.update(image.meta)
# Make the original 1D geom (required for the 2D to 1D geometry
# conversion, for Tailcut and for Hillas)
cam_id = image.meta['cam_id']
cleaning_function_params["cam_id"] = cam_id
geom1d = geometry_converter.get_geom1d(cam_id)
if benchmark_method is not None:
# FETCH ADDITIONAL IMAGE METADATA #####################
image_dict["img_ref_signal_to_border"] = signal_to_border(
reference_img) # TODO: NaN
image_dict[
"img_ref_signal_to_border_distance"] = signal_to_border_distance(
reference_img) # TODO: NaN
image_dict["img_ref_pemax_on_border"] = pemax_on_border(
reference_img) # TODO: NaN
delta_pe, delta_abs_pe, delta_num_pixels = filter_pixels_clusters_stats(
reference_img) # TODO: NaN
num_islands = number_of_pixels_clusters(
reference_img) # TODO: NaN
image_dict["img_ref_islands_delta_pe"] = delta_pe
image_dict["img_ref_islands_delta_abs_pe"] = delta_abs_pe
image_dict[
"img_ref_islands_delta_num_pixels"] = delta_num_pixels
image_dict["img_ref_num_islands"] = num_islands
image_dict["img_ref_sum_pe"] = float(
np.nansum(reference_img))
image_dict["img_ref_min_pe"] = float(
np.nanmin(reference_img))
image_dict["img_ref_max_pe"] = float(
np.nanmax(reference_img))
image_dict["img_ref_num_pix"] = int(
(reference_img[np.isfinite(reference_img)] > 0).sum())
image_dict["img_in_sum_pe"] = float(np.nansum(input_img))
image_dict["img_in_min_pe"] = float(np.nanmin(input_img))
image_dict["img_in_max_pe"] = float(np.nanmax(input_img))
image_dict["img_in_num_pix"] = int(
(input_img[np.isfinite(input_img)] > 0).sum())
reference_img1d = geometry_converter.image_2d_to_1d(
reference_img, cam_id)
try:
hillas_params_2_ref_img = get_hillas_parameters(
geom1d, reference_img1d,
HILLAS_IMPLEMENTATION) # TODO GEOM
except Exception as e:
hillas_params_2_ref_img = float('nan')
print(e)
#traceback.print_tb(e.__traceback__, file=sys.stdout)
image_dict["img_ref_hillas_2_size"] = float(
hillas_params_2_ref_img.intensity)
image_dict[
"img_ref_hillas_2_cen_x"] = hillas_params_2_ref_img.x.value
image_dict[
"img_ref_hillas_2_cen_y"] = hillas_params_2_ref_img.y.value
image_dict[
"img_ref_hillas_2_length"] = hillas_params_2_ref_img.length.value
image_dict[
"img_ref_hillas_2_width"] = hillas_params_2_ref_img.width.value
image_dict[
"img_ref_hillas_2_r"] = hillas_params_2_ref_img.r.value
image_dict[
"img_ref_hillas_2_phi"] = hillas_params_2_ref_img.phi.to(
u.rad).value
image_dict[
"img_ref_hillas_2_psi"] = hillas_params_2_ref_img.psi.to(
u.rad).value
try:
image_dict["img_ref_hillas_2_miss"] = float(
hillas_params_2_ref_img.miss.value)
except:
image_dict["img_ref_hillas_2_miss"] = None
image_dict[
"img_ref_hillas_2_kurtosis"] = hillas_params_2_ref_img.kurtosis
image_dict[
"img_ref_hillas_2_skewness"] = hillas_params_2_ref_img.skewness
# CLEAN THE INPUT IMAGE ###################################
# Copy the image (otherwise some cleaning functions like Tailcut may change it)
#input_img_copy = copy.deepcopy(input_img)
input_img_copy = input_img.astype('float64', copy=True)
cleaning_function_params["output_data_dict"] = {}
initial_time = time.perf_counter()
cleaned_img = self.clean_image(
input_img_copy, **cleaning_function_params) # TODO: NaN
full_clean_execution_time_sec = time.perf_counter(
) - initial_time
if benchmark_method is not None:
image_dict.update(
cleaning_function_params["output_data_dict"])
del cleaning_function_params["output_data_dict"]
# ASSESS OR PRINT THE CLEANED IMAGE #######################
if benchmark_method is not None:
# ASSESS THE CLEANING #################################
kwargs = {
'geom': geom1d,
'hillas_implementation': HILLAS_IMPLEMENTATION
} # TODO GEOM
score_tuple, score_name_tuple = assess.assess_image_cleaning(
input_img, cleaned_img, reference_img,
benchmark_method, **kwargs)
image_dict[
"img_cleaned_signal_to_border"] = signal_to_border(
cleaned_img)
image_dict[
"img_cleaned_signal_to_border_distance"] = signal_to_border_distance(
cleaned_img)
image_dict[
"img_cleaned_pemax_on_border"] = pemax_on_border(
cleaned_img)
image_dict["score"] = score_tuple
image_dict["score_name"] = score_name_tuple
image_dict[
"full_clean_execution_time_sec"] = full_clean_execution_time_sec
image_dict["img_cleaned_sum_pe"] = float(
np.nansum(cleaned_img))
image_dict["img_cleaned_min_pe"] = float(
np.nanmin(cleaned_img))
image_dict["img_cleaned_max_pe"] = float(
np.nanmax(cleaned_img))
image_dict["img_cleaned_num_pix"] = int(
(cleaned_img[np.isfinite(cleaned_img)] > 0).sum())
cleaned_img1d = geometry_converter.image_2d_to_1d(
cleaned_img, cam_id)
try:
hillas_params_2_cleaned_img = get_hillas_parameters(
geom1d, cleaned_img1d,
HILLAS_IMPLEMENTATION) # GEOM
except Exception as e:
hillas_params_2_cleaned_img = float('nan')
print(e)
#traceback.print_tb(e.__traceback__, file=sys.stdout)
try:
image_dict["img_cleaned_hillas_2_size"] = float(
hillas_params_2_cleaned_img.intensity)
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_size"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_cen_x"] = hillas_params_2_cleaned_img.x.value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_cen_x"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_cen_y"] = hillas_params_2_cleaned_img.y.value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_cen_y"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_length"] = hillas_params_2_cleaned_img.length.value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_length"] = float(
'nan')
try:
image_dict[
"img_cleaned_hillas_2_width"] = hillas_params_2_cleaned_img.width.value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_width"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_r"] = hillas_params_2_cleaned_img.r.value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_r"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_phi"] = hillas_params_2_cleaned_img.phi.to(
u.rad).value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_phi"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_psi"] = hillas_params_2_cleaned_img.psi.to(
u.rad).value
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_psi"] = float('nan')
try:
image_dict["img_cleaned_hillas_2_miss"] = float(
hillas_params_2_cleaned_img.miss.value)
except:
image_dict["img_cleaned_hillas_2_miss"] = float('nan')
try:
image_dict[
"img_cleaned_hillas_2_kurtosis"] = hillas_params_2_cleaned_img.kurtosis
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_kurtosis"] = float(
'nan')
try:
image_dict[
"img_cleaned_hillas_2_skewness"] = hillas_params_2_cleaned_img.skewness
except AttributeError as e:
print(e)
image_dict["img_cleaned_hillas_2_skewness"] = float(
'nan')
# PLOT IMAGES #########################################################
if plot or (saveplot is not None):
image_list = [
geometry_converter.image_2d_to_1d(input_img, cam_id),
geometry_converter.image_2d_to_1d(
reference_img, cam_id),
geometry_converter.image_2d_to_1d(cleaned_img, cam_id)
]
title_list = [
"Input image", "Reference image", "Cleaned image"
]
geom_list = [geom1d, geom1d, geom1d]
hillas_list = [False, True, True]
if plot:
pywicta.io.images.plot_list(image_list,
geom_list=geom_list,
title_list=title_list,
hillas_list=hillas_list,
metadata_dict=image.meta)
if saveplot is not None:
basename, extension = os.path.splitext(saveplot)
plot_file_path = "{}_E{}_T{}{}".format(
basename, image.meta["event_id"],
image.meta["tel_id"], extension)
print("Saving {}".format(plot_file_path))
pywicta.io.images.mpl_save_list(
image_list,
geom_list=geom_list,
output_file_path=plot_file_path,
title_list=title_list,
hillas_list=hillas_list,
metadata_dict=image.meta)
except Exception as e:
print("Abort image {}: {} ({})".format(input_file_path, e,
type(e)))
if debug:
# The following line print the full trackback
traceback.print_tb(e.__traceback__, file=sys.stdout)
if benchmark_method is not None:
# http://docs.python.org/2/library/sys.html#sys.exc_info
exc_type, exc_value, exc_traceback = sys.exc_info(
) # most recent (if any) by default
'''
Reason this _can_ be bad: If an (unhandled) exception happens AFTER this,
or if we do not delete the labels on (not much) older versions of Py, the
reference we created can linger.
traceback.format_exc/print_exc do this very thing, BUT note this creates a
temp scope within the function.
'''
error_dict = {
'filename': exc_traceback.tb_frame.f_code.co_filename,
'lineno': exc_traceback.tb_lineno,
'name': exc_traceback.tb_frame.f_code.co_name,
'type': exc_type.__name__,
#'message' : exc_value.message
'message': str(e)
}
del (
exc_type, exc_value, exc_traceback
) # So we don't leave our local labels/objects dangling
# This still isn't "completely safe", though!
#error_dict = {"type": str(type(e)),
# "message": str(e)}
image_dict["error"] = error_dict
finally:
if benchmark_method is not None:
io_list.append(image_dict)
if benchmark_method is not None:
error_list = [
image_dict["error"] for image_dict in io_list
if "error" in image_dict
]
print("{} images aborted".format(len(error_list)))
# GENERAL EXPERIMENT METADATA
output_dict = {}
output_dict["benchmark_execution_time_sec"] = str(
time.perf_counter() - launch_time)
output_dict["date_time"] = str(datetime.datetime.now())
output_dict["class_name"] = self.__class__.__name__
output_dict["algo_code_ref"] = str(
self.__class__.clean_image.__code__)
output_dict["label"] = self.label
output_dict["cmd"] = " ".join(sys.argv)
output_dict["algo_params"] = cleaning_function_params
if "noise_distribution" in output_dict["algo_params"]:
del output_dict["algo_params"][
"noise_distribution"] # not JSON serializable...
output_dict["benchmark_method"] = benchmark_method
output_dict["system"] = " ".join(os.uname())
output_dict["io"] = io_list
if output_file_path is not None:
with open(output_file_path, "w") as fd:
json.dump(output_dict, fd, sort_keys=True,
indent=4) # pretty print format
return output_dict
def clean_image(self,
input_img,
high_threshold=10.,
low_threshold=8.,
pixels_clusters_filtering="off",
verbose=False,
cam_id=None,
output_data_dict=None,
**kwargs):
"""Apply ctapipe's tail-cut image cleaning on ``input_img``.
Note
----
The main difference with :mod:`ctapipe.image.cleaning.tailcuts_clean` is that here the cleaning function
takes 2D Numpy arrays.
Parameters
----------
input_img : array_like
The image to clean. Should be a **2D** Numpy array.
high_threshold : float
The *core threshold* (a.k.a. *picture threshold*).
low_threshold : float
The *boundary threshold*.
pixels_clusters_filtering : str
Defines the method used to remove isolated pixels after the tail-cut image cleaning.
Accepted values are: "off", "scipy" or "mars".
- "off": don't apply any filtering after the tail-cut image cleaning.
- "scipy": keep only the largest cluster of pixels after the tail-cut image cleaning.
See :mod:`pywi.processing.filtering.pixel_clusters` for more information.
- "mars": apply the same filtering than in CTA-Mars analysis, keep only *significant core pixels* that have
at least two others *significant core pixels* among its neighbors (a *significant core pixels* is a pixel
above the *core threshold*).
verbose : bool
Print additional messages if ``True``.
cam_id : str
The camera ID from which ``input_img`` came from: "ASTRICam", "CHEC", "DigiCam", "FlashCam", "NectarCam" or
"LSTCam".
output_data_dict : dict
An optional dictionary used to transmit internal information to the caller.
Returns
-------
array_like
The ``input_img`` after tail-cut image cleaning. This is a **2D** Numpy array (i.e. it should be converted
with :func:`pywicta.io.geometry_converter.image_2d_to_1d` before any usage in ctapipe).
"""
if cam_id is None:
raise Exception("cam_id have to be defined") # TODO
if not (pixels_clusters_filtering.lower() in ("off", "scipy", "mars")):
raise ValueError(
'pixels_clusters_filtering = {}. Accepted values are: "off", "scipy" or "mars".'
.format(pixels_clusters_filtering))
# If low_threshold > high_threshold then low_threshold = high_threshold
low_threshold = min(high_threshold, low_threshold)
# 2D ARRAY (FITS IMAGE) TO CTAPIPE IMAGE ###############
geom_1d = geometry_converter.get_geom1d(cam_id)
img_1d = geometry_converter.image_2d_to_1d(input_img, cam_id)
# APPLY TAILCUT CLEANING ##############################
if pixels_clusters_filtering.lower() == "mars":
if verbose:
print("Mars pixels clusters filtering")
mask = tailcuts_clean(geom_1d,
img_1d,
picture_thresh=high_threshold,
boundary_thresh=low_threshold,
keep_isolated_pixels=False,
min_number_picture_neighbors=2)
else:
mask = tailcuts_clean(geom_1d,
img_1d,
picture_thresh=high_threshold,
boundary_thresh=low_threshold,
keep_isolated_pixels=True)
img_1d[mask == False] = 0
# CTAPIPE IMAGE TO 2D ARRAY (FITS IMAGE) ###############
cleaned_img_2d = geometry_converter.image_1d_to_2d(img_1d, cam_id)
# KILL ISOLATED PIXELS #################################
img_cleaned_islands_delta_pe, img_cleaned_islands_delta_abs_pe, img_cleaned_islands_delta_num_pixels = filter_pixels_clusters_stats(
cleaned_img_2d)
img_cleaned_num_islands = number_of_pixels_clusters(cleaned_img_2d)
if output_data_dict is not None:
output_data_dict[
"img_cleaned_islands_delta_pe"] = img_cleaned_islands_delta_pe
output_data_dict[
"img_cleaned_islands_delta_abs_pe"] = img_cleaned_islands_delta_abs_pe
output_data_dict[
"img_cleaned_islands_delta_num_pixels"] = img_cleaned_islands_delta_num_pixels
output_data_dict[
"img_cleaned_num_islands"] = img_cleaned_num_islands
if pixels_clusters_filtering.lower() == "scipy":
if verbose:
print("Scipy pixels clusters filtering")
cleaned_img_2d = scipy_pixels_clusters_filtering(cleaned_img_2d)
return cleaned_img_2d