def combine_median(cls, file_names: [str], calibrator: Calibrator,
console: Console,
session_controller: SessionController) -> ndarray:
"""
Combine the files in the given list using a simple median
Check, as reading, that they all have the same dimensions
:param file_names: Names of files to be combined
:param calibrator: Calibration object, abstracting precalibration operations
:param console: Redirectable console output handler
:param session_controller: Controller for this subtask, checking for cancellation
:return: ndarray giving the 2-dimensional matrix of resulting pixel values
"""
assert len(
file_names
) > 0 # Otherwise the combine button would have been disabled
console.push_level()
console.message("Combine by simple Median", +1)
descriptors = RmFitsUtil.make_file_descriptions(file_names)
file_data = RmFitsUtil.read_all_files_data(file_names)
cls.check_cancellation(session_controller)
file_data = calibrator.calibrate_images(file_data, descriptors,
console, session_controller)
cls.check_cancellation(session_controller)
median_result = numpy.median(file_data, axis=0)
console.pop_level()
return median_result
def combine_median(cls, file_names: [str], calibrator: Calibrator,
console: Console,
session_controller: SessionController) -> ndarray:
assert len(
file_names
) > 0 # Otherwise the combine button would have been disabled
console.push_level()
console.message("Combine by simple Median", +1)
file_data = RmFitsUtil.read_all_files_data(file_names)
cls.check_cancellation(session_controller)
sample_file = RmFitsUtil.make_file_descriptor(file_names[0])
file_data = calibrator.calibrate_images(file_data, sample_file,
console, session_controller)
cls.check_cancellation(session_controller)
median_result = numpy.median(file_data, axis=0)
console.pop_level()
return median_result
def combine_mean(cls, file_names: [str], calibrator: Calibrator,
console: Console,
session_controller: SessionController) -> ndarray:
"""Combine FITS files in given list using simple mean. Return an ndarray containing the combined data."""
assert len(
file_names
) > 0 # Otherwise the combine button would have been disabled
console.push_level()
console.message("Combining by simple mean", +1)
sample_file = RmFitsUtil.make_file_descriptor(file_names[0])
file_data: [ndarray]
file_data = RmFitsUtil.read_all_files_data(file_names)
cls.check_cancellation(session_controller)
calibrated_data = calibrator.calibrate_images(file_data, sample_file,
console,
session_controller)
cls.check_cancellation(session_controller)
mean_result = numpy.mean(calibrated_data, axis=0)
console.pop_level()
return mean_result
def combine_min_max_clip(
cls, file_names: [str], number_dropped_values: int,
calibrator: Calibrator, console: Console,
session_controller: SessionController) -> Optional[ndarray]:
"""Combine FITS files in given list using min/max-clipped mean.
Return an ndarray containing the combined data."""
success: bool
assert len(
file_names
) > 0 # Otherwise the combine button would have been disabled
# Get the data to be processed
file_data_list: [ndarray] = RmFitsUtil.read_all_files_data(file_names)
cls.check_cancellation(session_controller)
file_data = numpy.asarray(file_data_list)
sample_file = RmFitsUtil.make_file_descriptor(file_names[0])
file_data = calibrator.calibrate_images(file_data, sample_file,
console, session_controller)
cls.check_cancellation(session_controller)
# Do the math using each algorithm, and display how long it takes
# time_before_0 = datetime.now()
# result0 = cls.min_max_clip_version_0(file_data, number_dropped_values, progress_dots)
# time_after_0 = datetime.now()
# duration_0 = time_after_0 - time_before_0
#
# time_before_1 = datetime.now()
# result1 = cls.min_max_clip_version_1(file_data, number_dropped_values, progress_dots)
# time_after_1 = datetime.now()
# duration_1 = time_after_1 - time_before_1
#
# time_before_2 = datetime.now()
# result2 = cls.min_max_clip_version_2(file_data, number_dropped_values, progress_dots)
# time_after_2 = datetime.now()
# duration_2 = time_after_2 - time_before_2
#
# time_before_3 = datetime.now()
# result3 = cls.min_max_clip_version_3(file_data, number_dropped_values, progress_dots)
# time_after_3 = datetime.now()
# duration_3 = time_after_3 - time_before_3
#
# time_before_4 = datetime.now()
# result4 = cls.min_max_clip_version_4(file_data, number_dropped_values)
# time_after_4 = datetime.now()
# duration_4 = time_after_4 - time_before_4
#
# time_before_5 = datetime.now()
# result5 = cls.min_max_clip_version_5(file_data, number_dropped_values)
# time_after_5 = datetime.now()
# duration_5 = time_after_5 - time_before_5
#
# print(f"Method 0 time: {duration_0}")
# print(f"Method 1 time: {duration_1}")
# print(f"Method 2 time: {duration_2}")
# print(f"Method 3 time: {duration_3}")
# print(f"Method 4 time: {duration_4}")
# print(f"Method 5 time: {duration_5}")
#
# # Also ensure that the different algorithm versions produced exactly the same result
# # Using method-0 as the reference
# cls.compare_results(result0, result1, "1")
# cls.compare_results(result0, result2, "2")
# cls.compare_results(result0, result3, "3")
# cls.compare_results(result0, result4, "4", dump=False)
# cls.compare_results(result0, result5, "5")
#
# return result0
result5 = cls.min_max_clip_version_5(file_data, number_dropped_values,
console, session_controller)
cls.check_cancellation(session_controller)
result = result5.filled()
return result
def combine_sigma_clip(
cls, file_names: [str], sigma_threshold: float,
calibrator: Calibrator, console: Console,
session_controller: SessionController) -> Optional[ndarray]:
console.push_level()
console.message(
f"Combine by sigma-clipped mean, z-score threshold {sigma_threshold}",
+1)
sample_file = RmFitsUtil.make_file_descriptor(file_names[0])
file_data = numpy.asarray(RmFitsUtil.read_all_files_data(file_names))
cls.check_cancellation(session_controller)
file_data = calibrator.calibrate_images(file_data, sample_file,
console, session_controller)
cls.check_cancellation(session_controller)
console.message("Calculating unclipped means", +1)
column_means = numpy.mean(file_data, axis=0)
cls.check_cancellation(session_controller)
console.message("Calculating standard deviations", 0)
column_stdevs = numpy.std(file_data, axis=0)
cls.check_cancellation(session_controller)
console.message("Calculating z-scores", 0)
# Now what we'd like to do is just:
# z_scores = abs(file_data - column_means) / column_stdevs
# Unfortunately, standard deviations can be zero, so that simplistic
# statement would generate division-by-zero errors.
# Std for a column would be zero if all the values in the column were identical.
# In that case we wouldn't want to eliminate any anyway, so we'll set the
# zero stdevs to a large number, which causes the z-scores to be small, which
# causes no values to be eliminated.
column_stdevs[column_stdevs == 0.0] = sys.float_info.max
z_scores = abs(file_data - column_means) / column_stdevs
cls.check_cancellation(session_controller)
console.message("Eliminated data outside threshold", 0)
exceeds_threshold = z_scores > sigma_threshold
cls.check_cancellation(session_controller)
# Calculate and display how much data we are ignoring
dimensions = exceeds_threshold.shape
total_pixels = dimensions[0] * dimensions[1] * dimensions[2]
number_masked = numpy.count_nonzero(exceeds_threshold)
percentage_masked = 100.0 * number_masked / total_pixels
console.message(
f"Discarded {number_masked:,} pixels of {total_pixels:,} "
f"({percentage_masked:.3f}% of data)", +1)
masked_array = ma.masked_array(file_data, exceeds_threshold)
cls.check_cancellation(session_controller)
console.message("Calculating adjusted means", -1)
masked_means = ma.mean(masked_array, axis=0)
cls.check_cancellation(session_controller)
# If the means matrix contains any masked values, that means that in that column the clipping
# eliminated *all* the data. We will find the offending columns and re-calculate those using
# simple min-max clipping.
if ma.is_masked(masked_means):
console.message(
"Some columns lost all their values; min-max clipping those columns.",
0)
# Get the mask, and get a 2D matrix showing which columns were entirely masked
eliminated_columns_map = ndarray.all(exceeds_threshold, axis=0)
masked_coordinates = numpy.where(eliminated_columns_map)
x_coordinates = masked_coordinates[0]
y_coordinates = masked_coordinates[1]
assert len(x_coordinates) == len(y_coordinates)
for index in range(len(x_coordinates)):
cls.check_cancellation(session_controller)
column_x = x_coordinates[index]
column_y = y_coordinates[index]
column = file_data[:, column_x, column_y]
min_max_clipped_mean: int = round(
cls.calc_mm_clipped_mean(column, 2, console,
session_controller))
masked_means[column_x, column_y] = min_max_clipped_mean
# We've replaced the problematic columns, now the mean should calculate cleanly
assert not ma.is_masked(masked_means)
cls.check_cancellation(session_controller)
console.pop_level()
result = masked_means.round().filled()
return result
def combine_min_max_clip(
cls, file_names: [str], number_dropped_values: int,
calibrator: Calibrator, console: Console,
session_controller: SessionController) -> Optional[ndarray]:
"""
Combine the files in the given list using min-max clip algorithm
Check, as reading, that they all have the same dimensions
:param file_names: Names of files to be combined
:param number_dropped_values Number of min and max values to drop from each column
:param calibrator: Calibration object, abstracting precalibration operations
:param console: Redirectable console output handler
:param session_controller: Controller for this subtask, checking for cancellation
:return: ndarray giving the 2-dimensional matrix of resulting pixel values
"""
success: bool
assert len(
file_names
) > 0 # Otherwise the combine button would have been disabled
# Get the data to be processed
file_data_list: [ndarray] = RmFitsUtil.read_all_files_data(file_names)
cls.check_cancellation(session_controller)
descriptors = RmFitsUtil.make_file_descriptions(file_names)
file_data = numpy.asarray(file_data_list)
file_data = calibrator.calibrate_images(file_data, descriptors,
console, session_controller)
cls.check_cancellation(session_controller)
# Do the math using each algorithm, and display how long it takes
# time_before_0 = datetime.now()
# result0 = cls.min_max_clip_version_0(file_data, number_dropped_values, progress_dots)
# time_after_0 = datetime.now()
# duration_0 = time_after_0 - time_before_0
#
# time_before_1 = datetime.now()
# result1 = cls.min_max_clip_version_1(file_data, number_dropped_values, progress_dots)
# time_after_1 = datetime.now()
# duration_1 = time_after_1 - time_before_1
#
# time_before_2 = datetime.now()
# result2 = cls.min_max_clip_version_2(file_data, number_dropped_values, progress_dots)
# time_after_2 = datetime.now()
# duration_2 = time_after_2 - time_before_2
#
# time_before_3 = datetime.now()
# result3 = cls.min_max_clip_version_3(file_data, number_dropped_values, progress_dots)
# time_after_3 = datetime.now()
# duration_3 = time_after_3 - time_before_3
#
# time_before_4 = datetime.now()
# result4 = cls.min_max_clip_version_4(file_data, number_dropped_values)
# time_after_4 = datetime.now()
# duration_4 = time_after_4 - time_before_4
#
# time_before_5 = datetime.now()
# result5 = cls.min_max_clip_version_5(file_data, number_dropped_values)
# time_after_5 = datetime.now()
# duration_5 = time_after_5 - time_before_5
#
# print(f"Method 0 time: {duration_0}")
# print(f"Method 1 time: {duration_1}")
# print(f"Method 2 time: {duration_2}")
# print(f"Method 3 time: {duration_3}")
# print(f"Method 4 time: {duration_4}")
# print(f"Method 5 time: {duration_5}")
#
# # Also ensure that the different algorithm versions produced exactly the same result
# # Using method-0 as the reference
# cls.compare_results(result0, result1, "1")
# cls.compare_results(result0, result2, "2")
# cls.compare_results(result0, result3, "3")
# cls.compare_results(result0, result4, "4", dump=False)
# cls.compare_results(result0, result5, "5")
#
# return result0
result5 = cls.min_max_clip_version_5(file_data, number_dropped_values,
console, session_controller)
cls.check_cancellation(session_controller)
result = result5.filled()
return result
