def combine_median(cls, file_names: [str], calibrator: Calibrator, console: Console, session_controller: SessionController) -> ndarray: """ Combine the files in the given list using a simple median Check, as reading, that they all have the same dimensions :param file_names: Names of files to be combined :param calibrator: Calibration object, abstracting precalibration operations :param console: Redirectable console output handler :param session_controller: Controller for this subtask, checking for cancellation :return: ndarray giving the 2-dimensional matrix of resulting pixel values """ assert len( file_names ) > 0 # Otherwise the combine button would have been disabled console.push_level() console.message("Combine by simple Median", +1) descriptors = RmFitsUtil.make_file_descriptions(file_names) file_data = RmFitsUtil.read_all_files_data(file_names) cls.check_cancellation(session_controller) file_data = calibrator.calibrate_images(file_data, descriptors, console, session_controller) cls.check_cancellation(session_controller) median_result = numpy.median(file_data, axis=0) console.pop_level() return median_result
def combine_median(cls, file_names: [str], calibrator: Calibrator, console: Console, session_controller: SessionController) -> ndarray: assert len( file_names ) > 0 # Otherwise the combine button would have been disabled console.push_level() console.message("Combine by simple Median", +1) file_data = RmFitsUtil.read_all_files_data(file_names) cls.check_cancellation(session_controller) sample_file = RmFitsUtil.make_file_descriptor(file_names[0]) file_data = calibrator.calibrate_images(file_data, sample_file, console, session_controller) cls.check_cancellation(session_controller) median_result = numpy.median(file_data, axis=0) console.pop_level() return median_result
def combine_mean(cls, file_names: [str], calibrator: Calibrator, console: Console, session_controller: SessionController) -> ndarray: """Combine FITS files in given list using simple mean. Return an ndarray containing the combined data.""" assert len( file_names ) > 0 # Otherwise the combine button would have been disabled console.push_level() console.message("Combining by simple mean", +1) sample_file = RmFitsUtil.make_file_descriptor(file_names[0]) file_data: [ndarray] file_data = RmFitsUtil.read_all_files_data(file_names) cls.check_cancellation(session_controller) calibrated_data = calibrator.calibrate_images(file_data, sample_file, console, session_controller) cls.check_cancellation(session_controller) mean_result = numpy.mean(calibrated_data, axis=0) console.pop_level() return mean_result
def combine_min_max_clip( cls, file_names: [str], number_dropped_values: int, calibrator: Calibrator, console: Console, session_controller: SessionController) -> Optional[ndarray]: """Combine FITS files in given list using min/max-clipped mean. Return an ndarray containing the combined data.""" success: bool assert len( file_names ) > 0 # Otherwise the combine button would have been disabled # Get the data to be processed file_data_list: [ndarray] = RmFitsUtil.read_all_files_data(file_names) cls.check_cancellation(session_controller) file_data = numpy.asarray(file_data_list) sample_file = RmFitsUtil.make_file_descriptor(file_names[0]) file_data = calibrator.calibrate_images(file_data, sample_file, console, session_controller) cls.check_cancellation(session_controller) # Do the math using each algorithm, and display how long it takes # time_before_0 = datetime.now() # result0 = cls.min_max_clip_version_0(file_data, number_dropped_values, progress_dots) # time_after_0 = datetime.now() # duration_0 = time_after_0 - time_before_0 # # time_before_1 = datetime.now() # result1 = cls.min_max_clip_version_1(file_data, number_dropped_values, progress_dots) # time_after_1 = datetime.now() # duration_1 = time_after_1 - time_before_1 # # time_before_2 = datetime.now() # result2 = cls.min_max_clip_version_2(file_data, number_dropped_values, progress_dots) # time_after_2 = datetime.now() # duration_2 = time_after_2 - time_before_2 # # time_before_3 = datetime.now() # result3 = cls.min_max_clip_version_3(file_data, number_dropped_values, progress_dots) # time_after_3 = datetime.now() # duration_3 = time_after_3 - time_before_3 # # time_before_4 = datetime.now() # result4 = cls.min_max_clip_version_4(file_data, number_dropped_values) # time_after_4 = datetime.now() # duration_4 = time_after_4 - time_before_4 # # time_before_5 = datetime.now() # result5 = cls.min_max_clip_version_5(file_data, number_dropped_values) # time_after_5 = datetime.now() # duration_5 = time_after_5 - time_before_5 # # print(f"Method 0 time: {duration_0}") # print(f"Method 1 time: {duration_1}") # print(f"Method 2 time: {duration_2}") # print(f"Method 3 time: {duration_3}") # print(f"Method 4 time: {duration_4}") # print(f"Method 5 time: {duration_5}") # # # Also ensure that the different algorithm versions produced exactly the same result # # Using method-0 as the reference # cls.compare_results(result0, result1, "1") # cls.compare_results(result0, result2, "2") # cls.compare_results(result0, result3, "3") # cls.compare_results(result0, result4, "4", dump=False) # cls.compare_results(result0, result5, "5") # # return result0 result5 = cls.min_max_clip_version_5(file_data, number_dropped_values, console, session_controller) cls.check_cancellation(session_controller) result = result5.filled() return result
def combine_sigma_clip( cls, file_names: [str], sigma_threshold: float, calibrator: Calibrator, console: Console, session_controller: SessionController) -> Optional[ndarray]: console.push_level() console.message( f"Combine by sigma-clipped mean, z-score threshold {sigma_threshold}", +1) sample_file = RmFitsUtil.make_file_descriptor(file_names[0]) file_data = numpy.asarray(RmFitsUtil.read_all_files_data(file_names)) cls.check_cancellation(session_controller) file_data = calibrator.calibrate_images(file_data, sample_file, console, session_controller) cls.check_cancellation(session_controller) console.message("Calculating unclipped means", +1) column_means = numpy.mean(file_data, axis=0) cls.check_cancellation(session_controller) console.message("Calculating standard deviations", 0) column_stdevs = numpy.std(file_data, axis=0) cls.check_cancellation(session_controller) console.message("Calculating z-scores", 0) # Now what we'd like to do is just: # z_scores = abs(file_data - column_means) / column_stdevs # Unfortunately, standard deviations can be zero, so that simplistic # statement would generate division-by-zero errors. # Std for a column would be zero if all the values in the column were identical. # In that case we wouldn't want to eliminate any anyway, so we'll set the # zero stdevs to a large number, which causes the z-scores to be small, which # causes no values to be eliminated. column_stdevs[column_stdevs == 0.0] = sys.float_info.max z_scores = abs(file_data - column_means) / column_stdevs cls.check_cancellation(session_controller) console.message("Eliminated data outside threshold", 0) exceeds_threshold = z_scores > sigma_threshold cls.check_cancellation(session_controller) # Calculate and display how much data we are ignoring dimensions = exceeds_threshold.shape total_pixels = dimensions[0] * dimensions[1] * dimensions[2] number_masked = numpy.count_nonzero(exceeds_threshold) percentage_masked = 100.0 * number_masked / total_pixels console.message( f"Discarded {number_masked:,} pixels of {total_pixels:,} " f"({percentage_masked:.3f}% of data)", +1) masked_array = ma.masked_array(file_data, exceeds_threshold) cls.check_cancellation(session_controller) console.message("Calculating adjusted means", -1) masked_means = ma.mean(masked_array, axis=0) cls.check_cancellation(session_controller) # If the means matrix contains any masked values, that means that in that column the clipping # eliminated *all* the data. We will find the offending columns and re-calculate those using # simple min-max clipping. if ma.is_masked(masked_means): console.message( "Some columns lost all their values; min-max clipping those columns.", 0) # Get the mask, and get a 2D matrix showing which columns were entirely masked eliminated_columns_map = ndarray.all(exceeds_threshold, axis=0) masked_coordinates = numpy.where(eliminated_columns_map) x_coordinates = masked_coordinates[0] y_coordinates = masked_coordinates[1] assert len(x_coordinates) == len(y_coordinates) for index in range(len(x_coordinates)): cls.check_cancellation(session_controller) column_x = x_coordinates[index] column_y = y_coordinates[index] column = file_data[:, column_x, column_y] min_max_clipped_mean: int = round( cls.calc_mm_clipped_mean(column, 2, console, session_controller)) masked_means[column_x, column_y] = min_max_clipped_mean # We've replaced the problematic columns, now the mean should calculate cleanly assert not ma.is_masked(masked_means) cls.check_cancellation(session_controller) console.pop_level() result = masked_means.round().filled() return result
def combine_min_max_clip( cls, file_names: [str], number_dropped_values: int, calibrator: Calibrator, console: Console, session_controller: SessionController) -> Optional[ndarray]: """ Combine the files in the given list using min-max clip algorithm Check, as reading, that they all have the same dimensions :param file_names: Names of files to be combined :param number_dropped_values Number of min and max values to drop from each column :param calibrator: Calibration object, abstracting precalibration operations :param console: Redirectable console output handler :param session_controller: Controller for this subtask, checking for cancellation :return: ndarray giving the 2-dimensional matrix of resulting pixel values """ success: bool assert len( file_names ) > 0 # Otherwise the combine button would have been disabled # Get the data to be processed file_data_list: [ndarray] = RmFitsUtil.read_all_files_data(file_names) cls.check_cancellation(session_controller) descriptors = RmFitsUtil.make_file_descriptions(file_names) file_data = numpy.asarray(file_data_list) file_data = calibrator.calibrate_images(file_data, descriptors, console, session_controller) cls.check_cancellation(session_controller) # Do the math using each algorithm, and display how long it takes # time_before_0 = datetime.now() # result0 = cls.min_max_clip_version_0(file_data, number_dropped_values, progress_dots) # time_after_0 = datetime.now() # duration_0 = time_after_0 - time_before_0 # # time_before_1 = datetime.now() # result1 = cls.min_max_clip_version_1(file_data, number_dropped_values, progress_dots) # time_after_1 = datetime.now() # duration_1 = time_after_1 - time_before_1 # # time_before_2 = datetime.now() # result2 = cls.min_max_clip_version_2(file_data, number_dropped_values, progress_dots) # time_after_2 = datetime.now() # duration_2 = time_after_2 - time_before_2 # # time_before_3 = datetime.now() # result3 = cls.min_max_clip_version_3(file_data, number_dropped_values, progress_dots) # time_after_3 = datetime.now() # duration_3 = time_after_3 - time_before_3 # # time_before_4 = datetime.now() # result4 = cls.min_max_clip_version_4(file_data, number_dropped_values) # time_after_4 = datetime.now() # duration_4 = time_after_4 - time_before_4 # # time_before_5 = datetime.now() # result5 = cls.min_max_clip_version_5(file_data, number_dropped_values) # time_after_5 = datetime.now() # duration_5 = time_after_5 - time_before_5 # # print(f"Method 0 time: {duration_0}") # print(f"Method 1 time: {duration_1}") # print(f"Method 2 time: {duration_2}") # print(f"Method 3 time: {duration_3}") # print(f"Method 4 time: {duration_4}") # print(f"Method 5 time: {duration_5}") # # # Also ensure that the different algorithm versions produced exactly the same result # # Using method-0 as the reference # cls.compare_results(result0, result1, "1") # cls.compare_results(result0, result2, "2") # cls.compare_results(result0, result3, "3") # cls.compare_results(result0, result4, "4", dump=False) # cls.compare_results(result0, result5, "5") # # return result0 result5 = cls.min_max_clip_version_5(file_data, number_dropped_values, console, session_controller) cls.check_cancellation(session_controller) result = result5.filled() return result