def filter_func(images: Images, diff=None, radius=_default_radius, mode=_default_mode, cores=None, progress: Progress = None): """ :param images: Input data :param diff: Pixel value difference above which to crop bright pixels :param radius: Size of the median filter to apply :param cores: The number of cores that will be used to process the data. :return: The processed 3D numpy.ndarray """ if diff and radius and diff > 0 and radius > 0: func = ps.create_partial(OutliersFilter._execute, ps.return_to_self, diff=diff, radius=radius, mode=mode) ps.shared_list = [images.data] ps.execute( func, images.num_projections, progress=progress, msg=f"Outliers with threshold {diff} and kernel {radius}") return images
def filter_func(images: Images, diff=None, radius=_default_radius, mode=_default_mode, cores=None, progress: Progress = None): """ :param images: Input data :param diff: Pixel value difference above which to crop bright pixels :param radius: Size of the median filter to apply :param mode: Whether to remove bright or dark outliers One of [OUTLIERS_BRIGHT, OUTLIERS_DARK] :param cores: The number of cores that will be used to process the data. :return: The processed 3D numpy.ndarray """ if not diff or not diff > 0: raise ValueError(f'diff parameter must be greater than 0. Value provided was {diff}') if not radius or not radius > 0: raise ValueError(f'radius parameter must be greater than 0. Value provided was {radius}') func = ps.create_partial(OutliersFilter._execute, ps.return_to_self, diff=diff, radius=radius, mode=mode) ps.shared_list = [images.data] ps.execute(func, images.num_projections, progress=progress, msg=f"Outliers with threshold {diff} and kernel {radius}") return images
def _execute(data: np.ndarray, flat=None, dark=None, cores=None, chunksize=None, progress=None): """A benchmark justifying the current implementation, performed on 500x2048x2048 images. #1 Separate runs Subtract (sequential with np.subtract(data, dark, out=data)) - 13s Divide (par) - 1.15s #2 Separate parallel runs Subtract (par) - 5.5s Divide (par) - 1.15s #3 Added subtract into _divide so that it is: np.true_divide( np.subtract(data, dark, out=data), norm_divide, out=data) Subtract then divide (par) - 55s """ with progress: progress.update(msg="Applying background correction") norm_divide = pu.create_array((data.shape[1], data.shape[2]), data.dtype) # subtract dark from flat and copy into shared array with [:] norm_divide[:] = np.subtract(flat, dark) # prevent divide-by-zero issues, and negative pixels make no sense norm_divide[norm_divide == 0] = MINIMUM_PIXEL_VALUE # subtract the dark from all images do_subtract = ps.create_partial(_subtract, fwd_function=ps.inplace_second_2d) ps.shared_list = [data, dark] ps.execute(do_subtract, data.shape[0], progress, cores=cores) # divide the data by (flat - dark) do_divide = ps.create_partial(_divide, fwd_function=ps.inplace_second_2d) ps.shared_list = [data, norm_divide] ps.execute(do_divide, data.shape[0], progress, cores=cores) return data
def _execute(data: np.ndarray, angle: float, cores: int, chunksize: int, progress: Progress): progress = Progress.ensure_instance(progress, task_name='Rotate Stack') with progress: f = ps.create_partial(_rotate_image_inplace, ps.inplace1, angle=angle) ps.shared_list = [data] ps.execute(f, data.shape[0], progress, msg=f"Rotating by {angle} degrees", cores=cores) return data
def _execute(data: np.ndarray, air_region: SensibleROI, cores=None, chunksize=None, progress=None): log = getLogger(__name__) with progress: progress.update(msg="Normalization by air region") if isinstance(air_region, list): air_region = SensibleROI.from_list(air_region) # initialise same number of air sums img_num = data.shape[0] air_sums = pu.create_array((img_num, ), data.dtype) do_calculate_air_sums = ps.create_partial(_calc_sum, ps.return_to_second_at_i, air_left=air_region.left, air_top=air_region.top, air_right=air_region.right, air_bottom=air_region.bottom) ps.shared_list = [data, air_sums] ps.execute(do_calculate_air_sums, data.shape[0], progress, cores=cores) do_divide = ps.create_partial(_divide_by_air_sum, fwd_function=ps.inplace2) ps.shared_list = [data, air_sums] ps.execute(do_divide, data.shape[0], progress, cores=cores) avg = np.average(air_sums) max_avg = np.max(air_sums) / avg min_avg = np.min(air_sums) / avg log.info( f"Normalization by air region. " f"Average: {avg}, max ratio: {max_avg}, min ratio: {min_avg}.")
def filter_func(images: Images, sigma=3, size=21, window_dim=1, filtering_dim=1, cores=None, chunksize=None, progress=None): if filtering_dim == 1: f = ps.create_partial(remove_stripe_based_filtering_sorting, ps.return_to_self, sigma=sigma, size=size, dim=window_dim) else: f = ps.create_partial(remove_stripe_based_2d_filtering_sorting, ps.return_to_self, sigma=sigma, size=size, dim=window_dim) ps.shared_list = [images.data] ps.execute(f, images.num_projections, progress, cores=cores) return images
def filter_func(images: Images, snr=3, size=61, cores=None, chunksize=None, progress=None): f = ps.create_partial(remove_dead_stripe, ps.return_to_self, snr=snr, size=size, residual=False) ps.shared_list = [images.data] ps.execute(f, images.data.shape[0], progress, cores=cores) return images
def filter_func(images, snr=3, la_size=61, cores=None, chunksize=None, progress=None): f = ps.create_partial( remove_large_stripe, ps.return_to_self, snr=snr, size=la_size, ) ps.shared_list = [images.data] ps.execute(f, images.num_projections, progress, cores=cores) return images
def _calculate_correlation_error(images, shared_search_range, min_correlation_error, progress): # if the projections are passed in the partial they are copied to every process on every iteration # this makes the multiprocessing significantly slower # so they are copied into a shared array to avoid that copying with pu.temp_shared_array((2, images.height, images.width)) as shared_projections: shared_projections[0][:] = images.projection(0) shared_projections[1][:] = np.fliplr(images.proj180deg.data[0]) do_search_partial = ps.create_partial(do_calculate_correlation_err, ps.inplace3, image_width=images.width) ps.shared_list = [min_correlation_error, shared_search_range, shared_projections] ps.execute(do_search_partial, num_operations=min_correlation_error.shape[0], progress=progress, msg="Finding correlation on row")
def filter_func(images: Images, order=1, sigma=3, cores=None, chunksize=None, progress=None): f = ps.create_partial(remove_stripe_based_fitting, ps.return_to_self, order=order, sigma=sigma, sort=True) ps.shared_list = [images.data] ps.execute(f, images.data.shape[0], progress, cores=cores) return images
def filter_func(images: Images, snr=3, size=61, cores=None, chunksize=None, progress=None): f = ps.create_partial( remove_unresponsive_and_fluctuating_stripe, ps.return_to_self, snr=snr, size=size, ) ps.shared_list = [images.data] ps.execute(f, images.num_projections, progress, cores=cores) return images
def filter_func(images: Images, order=1, sigmax=3, sigmay=3, cores=None, chunksize=None, progress=None): f = ps.create_partial(remove_stripe_based_sorting_fitting, ps.return_to_self, order=order, sigmax=sigmax, sigmay=sigmay) ps.shared_list = [images.data] ps.execute(f, images.num_projections, progress, cores=cores) return images
def filter_func(images: Images, snr=3, la_size=61, sm_size=21, dim=1, cores=None, chunksize=None, progress=None): f = ps.create_partial(remove_all_stripe, ps.return_to_self, snr=snr, la_size=la_size, sm_size=sm_size, dim=dim) ps.shared_list = [images.data] ps.execute(f, images.data.shape[0], progress, cores=cores) return images
def _execute(data, size, edgemode, cores=None, chunksize=None, progress=None): log = getLogger(__name__) progress = Progress.ensure_instance(progress, task_name='NaN Removal') # create the partial function to forward the parameters f = ps.create_partial(_nan_to_median, ps.return_to_self, size=size, edgemode=edgemode) with progress: log.info( "PARALLEL NaN Removal filter, with pixel data type: {0}".format( data.dtype)) ps.shared_list = [data] ps.execute(f, data.shape[0], progress, msg="NaN Removal", cores=cores) return data
def filter_func(images: Images, cores=None, chunksize=None, progress=None) -> Images: if images.num_projections == 1: # we can't really compute the preview as the image stack copy # passed in doesn't have the logfile in it raise RuntimeError("No logfile available for this stack.") counts = images.counts() if counts is None: raise RuntimeError("No loaded log values for this stack.") counts_val = counts.value / counts.value[0] do_division = ps.create_partial(_divide_by_counts, fwd_function=ps.inplace2) ps.shared_list = [images.data, counts_val] ps.execute(do_division, images.num_projections, progress, cores=cores) return images
def _execute(data: np.ndarray, size, mode, order, cores=None, progress=None): log = getLogger(__name__) progress = Progress.ensure_instance(progress, task_name='Gaussian filter') f = ps.create_partial(scipy_ndimage.gaussian_filter, ps.return_to_self, sigma=size, mode=mode, order=order) log.info("Starting PARALLEL gaussian filter, with pixel data type: {0}, " "filter size/width: {1}.".format(data.dtype, size)) progress.update() ps.shared_list = [data] ps.execute(f, data.shape[0], progress, msg="Gaussian filter", cores=cores) progress.mark_complete() log.info("Finished gaussian filter, with pixel data type: {0}, " "filter size/width: {1}.".format(data.dtype, size))
def filter_func(images: Images, rebin_param=0.5, mode=None, cores=None, chunksize=None, progress=None) -> Images: """ :param images: Sample data which is to be processed. Expects radiograms :param rebin_param: int, float or tuple int - Percentage of current size. float - Fraction of current size. tuple - Size of the output image (x, y). :param mode: Interpolation to use for re-sizing ('nearest', 'lanczos', 'bilinear', 'bicubic' or 'cubic'). :param cores: The number of cores that will be used to process the data. :param chunksize: The number of chunks that each worker will receive. :return: The processed 3D numpy.ndarray """ h.check_data_stack(images) if isinstance(rebin_param, tuple): param_valid = rebin_param[0] > 0 and rebin_param[1] > 0 else: param_valid = rebin_param > 0 if param_valid: sample = images.data empty_resized_data = _create_reshaped_array(images, rebin_param) f = ps.create_partial(skimage.transform.resize, ps.return_to_second_at_i, mode=mode, output_shape=empty_resized_data.shape[1:]) ps.shared_list = [sample, empty_resized_data] ps.execute(f, sample.shape[0], cores, "Applying Rebin", progress) images.data = empty_resized_data return images
def _execute(data, size, mode, cores=None, chunksize=None, progress=None): log = getLogger(__name__) progress = Progress.ensure_instance(progress, task_name='Median filter') # create the partial function to forward the parameters f = ps.create_partial(_median_filter, ps.return_to_self, size=size, mode=mode) with progress: log.info("PARALLEL median filter, with pixel data type: {0}, filter " "size/width: {1}.".format(data.dtype, size)) ps.shared_list = [data] ps.execute(f, data.shape[0], progress, msg="Median filter", cores=cores) return data
def _execute(data: np.ndarray, div_val: float, mult_val: float, add_val: float, sub_val: float, cores: Optional[int], progress): do_arithmetic = ps.create_partial(_arithmetic_func, fwd_function=ps.arithmetic) ps.shared_list = [data, div_val, mult_val, add_val, sub_val] ps.execute(do_arithmetic, data.shape[0], progress, cores=cores)
def _execute(data: np.ndarray, air_region: SensibleROI, normalisation_mode: str, flat_field: Optional[np.ndarray], cores=None, chunksize=None, progress=None): log = getLogger(__name__) with progress: progress.update(msg="Normalization by air region") if isinstance(air_region, list): air_region = SensibleROI.from_list(air_region) # initialise same number of air sums img_num = data.shape[0] air_means = pu.create_array((img_num, ), data.dtype) do_calculate_air_means = ps.create_partial( _calc_mean, ps.return_to_second_at_i, air_left=air_region.left, air_top=air_region.top, air_right=air_region.right, air_bottom=air_region.bottom) ps.shared_list = [data, air_means] ps.execute(do_calculate_air_means, data.shape[0], progress, cores=cores) if normalisation_mode == 'Preserve Max': air_maxs = pu.create_array((img_num, ), data.dtype) do_calculate_air_max = ps.create_partial(_calc_max, ps.return_to_second_at_i) ps.shared_list = [data, air_maxs] ps.execute(do_calculate_air_max, data.shape[0], progress, cores=cores) if np.isnan(air_maxs).any(): raise ValueError("Image contains invalid (NaN) pixels") # calculate the before and after maximum init_max = air_maxs.max() post_max = (air_maxs / air_means).max() air_means *= post_max / init_max elif normalisation_mode == 'Stack Average': air_means /= air_means.mean() elif normalisation_mode == 'Flat Field' and flat_field is not None: flat_mean = pu.create_array((flat_field.shape[0], ), flat_field.dtype) ps.shared_list = [flat_field, flat_mean] ps.execute(do_calculate_air_means, flat_field.shape[0], progress, cores=cores) air_means /= flat_mean.mean() if np.isnan(air_means).any(): raise ValueError("Air region contains invalid (NaN) pixels") do_divide = ps.create_partial(_divide_by_air, fwd_function=ps.inplace2) ps.shared_list = [data, air_means] ps.execute(do_divide, data.shape[0], progress, cores=cores) avg = np.average(air_means) max_avg = np.max(air_means) / avg min_avg = np.min(air_means) / avg log.info( f"Normalization by air region. " f"Average: {avg}, max ratio: {max_avg}, min ratio: {min_avg}.")