def test_get_reference_sample_stacks(self): num_stack = 3 for i in range(num_stack): file_path = "data/speck_" + str(i) + ".hdf" ifile = h5py.File(file_path, "w") data = np.ones((2, 64, 64)) ifile.create_dataset("entry/speck", data=np.float32(data)) ifile.close() file_path = "data/tomo_" + str(i) + ".hdf" ifile = h5py.File(file_path, "w") data = 2 * np.ones((4, 64, 64)) ifile.create_dataset("entry/tomo", data=np.float32(data)) ifile.close() ref_path = losa.find_file("data/speck*") sam_path = losa.find_file("data/tomo*") ref_key, sam_key = "entry/speck", "entry/tomo" f_alias = losa.get_reference_sample_stacks ref_stack, sam_stack = f_alias(0, ref_path, sam_path, ref_key, sam_key, crop=(0, 0, 0, 0), flat_field=None, dark_field=None, num_use=None, fix_zero_div=False) check1 = True if (ref_stack.shape[0] == num_stack) \ and (sam_stack.shape[0] == num_stack) else False check2 = True if (np.mean(ref_stack) == 1.0) \ and (np.mean(sam_stack) == 2.0) else False self.assertTrue(check1 and check2)
def convert_tif_to_hdf(input_path, output_path, key_path="entry/data", crop=(0, 0, 0, 0), pattern=None, **options): """ Convert a folder of tif files to a hdf/nxs file. Parameters ---------- input_path : str Folder path to the tif files. output_path : str Path to the hdf/nxs file. key_path : str, optional Key path to the dataset. crop : tuple of int, optional Crop the images from the edges, i.e. crop = (crop_top, crop_bottom, crop_left, crop_right). pattern : str, optional Used to find tif files with names matching the pattern. options : dict, optional Add metadata. E.g. options={"entry/angles": angles, "entry/energy": 53}. Returns ------- str Path to the hdf/nxs file. """ if pattern is None: list_file = losa.find_file(input_path + "/*.tif*") else: list_file = losa.find_file(input_path + "/*" + pattern + "*.tif*") depth = len(list_file) if depth == 0: raise ValueError("No tif files in the folder: {}".format(input_path)) (height, width) = np.shape(losa.load_image(list_file[0])) file_base, file_ext = os.path.splitext(output_path) if not (file_ext == '.hdf' or file_ext == '.h5' or file_ext == ".nxs"): file_ext = '.hdf' output_path = file_base + file_ext cr_top, cr_bottom, cr_left, cr_right = crop cr_height = height - cr_top - cr_bottom cr_width = width - cr_left - cr_right data_out = losa.open_hdf_stream(output_path, (depth, cr_height, cr_width), key_path=key_path, overwrite=True, **options) for i, fname in enumerate(list_file): data_out[i] = losa.load_image(fname)[cr_top:cr_height + cr_top, cr_left:cr_width + cr_left] return output_path
def test_get_reference_sample_stacks_dls(self): num_stack = 3 for i in range(num_stack): file_path = "data/speck_tomo_" + str(i) + ".hdf" ifile = h5py.File(file_path, "w") data = np.concatenate((np.zeros((2, 64, 64)), np.ones( (3, 64, 64)), 2 * np.ones((4, 64, 64))), axis=0) ifile.create_dataset("entry/data/data", data=np.float32(data)) keys = np.concatenate((2.0 * np.ones(2), np.ones(3), np.zeros(4))) ifile.create_dataset("entry/data/image_key", data=np.float32(keys)) ifile.close() list_path = losa.find_file("data/speck_tomo*") f_alias = losa.get_reference_sample_stacks_dls ref_stack, sam_stack = f_alias(0, list_path, data_key=None, image_key=None, crop=(0, 0, 0, 0), flat_field=None, dark_field=None, num_use=None, fix_zero_div=False) check1 = True if (ref_stack.shape[0] == num_stack) \ and (sam_stack.shape[0] == num_stack) else False check2 = True if (np.mean(ref_stack) == 1.0) \ and (np.mean(sam_stack) == 2.0) else False self.assertTrue(check1 and check2)
def test_extract_tif_from_hdf(self): file_path = "data/data.hdf" out_base = "data/extract_tif/" ifile = h5py.File(file_path, "w") ifile.create_dataset("entry/data", data=np.random.rand(2, 64, 64)) ifile.close() con.extract_tif_from_hdf(file_path, out_base, "entry/data") list_file = losa.find_file(out_base + "/*.tif") self.assertTrue(len(list_file) == 2)
factor of 8 without an intermediate step of combining 12 files to 1 huge file (11.3 TB in total). """ import timeit import numpy as np import algotom.io.loadersaver as losa input_base = "D:/Full_reconstruction/" # Where to save the outputs output_base = "D:/Dsp_grid_scan/" output_file = "full_size_dsp_8_8_8.hdf" cube = (8, 8, 8) # Downsampling factor list_file = losa.find_file(input_base + "*.hdf") key_path = "entry/data" list_hdf_object = [] num_file = len(list_file) list_nslice = [] for i in range(num_file): hdf_object = losa.load_hdf(list_file[i], key_path) list_hdf_object.append(hdf_object) (nslice, height, width) = hdf_object.shape list_nslice.append(nslice) total_slice = np.sum(np.asarray(list_nslice)) total_slice_r = (total_slice // cube[0]) * cube[0] height_r = (height // cube[1]) * cube[1] width_r = (width // cube[2]) * cube[2]
def get_statical_information_dataset(input_, percentile=(5, 95), skip=5, denoise=False, key_path=None): """ Get statical information of a dataset. This can be a folder of tif files, a hdf file, or a 3D array. Parameters ---------- input_ : str, hdf file, or array_like It can be a folder path to tif files, a hdf file, or a 3D array. percentile : tuple of floats Tuple of (min_percentile, max_percentile) to compute. Must be between 0 and 100 inclusive. skip : int Skipping step of reading input. denoise: bool, optional Enable/disable denoising before extracting statistical information. key_path : str, optional Key path to the dataset if the input is the hdf file. Returns ------- gmin : float The global minimum value of the data array. gmax : float The global maximum value of the data array. min_percent : float The global min of the first computed percentile of the data array. max_percent : tuple of floats The global min of the last computed percentile of the data array. mean : float The mean of the data array. median : float The median of the data array. variance : float The mean of the variance of the data array. """ if isinstance(input_, str) and (os.path.splitext(input_)[-1] == ""): list_file = losa.find_file(input_ + "/*.tif*") depth = len(list_file) if depth == 0: raise ValueError("No tif files in the folder: {}".format(input_)) list_stat = [] for i in range(0, depth, skip): mat = losa.load_image(list_file[i]) if denoise is True: mat = gaussian_filter(mat, 2) list_stat.append(get_statical_information(mat, percentile, denoise)) else: if isinstance(input_, str): file_ext = os.path.splitext(input_)[-1] if not (file_ext == '.hdf' or file_ext == '.h5' or file_ext == ".nxs"): raise ValueError( "Can't open this type of file format {}".format(file_ext)) if key_path is None: raise ValueError( "Please provide the key path to the dataset!!!") input_ = losa.load_hdf(input_, key_path) depth = len(input_) list_stat = [] for i in range(0, depth, skip): mat = input_[i] if denoise is True: mat = gaussian_filter(mat, 2) list_stat.append(get_statical_information(mat, percentile, denoise)) list_stat = np.asarray(list_stat) gmin = np.min(list_stat[:, 0]) gmax = np.max(list_stat[:, 1]) min_percent = np.min(list_stat[:, 2]) max_percent = np.max(list_stat[:, 3]) median = np.median(list_stat[:, 4]) mean = np.mean(list_stat[:, 5]) variance = np.mean(list_stat[:, 6]) return gmin, gmax, min_percent, max_percent, mean, median, variance
def rescale_dataset(input_, output, nbit=16, minmax=None, skip=None, key_path=None): """ Rescale a dataset to 8-bit or 16-bit data-type. The dataset can be a folder of tif files, a hdf file, or a 3D array. Parameters ---------- input_ : str, array_like It can be a folder path to tif files, a hdf file, or 3D array. output : str, None It can be a folder path, a hdf file path, or None (memory consuming). nbit : {8,16} Rescaled data-type: 8-bit or 16-bit. minmax : tuple of float, or None Minimum and maximum values used for rescaling. They are calculated if None is given. skip : int or None Skipping step of reading input used for getting statistical information. key_path : str, optional Key path to the dataset if the input is the hdf file. Returns ------- array_like or None If output is None, returning an 3D array. """ if output is not None: file_base, file_ext = os.path.splitext(output) if file_ext != "": file_base = os.path.dirname(output) if os.path.exists(file_base): raise ValueError("Folder exists!!! Please choose another path!!!") if isinstance(input_, str) and (os.path.splitext(input_)[-1] == ""): list_file = losa.find_file(input_ + "/*.tif*") depth = len(list_file) if depth == 0: raise ValueError("No tif files in the folder: {}".format(input_)) if minmax is None: if skip is None: skip = int(np.ceil(0.15 * depth)) (gmin, gmax) = get_statical_information_dataset(input_, skip=skip)[0:2] else: (gmin, gmax) = minmax if output is not None: file_base, file_ext = os.path.splitext(output) if file_ext != "": if not (file_ext == '.hdf' or file_ext == '.h5' or file_ext == ".nxs"): raise ValueError("File extension must be hdf, h5, or nxs") output = file_base + file_ext (height, width) = np.shape(losa.load_image(list_file[0])) if nbit == 8: data_type = "uint8" else: data_type = "uint16" data_out = losa.open_hdf_stream(output, (depth, height, width), key_path="rescale/data", data_type=data_type, overwrite=False) data_res = [] for i in range(0, depth): mat = rescale(losa.load_image(list_file[i]), nbit=nbit, minmax=(gmin, gmax)) if output is None: data_res.append(mat) else: file_base, file_ext = os.path.splitext(output) if file_ext == "": out_name = "0000" + str(i) losa.save_image(output + "/img_" + out_name[-5:] + ".tif", mat) else: data_out[i] = mat else: if isinstance(input_, str): file_ext = os.path.splitext(input_)[-1] if not (file_ext == '.hdf' or file_ext == '.h5' or file_ext == ".nxs"): raise ValueError( "Can't open this type of file format {}".format(file_ext)) if key_path is None: raise ValueError( "Please provide the key path to the dataset!!!") input_ = losa.load_hdf(input_, key_path) (depth, height, width) = input_.shape if minmax is None: if skip is None: skip = int(np.ceil(0.15 * depth)) (gmin, gmax) = get_statical_information_dataset(input_, skip=skip, key_path=key_path)[0:2] else: (gmin, gmax) = minmax data_res = [] if output is not None: file_base, file_ext = os.path.splitext(output) if file_ext != "": if not (file_ext == '.hdf' or file_ext == '.h5' or file_ext == ".nxs"): raise ValueError("File extension must be hdf, h5, or nxs") output = file_base + file_ext if nbit == 8: data_type = "uint8" else: data_type = "uint16" data_out = losa.open_hdf_stream(output, (depth, height, width), key_path="rescale/data", data_type=data_type, overwrite=False) for i in range(0, depth): mat = rescale(input_[i], nbit=nbit, minmax=(gmin, gmax)) if output is None: data_res.append(mat) else: file_base, file_ext = os.path.splitext(output) if file_ext != "": data_out[i] = mat else: out_name = "0000" + str(i) losa.save_image(output + "/img_" + out_name[-5:] + ".tif", mat) if output is None: return np.asarray(data_res)
def downsample_dataset(input_, output, cell_size, method="mean", key_path=None): """ Downsample a dataset. This can be a folder of tif files, a hdf file, or a 3D array. Parameters ---------- input_ : str, array_like It can be a folder path to tif files, a hdf file, or 3D array. output : str, None It can be a folder path, a hdf file path, or None (memory consuming). cell_size : int or tuple of int Window size along axes used for grouping pixels. method : {"mean", "median", "max", "min"} Downsampling method. key_path : str, optional Key path to the dataset if the input is the hdf file. Returns ------- array_like or None If output is None, returning an 3D array. """ if output is not None: file_base, file_ext = os.path.splitext(output) if file_ext != "": file_base = os.path.dirname(output) if os.path.exists(file_base): raise ValueError("Folder exists!!! Please choose another path!!!") if method == "median": dsp_method = np.median elif method == "max": dsp_method = np.max elif method == "min": dsp_method = np.amin else: dsp_method = np.mean if isinstance(cell_size, int): cell_size = (cell_size, cell_size, cell_size) if isinstance(input_, str) and (os.path.splitext(input_)[-1] == ""): list_file = losa.find_file(input_ + "/*.tif*") depth = len(list_file) if depth == 0: raise ValueError("No tif files in the folder: {}".format(input_)) (height, width) = np.shape(losa.load_image(list_file[0])) depth_dsp = depth // cell_size[0] height_dsp = height // cell_size[1] width_dsp = width // cell_size[2] num = 0 if (depth_dsp != 0) and (height_dsp != 0) and (width_dsp != 0): if output is not None: file_base, file_ext = os.path.splitext(output) if file_ext != "": if not (file_ext == '.hdf' or file_ext == '.h5' or file_ext == ".nxs"): raise ValueError( "File extension must be hdf, h5, or nxs") output = file_base + file_ext data_out = losa.open_hdf_stream( output, (depth_dsp, height_dsp, width_dsp), key_path="downsample/data", overwrite=False) data_dsp = [] for i in range(0, depth, cell_size[0]): if (i + cell_size[0]) > depth: break else: mat = [] for j in range(i, i + cell_size[0]): mat.append(losa.load_image(list_file[j])) mat = np.asarray(mat) mat = mat[:, :height_dsp * cell_size[1], :width_dsp * cell_size[2]] mat = mat.reshape(1, cell_size[0], height_dsp, cell_size[1], width_dsp, cell_size[2]) mat_dsp = dsp_method(dsp_method(dsp_method(mat, axis=-1), axis=1), axis=2) if output is None: data_dsp.append(mat_dsp[0]) else: if file_ext == "": out_name = "0000" + str(num) losa.save_image( output + "/img_" + out_name[-5:] + ".tif", mat_dsp[0]) else: data_out[num] = mat_dsp[0] num += 1 else: raise ValueError("Incorrect cell size {}".format(cell_size)) else: if isinstance(input_, str): file_ext = os.path.splitext(input_)[-1] if not (file_ext == '.hdf' or file_ext == '.h5' or file_ext == ".nxs"): raise ValueError( "Can't open this type of file format {}".format(file_ext)) if key_path is None: raise ValueError( "Please provide the key path to the dataset!!!") input_ = losa.load_hdf(input_, key_path) (depth, height, width) = input_.shape depth_dsp = depth // cell_size[0] height_dsp = height // cell_size[1] width_dsp = width // cell_size[2] if (depth_dsp != 0) and (height_dsp != 0) and (width_dsp != 0): if output is None: input_ = input_[:depth_dsp * cell_size[0], :height_dsp * cell_size[1], :width_dsp * cell_size[2]] input_ = input_.reshape(depth_dsp, cell_size[0], height_dsp, cell_size[1], width_dsp, cell_size[2]) data_dsp = dsp_method(dsp_method(dsp_method(input_, axis=-1), axis=1), axis=2) else: file_base, file_ext = os.path.splitext(output) if file_ext != "": if not (file_ext == '.hdf' or file_ext == '.h5' or file_ext == ".nxs"): raise ValueError( "File extension must be hdf, h5, or nxs") output = file_base + file_ext data_out = losa.open_hdf_stream( output, (depth_dsp, height_dsp, width_dsp), key_path="downsample/data", overwrite=False) num = 0 for i in range(0, depth, cell_size[0]): if (i + cell_size[0]) > depth: break else: mat = input_[i:i + cell_size[0], :height_dsp * cell_size[1], :width_dsp * cell_size[2]] mat = mat.reshape(1, cell_size[0], height_dsp, cell_size[1], width_dsp, cell_size[2]) mat_dsp = dsp_method(dsp_method(dsp_method(mat, axis=-1), axis=1), axis=2) if file_ext != "": data_out[num] = mat_dsp[0] else: out_name = "0000" + str(num) losa.save_image( output + "/img_" + out_name[-5:] + ".tif", mat_dsp[0]) num += 1 else: raise ValueError("Incorrect cell size {}".format(cell_size)) if output is None: return np.asarray(data_dsp)
prefix = "0000" + str(df_scan) df_name = "scan_" + prefix[-5:] proj_name = [] for i in proj_scan: prefix = "0000" + str(i) proj_name.append("scan_" + prefix[-5:]) # Separate scans to 8 rows x 3 columns num_scan_total = len(proj_scan) num_scan_col = 3 num_scan_row = num_scan_total // num_scan_col scan_grid = np.reshape(proj_scan, (num_scan_row, num_scan_col)) overlap_window = 100 # Used to calculate the overlap-area and overlap-side. # Load dark-field and flat-field images, average each of them. flat_path = losa.find_file(input_base + "/" + df_name + "/*flat*")[0] flat_field = np.mean(losa.load_hdf(flat_path, key_path)[:], axis=0) dark_path = losa.find_file(input_base + "/" + df_name + "/*dark*")[0] dark_field = np.mean(losa.load_hdf(dark_path, key_path)[:], axis=0) # Load projection images of each scan as hdf objects data_objects = [] list_depth = [] list_height = [] list_width = [] for i in range(num_scan_total): file_path = losa.find_file(input_base + "/" + proj_name[i] + "/*proj*")[0] hdf_object = losa.load_hdf(file_path, key_path) (depth1, height1, width1) = hdf_object.shape list_depth.append(depth1) list_height.append(height1) list_width.append(width1)
# Initial parameters get_trans_dark_signal = True num_use = 40 # Number of speckle positions used for phase retrieval. gpu = True dim = 1 # Use 1D/2D-searching for finding shifts win_size = 7 # Size of window around each pixel margin = 10 # Searching range for finding shifts slice_idx = 1200 # Slice to reconstruct print("********************************") print("*************Start**************") print("********************************") list_file = losa.find_file(input_base + "/*.nxs") # Get keys to datasets data_key = losa.find_hdf_key(list_file[0], "data/data")[0][0] image_key = losa.find_hdf_key(list_file[0], "image_key")[0][-1] data_obj = losa.load_hdf(list_file[0], data_key) (height, width) = data_obj.shape[-2:] # Define the ROI area to retrieve phase around the given slice (row) index crop_top = slice_idx - 2 * margin crop_bot = height - (slice_idx + 2 * margin) crop_left = 0 crop_right = 0 crop = (crop_top, crop_bot, crop_left, crop_right) # Get number of projections. num_proj = []
import algotom.io.loadersaver as losa import algotom.prep.phase as ps # Input examples for beamline K11, DLS input_base = "/dls/k11/data/2022/cm31131-1/nexus/" sam_num = np.arange(15533, 15665 + 3, 3) ref_num = np.arange(15532, 15664 + 3, 3) dark_field_num = 15528 # Output base output_base = "/dls/k11/data/2022/cm31131-1/spool/speckle_phase/processed_projections/" sam_path = [] for scan in sam_num: sam_path.append( losa.find_file(input_base + "/k11-" + str(scan) + "/" + "*imaging*")[0]) ref_path = [] for scan in ref_num: ref_path.append( losa.find_file(input_base + "/k11-" + str(scan) + "/" + "*imaging*")[0]) dark_field_path = losa.find_file(input_base + "/k11-" + str(dark_field_num) + "/" + "*imaging*")[0] data_key = "entry/detector/detector" # Get dark-field image (camera noise). Note that it's different to dark-signal image. dark_field = np.mean(losa.load_hdf(dark_field_path, data_key)[:], axis=0) # Initial parameters get_trans_dark_signal = True num_use = 40 # Number of speckle positions used for phase retrieval. gpu = True # Use GPU for computing
def test_get_image_stack(self): num_stack = 3 for i in range(num_stack): file_path = "data/img_stk_" + str(i) + ".hdf" ifile = h5py.File(file_path, "w") data = np.ones((3, 64, 64)) ifile.create_dataset("entry/data", data=np.float32(data)) ifile.close() list_path = losa.find_file("data/img_stk*") f_alias = losa.get_image_stack img_stack = f_alias(0, list_path, data_key="entry/data", average=False, crop=(0, 0, 0, 0), flat_field=None, dark_field=None, num_use=None, fix_zero_div=False) check1 = True if (img_stack.shape[0] == num_stack) \ and (np.mean(img_stack) == 1.0) else False img_stack = f_alias(4, list_path, data_key="entry/data", average=True, crop=(0, 0, 0, 0), flat_field=None, dark_field=None, num_use=None, fix_zero_div=False) check2 = True if (img_stack.shape[0] == num_stack) \ and (np.mean(img_stack) == 1.0) else False num_img = 4 for i in range(num_stack): folder_path = "data/img_stk_tif_" + str(i) + "/" for j in range(num_img): file_path = folder_path + "/img_" + str(j) + ".tif" losa.save_image(file_path, np.ones((64, 64))) list_path = losa.find_file("data/img_stk_tif*") img_stack = f_alias(0, list_path, data_key=None, average=False, crop=(0, 0, 0, 0), flat_field=None, dark_field=None, num_use=None, fix_zero_div=False) check3 = True if (img_stack.shape[0] == num_stack) \ and (np.mean(img_stack) == 1.0) else False img_stack = f_alias(4, list_path, data_key=None, average=True, crop=(0, 0, 0, 0), flat_field=None, dark_field=None, num_use=None, fix_zero_div=False) check4 = True if (img_stack.shape[0] == num_stack) \ and (np.mean(img_stack) == 1.0) else False img_stack = f_alias(None, list_path[0], data_key=None, average=False, crop=(0, 0, 0, 0), flat_field=None, dark_field=None, num_use=None, fix_zero_div=False) check5 = True if (img_stack.shape[0] == num_img) \ and (np.mean(img_stack) == 1.0) else False self.assertTrue(check1 and check2 and check3 and check4 and check5)
scan_name = args.scan_name flat_name = args.flat_name dark_name = args.dark_name center = args.center crop_left = args.cleft crop_right = args.cright start_slice = args.startslice stop_slice = args.stopslice step_slice = args.stepslice input_base = "/home/user_id/raw_data/" output_base = "/home/user_id/processing/reconstruction/" proj_path = losa.find_file(input_base + "/*" + scan_name + "*.hdf")[0] flat_path = losa.find_file(input_base + "/*" + flat_name + "*.hdf")[0] dark_path = losa.find_file(input_base + "/*" + dark_name + "*.hdf")[0] hdf_key = "/entry/data/data" output_name = losa.make_folder_name(output_base, "Recon_few_slices") print("******************") print("Run the script....") print("******************") # Add body here # ... # ... print("******************")