def test_get_reference_sample_stacks(self):
num_stack = 3
for i in range(num_stack):
file_path = "data/speck_" + str(i) + ".hdf"
ifile = h5py.File(file_path, "w")
data = np.ones((2, 64, 64))
ifile.create_dataset("entry/speck", data=np.float32(data))
ifile.close()
file_path = "data/tomo_" + str(i) + ".hdf"
ifile = h5py.File(file_path, "w")
data = 2 * np.ones((4, 64, 64))
ifile.create_dataset("entry/tomo", data=np.float32(data))
ifile.close()
ref_path = losa.find_file("data/speck*")
sam_path = losa.find_file("data/tomo*")
ref_key, sam_key = "entry/speck", "entry/tomo"
f_alias = losa.get_reference_sample_stacks
ref_stack, sam_stack = f_alias(0,
ref_path,
sam_path,
ref_key,
sam_key,
crop=(0, 0, 0, 0),
flat_field=None,
dark_field=None,
num_use=None,
fix_zero_div=False)
check1 = True if (ref_stack.shape[0] == num_stack) \
and (sam_stack.shape[0] == num_stack) else False
check2 = True if (np.mean(ref_stack) == 1.0) \
and (np.mean(sam_stack) == 2.0) else False
self.assertTrue(check1 and check2)
def convert_tif_to_hdf(input_path,
output_path,
key_path="entry/data",
crop=(0, 0, 0, 0),
pattern=None,
**options):
"""
Convert a folder of tif files to a hdf/nxs file.
Parameters
----------
input_path : str
Folder path to the tif files.
output_path : str
Path to the hdf/nxs file.
key_path : str, optional
Key path to the dataset.
crop : tuple of int, optional
Crop the images from the edges, i.e.
crop = (crop_top, crop_bottom, crop_left, crop_right).
pattern : str, optional
Used to find tif files with names matching the pattern.
options : dict, optional
Add metadata. E.g. options={"entry/angles": angles, "entry/energy": 53}.
Returns
-------
str
Path to the hdf/nxs file.
"""
if pattern is None:
list_file = losa.find_file(input_path + "/*.tif*")
else:
list_file = losa.find_file(input_path + "/*" + pattern + "*.tif*")
depth = len(list_file)
if depth == 0:
raise ValueError("No tif files in the folder: {}".format(input_path))
(height, width) = np.shape(losa.load_image(list_file[0]))
file_base, file_ext = os.path.splitext(output_path)
if not (file_ext == '.hdf' or file_ext == '.h5' or file_ext == ".nxs"):
file_ext = '.hdf'
output_path = file_base + file_ext
cr_top, cr_bottom, cr_left, cr_right = crop
cr_height = height - cr_top - cr_bottom
cr_width = width - cr_left - cr_right
data_out = losa.open_hdf_stream(output_path, (depth, cr_height, cr_width),
key_path=key_path,
overwrite=True,
**options)
for i, fname in enumerate(list_file):
data_out[i] = losa.load_image(fname)[cr_top:cr_height + cr_top,
cr_left:cr_width + cr_left]
return output_path
def test_get_reference_sample_stacks_dls(self):
num_stack = 3
for i in range(num_stack):
file_path = "data/speck_tomo_" + str(i) + ".hdf"
ifile = h5py.File(file_path, "w")
data = np.concatenate((np.zeros((2, 64, 64)), np.ones(
(3, 64, 64)), 2 * np.ones((4, 64, 64))),
axis=0)
ifile.create_dataset("entry/data/data", data=np.float32(data))
keys = np.concatenate((2.0 * np.ones(2), np.ones(3), np.zeros(4)))
ifile.create_dataset("entry/data/image_key", data=np.float32(keys))
ifile.close()
list_path = losa.find_file("data/speck_tomo*")
f_alias = losa.get_reference_sample_stacks_dls
ref_stack, sam_stack = f_alias(0,
list_path,
data_key=None,
image_key=None,
crop=(0, 0, 0, 0),
flat_field=None,
dark_field=None,
num_use=None,
fix_zero_div=False)
check1 = True if (ref_stack.shape[0] == num_stack) \
and (sam_stack.shape[0] == num_stack) else False
check2 = True if (np.mean(ref_stack) == 1.0) \
and (np.mean(sam_stack) == 2.0) else False
self.assertTrue(check1 and check2)
def test_extract_tif_from_hdf(self):
file_path = "data/data.hdf"
out_base = "data/extract_tif/"
ifile = h5py.File(file_path, "w")
ifile.create_dataset("entry/data", data=np.random.rand(2, 64, 64))
ifile.close()
con.extract_tif_from_hdf(file_path, out_base, "entry/data")
list_file = losa.find_file(out_base + "/*.tif")
self.assertTrue(len(list_file) == 2)
factor of 8 without an intermediate step of combining 12 files to 1 huge
file (11.3 TB in total).
"""
import timeit
import numpy as np
import algotom.io.loadersaver as losa
input_base = "D:/Full_reconstruction/"
# Where to save the outputs
output_base = "D:/Dsp_grid_scan/"
output_file = "full_size_dsp_8_8_8.hdf"
cube = (8, 8, 8) # Downsampling factor
list_file = losa.find_file(input_base + "*.hdf")
key_path = "entry/data"
list_hdf_object = []
num_file = len(list_file)
list_nslice = []
for i in range(num_file):
hdf_object = losa.load_hdf(list_file[i], key_path)
list_hdf_object.append(hdf_object)
(nslice, height, width) = hdf_object.shape
list_nslice.append(nslice)
total_slice = np.sum(np.asarray(list_nslice))
total_slice_r = (total_slice // cube[0]) * cube[0]
height_r = (height // cube[1]) * cube[1]
width_r = (width // cube[2]) * cube[2]
def get_statical_information_dataset(input_,
percentile=(5, 95),
skip=5,
denoise=False,
key_path=None):
"""
Get statical information of a dataset. This can be a folder of tif files,
a hdf file, or a 3D array.
Parameters
----------
input_ : str, hdf file, or array_like
It can be a folder path to tif files, a hdf file, or a 3D array.
percentile : tuple of floats
Tuple of (min_percentile, max_percentile) to compute.
Must be between 0 and 100 inclusive.
skip : int
Skipping step of reading input.
denoise: bool, optional
Enable/disable denoising before extracting statistical information.
key_path : str, optional
Key path to the dataset if the input is the hdf file.
Returns
-------
gmin : float
The global minimum value of the data array.
gmax : float
The global maximum value of the data array.
min_percent : float
The global min of the first computed percentile of the data array.
max_percent : tuple of floats
The global min of the last computed percentile of the data array.
mean : float
The mean of the data array.
median : float
The median of the data array.
variance : float
The mean of the variance of the data array.
"""
if isinstance(input_, str) and (os.path.splitext(input_)[-1] == ""):
list_file = losa.find_file(input_ + "/*.tif*")
depth = len(list_file)
if depth == 0:
raise ValueError("No tif files in the folder: {}".format(input_))
list_stat = []
for i in range(0, depth, skip):
mat = losa.load_image(list_file[i])
if denoise is True:
mat = gaussian_filter(mat, 2)
list_stat.append(get_statical_information(mat, percentile,
denoise))
else:
if isinstance(input_, str):
file_ext = os.path.splitext(input_)[-1]
if not (file_ext == '.hdf' or file_ext == '.h5'
or file_ext == ".nxs"):
raise ValueError(
"Can't open this type of file format {}".format(file_ext))
if key_path is None:
raise ValueError(
"Please provide the key path to the dataset!!!")
input_ = losa.load_hdf(input_, key_path)
depth = len(input_)
list_stat = []
for i in range(0, depth, skip):
mat = input_[i]
if denoise is True:
mat = gaussian_filter(mat, 2)
list_stat.append(get_statical_information(mat, percentile,
denoise))
list_stat = np.asarray(list_stat)
gmin = np.min(list_stat[:, 0])
gmax = np.max(list_stat[:, 1])
min_percent = np.min(list_stat[:, 2])
max_percent = np.max(list_stat[:, 3])
median = np.median(list_stat[:, 4])
mean = np.mean(list_stat[:, 5])
variance = np.mean(list_stat[:, 6])
return gmin, gmax, min_percent, max_percent, mean, median, variance
def rescale_dataset(input_,
output,
nbit=16,
minmax=None,
skip=None,
key_path=None):
"""
Rescale a dataset to 8-bit or 16-bit data-type. The dataset can be a
folder of tif files, a hdf file, or a 3D array.
Parameters
----------
input_ : str, array_like
It can be a folder path to tif files, a hdf file, or 3D array.
output : str, None
It can be a folder path, a hdf file path, or None (memory consuming).
nbit : {8,16}
Rescaled data-type: 8-bit or 16-bit.
minmax : tuple of float, or None
Minimum and maximum values used for rescaling. They are calculated if
None is given.
skip : int or None
Skipping step of reading input used for getting statistical information.
key_path : str, optional
Key path to the dataset if the input is the hdf file.
Returns
-------
array_like or None
If output is None, returning an 3D array.
"""
if output is not None:
file_base, file_ext = os.path.splitext(output)
if file_ext != "":
file_base = os.path.dirname(output)
if os.path.exists(file_base):
raise ValueError("Folder exists!!! Please choose another path!!!")
if isinstance(input_, str) and (os.path.splitext(input_)[-1] == ""):
list_file = losa.find_file(input_ + "/*.tif*")
depth = len(list_file)
if depth == 0:
raise ValueError("No tif files in the folder: {}".format(input_))
if minmax is None:
if skip is None:
skip = int(np.ceil(0.15 * depth))
(gmin, gmax) = get_statical_information_dataset(input_,
skip=skip)[0:2]
else:
(gmin, gmax) = minmax
if output is not None:
file_base, file_ext = os.path.splitext(output)
if file_ext != "":
if not (file_ext == '.hdf' or file_ext == '.h5'
or file_ext == ".nxs"):
raise ValueError("File extension must be hdf, h5, or nxs")
output = file_base + file_ext
(height, width) = np.shape(losa.load_image(list_file[0]))
if nbit == 8:
data_type = "uint8"
else:
data_type = "uint16"
data_out = losa.open_hdf_stream(output, (depth, height, width),
key_path="rescale/data",
data_type=data_type,
overwrite=False)
data_res = []
for i in range(0, depth):
mat = rescale(losa.load_image(list_file[i]),
nbit=nbit,
minmax=(gmin, gmax))
if output is None:
data_res.append(mat)
else:
file_base, file_ext = os.path.splitext(output)
if file_ext == "":
out_name = "0000" + str(i)
losa.save_image(output + "/img_" + out_name[-5:] + ".tif",
mat)
else:
data_out[i] = mat
else:
if isinstance(input_, str):
file_ext = os.path.splitext(input_)[-1]
if not (file_ext == '.hdf' or file_ext == '.h5'
or file_ext == ".nxs"):
raise ValueError(
"Can't open this type of file format {}".format(file_ext))
if key_path is None:
raise ValueError(
"Please provide the key path to the dataset!!!")
input_ = losa.load_hdf(input_, key_path)
(depth, height, width) = input_.shape
if minmax is None:
if skip is None:
skip = int(np.ceil(0.15 * depth))
(gmin,
gmax) = get_statical_information_dataset(input_,
skip=skip,
key_path=key_path)[0:2]
else:
(gmin, gmax) = minmax
data_res = []
if output is not None:
file_base, file_ext = os.path.splitext(output)
if file_ext != "":
if not (file_ext == '.hdf' or file_ext == '.h5'
or file_ext == ".nxs"):
raise ValueError("File extension must be hdf, h5, or nxs")
output = file_base + file_ext
if nbit == 8:
data_type = "uint8"
else:
data_type = "uint16"
data_out = losa.open_hdf_stream(output, (depth, height, width),
key_path="rescale/data",
data_type=data_type,
overwrite=False)
for i in range(0, depth):
mat = rescale(input_[i], nbit=nbit, minmax=(gmin, gmax))
if output is None:
data_res.append(mat)
else:
file_base, file_ext = os.path.splitext(output)
if file_ext != "":
data_out[i] = mat
else:
out_name = "0000" + str(i)
losa.save_image(output + "/img_" + out_name[-5:] + ".tif",
mat)
if output is None:
return np.asarray(data_res)
def downsample_dataset(input_,
output,
cell_size,
method="mean",
key_path=None):
"""
Downsample a dataset. This can be a folder of tif files, a hdf file,
or a 3D array.
Parameters
----------
input_ : str, array_like
It can be a folder path to tif files, a hdf file, or 3D array.
output : str, None
It can be a folder path, a hdf file path, or None (memory consuming).
cell_size : int or tuple of int
Window size along axes used for grouping pixels.
method : {"mean", "median", "max", "min"}
Downsampling method.
key_path : str, optional
Key path to the dataset if the input is the hdf file.
Returns
-------
array_like or None
If output is None, returning an 3D array.
"""
if output is not None:
file_base, file_ext = os.path.splitext(output)
if file_ext != "":
file_base = os.path.dirname(output)
if os.path.exists(file_base):
raise ValueError("Folder exists!!! Please choose another path!!!")
if method == "median":
dsp_method = np.median
elif method == "max":
dsp_method = np.max
elif method == "min":
dsp_method = np.amin
else:
dsp_method = np.mean
if isinstance(cell_size, int):
cell_size = (cell_size, cell_size, cell_size)
if isinstance(input_, str) and (os.path.splitext(input_)[-1] == ""):
list_file = losa.find_file(input_ + "/*.tif*")
depth = len(list_file)
if depth == 0:
raise ValueError("No tif files in the folder: {}".format(input_))
(height, width) = np.shape(losa.load_image(list_file[0]))
depth_dsp = depth // cell_size[0]
height_dsp = height // cell_size[1]
width_dsp = width // cell_size[2]
num = 0
if (depth_dsp != 0) and (height_dsp != 0) and (width_dsp != 0):
if output is not None:
file_base, file_ext = os.path.splitext(output)
if file_ext != "":
if not (file_ext == '.hdf' or file_ext == '.h5'
or file_ext == ".nxs"):
raise ValueError(
"File extension must be hdf, h5, or nxs")
output = file_base + file_ext
data_out = losa.open_hdf_stream(
output, (depth_dsp, height_dsp, width_dsp),
key_path="downsample/data",
overwrite=False)
data_dsp = []
for i in range(0, depth, cell_size[0]):
if (i + cell_size[0]) > depth:
break
else:
mat = []
for j in range(i, i + cell_size[0]):
mat.append(losa.load_image(list_file[j]))
mat = np.asarray(mat)
mat = mat[:, :height_dsp * cell_size[1], :width_dsp *
cell_size[2]]
mat = mat.reshape(1, cell_size[0], height_dsp,
cell_size[1], width_dsp, cell_size[2])
mat_dsp = dsp_method(dsp_method(dsp_method(mat, axis=-1),
axis=1),
axis=2)
if output is None:
data_dsp.append(mat_dsp[0])
else:
if file_ext == "":
out_name = "0000" + str(num)
losa.save_image(
output + "/img_" + out_name[-5:] + ".tif",
mat_dsp[0])
else:
data_out[num] = mat_dsp[0]
num += 1
else:
raise ValueError("Incorrect cell size {}".format(cell_size))
else:
if isinstance(input_, str):
file_ext = os.path.splitext(input_)[-1]
if not (file_ext == '.hdf' or file_ext == '.h5'
or file_ext == ".nxs"):
raise ValueError(
"Can't open this type of file format {}".format(file_ext))
if key_path is None:
raise ValueError(
"Please provide the key path to the dataset!!!")
input_ = losa.load_hdf(input_, key_path)
(depth, height, width) = input_.shape
depth_dsp = depth // cell_size[0]
height_dsp = height // cell_size[1]
width_dsp = width // cell_size[2]
if (depth_dsp != 0) and (height_dsp != 0) and (width_dsp != 0):
if output is None:
input_ = input_[:depth_dsp * cell_size[0], :height_dsp *
cell_size[1], :width_dsp * cell_size[2]]
input_ = input_.reshape(depth_dsp, cell_size[0], height_dsp,
cell_size[1], width_dsp, cell_size[2])
data_dsp = dsp_method(dsp_method(dsp_method(input_, axis=-1),
axis=1),
axis=2)
else:
file_base, file_ext = os.path.splitext(output)
if file_ext != "":
if not (file_ext == '.hdf' or file_ext == '.h5'
or file_ext == ".nxs"):
raise ValueError(
"File extension must be hdf, h5, or nxs")
output = file_base + file_ext
data_out = losa.open_hdf_stream(
output, (depth_dsp, height_dsp, width_dsp),
key_path="downsample/data",
overwrite=False)
num = 0
for i in range(0, depth, cell_size[0]):
if (i + cell_size[0]) > depth:
break
else:
mat = input_[i:i + cell_size[0], :height_dsp *
cell_size[1], :width_dsp * cell_size[2]]
mat = mat.reshape(1, cell_size[0], height_dsp,
cell_size[1], width_dsp,
cell_size[2])
mat_dsp = dsp_method(dsp_method(dsp_method(mat,
axis=-1),
axis=1),
axis=2)
if file_ext != "":
data_out[num] = mat_dsp[0]
else:
out_name = "0000" + str(num)
losa.save_image(
output + "/img_" + out_name[-5:] + ".tif",
mat_dsp[0])
num += 1
else:
raise ValueError("Incorrect cell size {}".format(cell_size))
if output is None:
return np.asarray(data_dsp)
prefix = "0000" + str(df_scan)
df_name = "scan_" + prefix[-5:]
proj_name = []
for i in proj_scan:
prefix = "0000" + str(i)
proj_name.append("scan_" + prefix[-5:])
# Separate scans to 8 rows x 3 columns
num_scan_total = len(proj_scan)
num_scan_col = 3
num_scan_row = num_scan_total // num_scan_col
scan_grid = np.reshape(proj_scan, (num_scan_row, num_scan_col))
overlap_window = 100 # Used to calculate the overlap-area and overlap-side.
# Load dark-field and flat-field images, average each of them.
flat_path = losa.find_file(input_base + "/" + df_name + "/*flat*")[0]
flat_field = np.mean(losa.load_hdf(flat_path, key_path)[:], axis=0)
dark_path = losa.find_file(input_base + "/" + df_name + "/*dark*")[0]
dark_field = np.mean(losa.load_hdf(dark_path, key_path)[:], axis=0)
# Load projection images of each scan as hdf objects
data_objects = []
list_depth = []
list_height = []
list_width = []
for i in range(num_scan_total):
file_path = losa.find_file(input_base + "/" + proj_name[i] + "/*proj*")[0]
hdf_object = losa.load_hdf(file_path, key_path)
(depth1, height1, width1) = hdf_object.shape
list_depth.append(depth1)
list_height.append(height1)
list_width.append(width1)
# Initial parameters
get_trans_dark_signal = True
num_use = 40 # Number of speckle positions used for phase retrieval.
gpu = True
dim = 1 # Use 1D/2D-searching for finding shifts
win_size = 7 # Size of window around each pixel
margin = 10 # Searching range for finding shifts
slice_idx = 1200 # Slice to reconstruct
print("********************************")
print("*************Start**************")
print("********************************")
list_file = losa.find_file(input_base + "/*.nxs")
# Get keys to datasets
data_key = losa.find_hdf_key(list_file[0], "data/data")[0][0]
image_key = losa.find_hdf_key(list_file[0], "image_key")[0][-1]
data_obj = losa.load_hdf(list_file[0], data_key)
(height, width) = data_obj.shape[-2:]
# Define the ROI area to retrieve phase around the given slice (row) index
crop_top = slice_idx - 2 * margin
crop_bot = height - (slice_idx + 2 * margin)
crop_left = 0
crop_right = 0
crop = (crop_top, crop_bot, crop_left, crop_right)
# Get number of projections.
num_proj = []
import algotom.io.loadersaver as losa
import algotom.prep.phase as ps
# Input examples for beamline K11, DLS
input_base = "/dls/k11/data/2022/cm31131-1/nexus/"
sam_num = np.arange(15533, 15665 + 3, 3)
ref_num = np.arange(15532, 15664 + 3, 3)
dark_field_num = 15528
# Output base
output_base = "/dls/k11/data/2022/cm31131-1/spool/speckle_phase/processed_projections/"
sam_path = []
for scan in sam_num:
sam_path.append(
losa.find_file(input_base + "/k11-" + str(scan) + "/" +
"*imaging*")[0])
ref_path = []
for scan in ref_num:
ref_path.append(
losa.find_file(input_base + "/k11-" + str(scan) + "/" +
"*imaging*")[0])
dark_field_path = losa.find_file(input_base + "/k11-" + str(dark_field_num) +
"/" + "*imaging*")[0]
data_key = "entry/detector/detector"
# Get dark-field image (camera noise). Note that it's different to dark-signal image.
dark_field = np.mean(losa.load_hdf(dark_field_path, data_key)[:], axis=0)
# Initial parameters
get_trans_dark_signal = True
num_use = 40 # Number of speckle positions used for phase retrieval.
gpu = True # Use GPU for computing
def test_get_image_stack(self):
num_stack = 3
for i in range(num_stack):
file_path = "data/img_stk_" + str(i) + ".hdf"
ifile = h5py.File(file_path, "w")
data = np.ones((3, 64, 64))
ifile.create_dataset("entry/data", data=np.float32(data))
ifile.close()
list_path = losa.find_file("data/img_stk*")
f_alias = losa.get_image_stack
img_stack = f_alias(0,
list_path,
data_key="entry/data",
average=False,
crop=(0, 0, 0, 0),
flat_field=None,
dark_field=None,
num_use=None,
fix_zero_div=False)
check1 = True if (img_stack.shape[0] == num_stack) \
and (np.mean(img_stack) == 1.0) else False
img_stack = f_alias(4,
list_path,
data_key="entry/data",
average=True,
crop=(0, 0, 0, 0),
flat_field=None,
dark_field=None,
num_use=None,
fix_zero_div=False)
check2 = True if (img_stack.shape[0] == num_stack) \
and (np.mean(img_stack) == 1.0) else False
num_img = 4
for i in range(num_stack):
folder_path = "data/img_stk_tif_" + str(i) + "/"
for j in range(num_img):
file_path = folder_path + "/img_" + str(j) + ".tif"
losa.save_image(file_path, np.ones((64, 64)))
list_path = losa.find_file("data/img_stk_tif*")
img_stack = f_alias(0,
list_path,
data_key=None,
average=False,
crop=(0, 0, 0, 0),
flat_field=None,
dark_field=None,
num_use=None,
fix_zero_div=False)
check3 = True if (img_stack.shape[0] == num_stack) \
and (np.mean(img_stack) == 1.0) else False
img_stack = f_alias(4,
list_path,
data_key=None,
average=True,
crop=(0, 0, 0, 0),
flat_field=None,
dark_field=None,
num_use=None,
fix_zero_div=False)
check4 = True if (img_stack.shape[0] == num_stack) \
and (np.mean(img_stack) == 1.0) else False
img_stack = f_alias(None,
list_path[0],
data_key=None,
average=False,
crop=(0, 0, 0, 0),
flat_field=None,
dark_field=None,
num_use=None,
fix_zero_div=False)
check5 = True if (img_stack.shape[0] == num_img) \
and (np.mean(img_stack) == 1.0) else False
self.assertTrue(check1 and check2 and check3 and check4 and check5)
scan_name = args.scan_name
flat_name = args.flat_name
dark_name = args.dark_name
center = args.center
crop_left = args.cleft
crop_right = args.cright
start_slice = args.startslice
stop_slice = args.stopslice
step_slice = args.stepslice
input_base = "/home/user_id/raw_data/"
output_base = "/home/user_id/processing/reconstruction/"
proj_path = losa.find_file(input_base + "/*" + scan_name + "*.hdf")[0]
flat_path = losa.find_file(input_base + "/*" + flat_name + "*.hdf")[0]
dark_path = losa.find_file(input_base + "/*" + dark_name + "*.hdf")[0]
hdf_key = "/entry/data/data"
output_name = losa.make_folder_name(output_base, "Recon_few_slices")
print("******************")
print("Run the script....")
print("******************")
# Add body here
# ...
# ...
print("******************")