def test_image_specific():
# build a temporary directory and save tensors inside
with tempfile.TemporaryDirectory() as tmp_dir:
# non-supported image (1 dimension)
test = np.array([1, 2, 3], dtype=np.float32)
path = os.path.join(tmp_dir, "test")
with pytest.raises(ValueError):
stack.save_image(test, path)
# non-supported image (6 dimensions)
test = np.zeros((8, 8, 8, 8, 8, 8), dtype=np.float32)
path = os.path.join(tmp_dir, "test")
with pytest.raises(ValueError):
stack.save_image(test, path)
# non-supported image (missing values)
test = np.zeros((8, 8), dtype=np.float32)
test[0, 0] = np.nan
path = os.path.join(tmp_dir, "test")
with pytest.raises(ValueError):
stack.save_image(test, path)
# non-supported extensions
test = np.zeros((8, 8), dtype=np.float32)
path = os.path.join(tmp_dir, "test.foo")
with pytest.raises(ValueError):
stack.save_image(test, path)
path = os.path.join(tmp_dir, "test")
with pytest.raises(ValueError):
stack.save_image(test, path, extension="bar")
def fct_to_process(i, n):
# simulate images
image, ground_truth = spots.simulate_image(
image_shape=image_shape,
image_dtype=image_dtype,
subpixel_factors=subpixel_factors,
voxel_size_z=voxel_size_z,
voxel_size_yx=voxel_size_yx,
n_spots=n,
random_n_spots=random_n_spots,
n_clusters=n_clusters,
random_n_clusters=random_n_clusters,
n_spots_cluster=n_spots_cluster,
random_n_spots_cluster=random_n_spots_cluster,
centered_cluster=False,
sigma_z=sigma_z,
sigma_yx=sigma_yx,
random_sigma=random_sigma,
amplitude=amplitude,
random_amplitude=random_amplitude,
noise_level=noise_level,
random_noise=random_noise)
# save image and ground truth
path = os.path.join(path_directory_image, "image_{0}.tif".format(i))
stack.save_image(image, path)
path = os.path.join(path_directory_gt, "gt_{0}.csv".format(i))
stack.save_data_to_csv(ground_truth, path)
# plot
path = os.path.join(path_directory_plot, "plot_{0}.png".format(i))
subpixel = True if subpixel_factors is not None else False
plot.plot_spots(
image,
ground_truth=None,
prediction=None,
subpixel=subpixel,
rescale=True,
contrast=False,
title="Number of mRNAs: {0}".format(len(ground_truth)),
framesize=(8, 8),
remove_frame=False,
path_output=path,
ext="png",
show=False)
return
def fct_to_process(i, n):
# simulate images
image, ground_truth = sim.simulate_image(
ndim=ndim,
n_spots=n_spots,
random_n_spots=random_n_spots,
n_clusters=n_clusters,
random_n_clusters=random_n_clusters,
n_spots_cluster=n,
random_n_spots_cluster=random_n_spots_cluster,
centered_cluster=centered_cluster,
image_shape=image_shape,
image_dtype=np.uint16,
subpixel_factors=subpixel_factors,
voxel_size=voxel_size,
sigma=sigma,
random_sigma=random_sigma,
amplitude=amplitude,
random_amplitude=random_amplitude,
noise_level=noise_level,
random_noise=random_noise)
# save image
path = os.path.join(path_directory_image, "image_{0}.tif".format(i))
stack.save_image(image, path)
# complete ground truth and save it
new_column = np.array([n] * len(ground_truth))
new_column = new_column[:, np.newaxis]
ground_truth = np.hstack([ground_truth, new_column])
path = os.path.join(path_directory_gt, "gt_{0}.csv".format(i))
stack.save_data_to_csv(ground_truth, path)
# plot
path = os.path.join(path_directory_plot, "plot_{0}.png".format(i))
image_mip = stack.maximum_projection(image)
plot.plot_images(
images=image_mip,
rescale=True,
titles=["Number of spots: {0}".format(len(ground_truth))],
framesize=(8, 8),
remove_frame=False,
path_output=path,
show=False)
return
def test_build_stack_from_path():
# build a temporary directory and save tensors inside
with tempfile.TemporaryDirectory() as tmp_dir:
# field of view
test_nuc = np.zeros((8, 8, 8), dtype=np.uint8)
test_cyt = np.zeros((8, 8, 8), dtype=np.uint8)
test_rna = np.zeros((8, 8, 8), dtype=np.uint8)
path_nuc = os.path.join(tmp_dir, "test_nuc.tif")
path_cyt = os.path.join(tmp_dir, "test_cyt.tif")
path_rna = os.path.join(tmp_dir, "test_rna.tif")
stack.save_image(test_nuc, path_nuc)
stack.save_image(test_cyt, path_cyt)
stack.save_image(test_rna, path_rna)
# build tensor from paths
paths = [path_nuc, path_cyt, path_rna]
tensor = multistack.build_stack_no_recipe(paths, input_dimension=3)
expected_tensor = np.zeros((1, 3, 8, 8, 8), dtype=np.uint8)
assert_array_equal(tensor, expected_tensor)
assert tensor.dtype == np.uint8
# wrong paths
paths = [path_nuc, path_cyt, "/foo/bar/test_rna.tif"]
with pytest.raises(FileNotFoundError):
multistack.build_stack_no_recipe(paths, input_dimension=3)
def test_image(shape, dtype, extension):
# build a temporary directory and save tensors inside
with tempfile.TemporaryDirectory() as tmp_dir:
test = np.zeros(shape, dtype=dtype)
path = os.path.join(tmp_dir, "test." + extension)
# error: boolean multidimensional image
if (extension in ["png", "jpg", "jpeg", "tif", "tiff"]
and len(test.shape) > 2 and test.dtype == bool):
with pytest.raises(ValueError):
stack.save_image(test, path)
# error: non-boolean multidimensional image with 'png', 'jpg' or 'jpeg'
elif (extension in ["png", "jpg", "jpeg"] and len(test.shape) > 2
and test.dtype != bool):
with pytest.raises(ValueError):
stack.save_image(test, path)
# error: boolean 2-d image with 'tig' and 'tiff'
elif (extension in ["tif", "tiff"] and len(test.shape) == 2
and test.dtype == bool):
with pytest.raises(ValueError):
stack.save_image(test, path)
# warning: 2-d image with 'png', 'jpg' or 'jpeg'
elif (extension in ["png", "jpg", "jpeg"] and len(test.shape) == 2):
with pytest.warns(UserWarning):
stack.save_image(test, path)
tensor = stack.read_image(path, sanity_check=True)
assert_array_equal(test, tensor)
# others valid images
else:
stack.save_image(test, path)
tensor = stack.read_image(path, sanity_check=True)
assert_array_equal(test, tensor)
assert test.dtype == tensor.dtype
print("\t", image.shape, image.dtype, filename)
# cyt
cyt = image[0, 1, :, :, :]
# get sigma
sigma_z, sigma_yx = detection.get_sigma(resolution_z=300,
resolution_yx=103,
psf_z=350, psf_yx=150)
sigma_log = (sigma_z, sigma_yx, sigma_yx)
sigma_background = (sigma_z*5, sigma_yx*5, sigma_yx*5)
# LoG filter
cyt_filtered_log = stack.log_filter(cyt, sigma_log, keep_dtype=True)
path = os.path.join(output_directory_log, filename + ".tiff")
stack.save_image(cyt_filtered_log, path)
# background filter
cyt_filtered_background = stack.remove_background_gaussian(
image=cyt, sigma=sigma_background)
path = os.path.join(output_directory_background, filename + ".tiff")
stack.save_image(cyt_filtered_background, path)
nb_images += 1
print("Done ({0} images)!".format(nb_images), "\n")
end_time = time.time()
duration = int(round((end_time - start_time) / 60))
print("Duration: {0} minutes.".format(duration))
# compute relief
relief = segmentation.build_cyt_relief(cyt_projected,
nuc_labelled=nuc_labelled,
mask_cyt=mask,
alpha=0.99)
# segment cytoplasm
cyt_mask = segmentation.cyt_watershed(relief=relief,
nuc_labelled=nuc_labelled,
mask=mask,
smooth=7)
# save cytoplasm mask
path = os.path.join(mask_cyt_directory, filename + ".png")
stack.save_image(cyt_mask, path)
# save plot segmentation
path = os.path.join(plot_directory, filename + ".png")
plot.plot_segmentation_boundary(tensor=cyt_projected_contrast,
mask_nuc=nuc_labelled,
mask_cyt=cyt_mask,
framesize=(10, 10),
remove_frame=True,
path_output=path,
ext="png",
show=False)
nb_images += 1
print()
def test_build_stacks_from_recipe():
# build a temporary directory and save tensors inside
with tempfile.TemporaryDirectory() as tmp_dir:
# field of view 1
test_nuc = np.zeros((8, 8, 8), dtype=np.uint8)
test_cyt = np.zeros((8, 8, 8), dtype=np.uint8)
test_rna = np.zeros((8, 8, 8), dtype=np.uint8)
path_nuc = os.path.join(tmp_dir, "test_nuc_1.tif")
path_cyt = os.path.join(tmp_dir, "test_cyt_1.tif")
path_rna = os.path.join(tmp_dir, "test_rna_1.tif")
stack.save_image(test_nuc, path_nuc)
stack.save_image(test_cyt, path_cyt)
stack.save_image(test_rna, path_rna)
# field of view 2
test_nuc = np.zeros((5, 5, 5), dtype=np.uint16)
test_cyt = np.zeros((5, 5, 5), dtype=np.uint16)
test_rna = np.zeros((5, 5, 5), dtype=np.uint16)
path_nuc = os.path.join(tmp_dir, "test_nuc_2.tif")
path_cyt = os.path.join(tmp_dir, "test_cyt_2.tif")
path_rna = os.path.join(tmp_dir, "test_rna_2.tif")
stack.save_image(test_nuc, path_nuc)
stack.save_image(test_cyt, path_cyt)
stack.save_image(test_rna, path_rna)
# define recipe to read tensors
recipe_1 = {
"fov": ["1", "2"],
"c": ["nuc", "cyt", "rna"],
"opt": "test",
"ext": "tif",
"pattern": "opt_c_fov.ext"
}
# build tensor without prior information
tensor = multistack.build_stack(recipe_1, input_folder=tmp_dir)
expected_tensor = np.zeros((1, 3, 8, 8, 8), dtype=np.uint8)
assert_array_equal(tensor, expected_tensor)
assert tensor.dtype == np.uint8
# build tensor with prior information
tensor = multistack.build_stack(recipe_1,
input_folder=tmp_dir,
input_dimension=3)
expected_tensor = np.zeros((1, 3, 8, 8, 8), dtype=np.uint8)
assert_array_equal(tensor, expected_tensor)
assert tensor.dtype == np.uint8
# build tensors with different fields of view
tensor = multistack.build_stack(recipe_1,
input_folder=tmp_dir,
input_dimension=3,
i_fov=0)
expected_tensor = np.zeros((1, 3, 8, 8, 8), dtype=np.uint8)
assert_array_equal(tensor, expected_tensor)
assert tensor.dtype == np.uint8
tensor = multistack.build_stack(recipe_1,
input_folder=tmp_dir,
input_dimension=3,
i_fov=1)
expected_tensor = np.zeros((1, 3, 5, 5, 5), dtype=np.uint16)
assert_array_equal(tensor, expected_tensor)
assert tensor.dtype == np.uint16
# wrong recipe
recipe_wrong = {
"fov": "test",
"c": ["nuc", "cyt", "rna"],
"ext": "tif",
"pattern": "fov_c.ext"
}
with pytest.raises(FileNotFoundError):
multistack.build_stack(recipe_wrong,
input_folder=tmp_dir,
input_dimension=3)
# wrong path
with pytest.raises(FileNotFoundError):
multistack.build_stack(recipe_1,
input_folder="/foo/bar",
input_dimension=3)
def test_build_stacks_from_datamap():
# build a temporary directory and save tensors inside
with tempfile.TemporaryDirectory() as tmp_dir:
# field of view 1
test_nuc = np.zeros((8, 8, 8), dtype=np.uint8)
test_cyt = np.zeros((8, 8, 8), dtype=np.uint8)
test_rna = np.zeros((8, 8, 8), dtype=np.uint8)
path_nuc = os.path.join(tmp_dir, "test_nuc_1.tif")
path_cyt = os.path.join(tmp_dir, "test_cyt_1.tif")
path_rna = os.path.join(tmp_dir, "test_rna_1.tif")
stack.save_image(test_nuc, path_nuc)
stack.save_image(test_cyt, path_cyt)
stack.save_image(test_rna, path_rna)
# field of view 2
test_nuc = np.zeros((5, 5, 5), dtype=np.uint16)
test_cyt = np.zeros((5, 5, 5), dtype=np.uint16)
test_rna = np.zeros((5, 5, 5), dtype=np.uint16)
path_nuc = os.path.join(tmp_dir, "test_nuc_2.tif")
path_cyt = os.path.join(tmp_dir, "test_cyt_2.tif")
path_rna = os.path.join(tmp_dir, "test_rna_2.tif")
stack.save_image(test_nuc, path_nuc)
stack.save_image(test_cyt, path_cyt)
stack.save_image(test_rna, path_rna)
# define datamap to read tensors
recipe_1 = {
"fov": ["1", "2"],
"c": ["nuc", "cyt", "rna"],
"opt": "test",
"ext": "tif",
"pattern": "opt_c_fov.ext"
}
recipe_2 = {
"fov": "2",
"c": ["nuc", "cyt", "rna"],
"opt": "test",
"ext": "tif",
"pattern": "opt_c_fov.ext"
}
data_map = [(recipe_1, tmp_dir), (recipe_2, tmp_dir)]
# build stacks from generator
generator = multistack.build_stacks(data_map, input_dimension=3)
expected_tensors = [
np.zeros((1, 3, 8, 8, 8), dtype=np.uint8),
np.zeros((1, 3, 5, 5, 5), dtype=np.uint16),
np.zeros((1, 3, 5, 5, 5), dtype=np.uint16)
]
for i, tensor in enumerate(generator):
expected_tensor = expected_tensors[i]
assert_array_equal(tensor, expected_tensor)
assert tensor.dtype == expected_tensor.dtype
# build stacks from generator with metadata
generator = multistack.build_stacks(data_map,
input_dimension=3,
return_origin=True)
expected_tensors = [
np.zeros((1, 3, 8, 8, 8), dtype=np.uint8),
np.zeros((1, 3, 5, 5, 5), dtype=np.uint16),
np.zeros((1, 3, 5, 5, 5), dtype=np.uint16)
]
expected_recipes = [recipe_1, recipe_1, recipe_2]
expected_i_fov = [0, 1, 0]
for i, (tensor, input_folder, recipe, i_fov) in enumerate(generator):
expected_tensor = expected_tensors[i]
assert_array_equal(tensor, expected_tensor)
assert tensor.dtype == expected_tensor.dtype
assert input_folder == tmp_dir
assert recipe == expected_recipes[i]
assert i_fov == expected_i_fov[i]
# wrong datamap
data_map = [(recipe_1, 3), (recipe_2, tmp_dir)]
generator = multistack.build_stacks(data_map, input_dimension=3)
with pytest.raises(TypeError):
next(generator)
data_map = [(recipe_1, "foo/bar"), (recipe_2, tmp_dir)]
generator = multistack.build_stacks(data_map, input_dimension=3)
with pytest.raises(NotADirectoryError):
next(generator)
mask_1 = stack.read_image(path)
# mask 2
path = os.path.join(mask_directory_2, filename_tiff)
mask_2 = stack.read_image(path)
# mask
mask = segmentation.merge_labels(mask_1, mask_2)
# postprocess mask and mask_1
mask_1 = segmentation.dilate_erode_labels(mask_1)
mask = segmentation.dilate_erode_labels(mask)
# save mask and mask_1
path = os.path.join(mask_directory_original, filename + ".png")
stack.save_image(mask_1, path)
path = os.path.join(mask_directory, filename + ".png")
stack.save_image(mask, path)
# plot
path = os.path.join(plot_directory_original, filename + ".png")
plot.plot_segmentation(nuc_focus,
mask_1,
rescale=False,
title=filename,
framesize=(15, 5),
remove_frame=True,
path_output=path,
ext="png",
show=False)
path = os.path.join(plot_directory, filename + ".png")
"\t '{0}' is not a valid file. Skipped.".format(filename_png))
continue
# filename
filename = str(filename_png.split(".")[0])
filename_base = str(filename.split("_")[:-1])
i = int(filename.split("_")[-1])
print("\t", filename)
# projection
path = os.path.join(projection_directory, filename_png)
nuc_projected = stack.read_image(path)
# mask
path = os.path.join(mask_directory, filename + ".tiff")
mask = stack.read_image(path)
# remove segmented nuclei
unsegmented_nuclei = segmentation.remove_segmented_nuc(
nuc_projected, mask)
path = os.path.join(removed_directory, filename_png)
stack.save_image(unsegmented_nuclei, path)
nb_images += 1
print("Done ({0} images)!".format(nb_images), "\n")
end_time = time.time()
duration = int(round((end_time - start_time) / 60))
print("Duration: {0} minutes.".format(duration))
cyt = image[0, 1, :, :, :]
# projections
nuc_focus = stack.focus_projection_fast(nuc,
proportion=0.7,
neighborhood_size=7)
nuc_focus = stack.rescale(nuc_focus, channel_to_stretch=0)
cyt_focus = stack.focus_projection_fast(cyt,
proportion=0.75,
neighborhood_size=7)
cyt_in_focus = stack.in_focus_selection(cyt,
proportion=0.80,
neighborhood_size=30)
cyt_mip = stack.maximum_projection(cyt_in_focus)
# save projections
path = os.path.join(output_directory_nuc_focus, filename + ".png")
stack.save_image(nuc_focus, path)
path = os.path.join(output_directory_cyt_focus, filename + ".png")
stack.save_image(cyt_focus, path)
path = os.path.join(output_directory_cyt_mip, filename + ".png")
stack.save_image(cyt_mip, path)
nb_images += 1
print("Done ({0} images)!".format(nb_images), "\n")
end_time = time.time()
duration = int(round((end_time - start_time) / 60))
print("Duration: {0} minutes.".format(duration))
def image_processing_function(image_loc, config):
# Read the image into a numpy array of format ZCYX
if isinstance(image_loc, str):
image_name = pathlib.Path(image_loc).stem
image = tifffile.imread(image_loc)
else:
# Establish connection with OMERO and actually connect
conn = omero.gateway.BlitzGateway(
host="omero1.bioch.ox.ac.uk",
port=4064,
# group=config["OMERO_group"],
username=config["OMERO_user"],
passwd=config["password"],
)
conn.connect()
conn.SERVICE_OPTS.setOmeroGroup(-1)
# Create a thread to keep the connection alive
ka_thread = threading.Thread(target=keep_connection_alive,
args=(conn, ))
ka_thread.daemon = True
ka_thread.start()
# Derive image and its name
image_name = pathlib.Path(image_loc[0]).stem
remote_image = conn.getObject("Image", image_loc[1])
image = np.array(
list(remote_image.getPrimaryPixels().getPlanes([
(z, c, 0) for z in range(0, remote_image.getSizeZ())
for c in range(0, remote_image.getSizeC())
])))
image = image.reshape(
image.shape[0] // remote_image.getSizeC(),
remote_image.getSizeC(),
image.shape[1],
image.shape[2],
)
# segment with cellpose
seg_img = np.max(image[:, config["seg_ch"], :, :], 0)
if config["cp_search_string"] in image_name:
seg_img = np.clip(seg_img, 0, config["cp_clip"])
seg_img = scipy.ndimage.median_filter(seg_img,
size=config["median_filter"])
model = models.Cellpose(gpu=config["gpu"], model_type="cyto")
channels = [0, 0] # greyscale segmentation
masks = model.eval(
seg_img,
channels=channels,
diameter=config["diameter"],
do_3D=config["do_3D"],
flow_threshold=config["flow_threshold"],
cellprob_threshold=config["cellprob_threshold"],
)[0]
# Calculate PSF
psf_z, psf_yx = calculate_psf(
config["voxel_size_z"],
config["voxel_size_yx"],
config["ex"],
config["em"],
config["NA"],
config["RI"],
config["microscope"],
)
sigma = detection.get_sigma(config["voxel_size_z"],
config["voxel_size_yx"], psf_z, psf_yx)
for image_channel in config["channels"]:
# detect spots
rna = image[:, image_channel, :, :]
# subtract background
rna_no_bg = []
for z in rna:
z_no_bg = subtract_background(z, config["bg_radius"])
rna_no_bg.append(z_no_bg)
rna = np.array(rna_no_bg)
# LoG filter
rna_log = stack.log_filter(rna, sigma)
# local maximum detection
mask = detection.local_maximum_detection(rna_log, min_distance=sigma)
# tresholding
if image_channel == config["smFISH_ch1"]:
threshold = config["smFISH_ch1_thresh"]
elif image_channel == config["smFISH_ch2"]:
threshold = config["smFISH_ch2_thresh"]
else:
print("smFISH channel and threshold not correctly defined!")
spots, _ = detection.spots_thresholding(rna_log, mask, threshold)
# detect and decompose clusters
spots_post_decomposition = detection.decompose_cluster(
rna,
spots,
config["voxel_size_z"],
config["voxel_size_yx"],
psf_z,
psf_yx,
alpha=config["alpha"], # impacts number of spots per cluster
beta=config["beta"], # impacts the number of detected clusters
)[0]
# separate spots from clusters
spots_post_clustering, foci = detection.detect_foci(
spots_post_decomposition,
config["voxel_size_z"],
config["voxel_size_yx"],
config["bf_radius"],
config["nb_min_spots"],
)
# extract cell level results
image_contrasted = stack.rescale(rna, channel_to_stretch=0)
image_contrasted = stack.maximum_projection(image_contrasted)
rna_mip = stack.maximum_projection(rna)
fov_results = stack.extract_cell(
cell_label=masks.astype(np.int64),
ndim=3,
rna_coord=spots_post_clustering,
others_coord={"foci": foci},
image=image_contrasted,
others_image={"smfish": rna_mip},
)
# save bigfish results
for i, cell_results in enumerate(fov_results):
output_path = pathlib.Path(config["output_dir"]).joinpath(
f"{image_name}_ch{image_channel + 1}_results_cell_{i}.npz")
stack.save_cell_extracted(cell_results, str(output_path))
# save reference spot for each image
# (Using undenoised image! not from denoised!)
reference_spot_undenoised = detection.build_reference_spot(
rna,
spots,
config["voxel_size_z"],
config["voxel_size_yx"],
psf_z,
psf_yx,
alpha=config["alpha"],
)
spot_output_path = pathlib.Path(config["output_refspot_dir"]).joinpath(
f"{image_name}_reference_spot_ch{image_channel + 1}")
stack.save_image(reference_spot_undenoised, str(spot_output_path),
"tif")
# Close the OMERO connection
conn.close()