def __init__(self,
fov_name,
fish_img,
dapi_img,
args,
nuc_mask=None,
cell_mask=None,
show=False):
self.voxel_size_z = args.voxel_size_z
self.voxel_size_yx = args.voxel_size_yx
self.nuc_mask = nuc_mask
self.cell_mask = cell_mask
if hasattr(args, 'manual_threshold'):
self.manual_threshold = args.manual_threshold
else:
self.manual_threshold = None
self.show = show
self.psf_z, self.psf_yx = calculate_psf(self.voxel_size_z,
self.voxel_size_yx, args.Ex,
args.Em, args.NA, args.RI,
args.microscope)
self.rna = stack.read_image(fish_img)
self.nuc = stack.read_image(dapi_img)
self.rna_mip = stack.maximum_projection(self.rna)
self.fov_name = fov_name
#would be good to determine vmax automatically from intensity of single RNAs
self.vmax = args.vmax
self.foci_radius = args.foci_radius
self.nb_in_foci = args.nb_in_foci
self.plotdir = os.path.join(args.outdir, 'plots')
self.datadir = os.path.join(args.outdir, 'results')
os.makedirs(self.plotdir, exist_ok=True)
os.makedirs(self.datadir, exist_ok=True)
def _get_input_dimension(recipe, input_folder):
""" Load an arbitrary image to get the original dimension of the files.
Parameters
----------
recipe : dict
Map the images according to their field of view, their round,
their channel and their spatial dimensions. Only contain the keys
'fov', 'r', 'c', 'z', 'ext' or 'opt'.
input_folder : str
Path of the folder containing the images.
Returns
-------
nb_dim : int
Number of dimensions of the original file.
"""
# get a valid path from the recipe
path = get_path_from_recipe(recipe, input_folder)
# load the image and return the number of dimensions
image = stack.read_image(path)
nb_dim = image.ndim
return nb_dim
def _build_stack_from_4d(recipe, input_folder, fov=0, nb_r=1):
"""Load and stack 4-d tensors.
Parameters
----------
recipe : dict
Map the images according to their field of view, their round,
their channel and their spatial dimensions. Only contain the keys
'fov', 'r', 'c', 'z', 'ext' or 'opt'.
input_folder : str
Path of the folder containing the images.
fov : int
Index of the fov to build.
nb_r : int
Number of round file to stack in order to get a 5-d tensor.
Returns
-------
tensor_5d : np.ndarray
Tensor with shape (round, channel, z, y, x).
"""
# load each file from a new round element and stack them
tensors_4d = []
for r in range(nb_r):
path = get_path_from_recipe(recipe, input_folder, fov=fov, r=r)
tensor_4d = stack.read_image(path)
tensors_4d.append(tensor_4d)
# stack 4-d tensors in 5-d
tensor_5d = np.stack(tensors_4d, axis=0)
return tensor_5d
def _build_stack_from_5d(recipe, input_folder, fov=0):
"""Load directly a 5-d tensor.
Parameters
----------
recipe : dict
Map the images according to their field of view, their round,
their channel and their spatial dimensions. Only contain the keys
'fov', 'r', 'c', 'z', 'ext' or 'opt'.
input_folder : str
Path of the folder containing the images.
fov : int
Index of the fov to build.
Returns
-------
tensor_5d : np.ndarray
Tensor with shape (round, channel, z, y, x).
"""
# the recipe can only contain one file with a 5-d tensor per fov
path = get_path_from_recipe(recipe, input_folder, fov=fov)
tensor_5d = stack.read_image(path)
return tensor_5d
def count_nuclei(mask_file, mode='whole', avg_area=None):
'''
Return the nuclei count of the image according to the following modes
whole = only count whole nuclei
all = count all nuclei
fractional = estimate fractional nuclei of ones touching the border
avg_area = average area (calculated elsewhere, e.g. from a whole dataset)
to use for calculating the fractional area
'''
nuc_label = stack.read_image(mask_file)
#separate into whole nuclei and fractional nuclei
whole_label = clear_border(nuc_label)
nuc_props = regionprops(nuc_label)
whole_props = regionprops(whole_label)
if mode == 'whole':
return len(whole_props)
elif mode == 'all':
return len(nuc_props)
elif mode == 'fractional':
whole_labels = [i.label for i in whole_props]
if not avg_area:
avg_area = np.mean([i.area for i in whole_props])
touching_border = [i for i in nuc_props if i.label not in whole_labels]
border_area = np.sum([i.area for i in touching_border])
extra_nuclei = border_area/avg_area
return len(whole_props) + extra_nuclei
def assign_foci_ts(self):
'''
Split foci into foci that overlap the nucleus (ts)
and foci which do not overlap.
Extract the results for the FOV.
'''
nuc_label = stack.read_image(self.nuc_mask)
if self.cell_mask is None:
cell_label = np.ones(nuc_label.shape, dtype='int64')
self.spots_no_ts, self.non_ts_foci, self.ts = stack.remove_transcription_site(
self.spots_and_foci, self.foci, nuc_label, ndim=3)
image_contrasted = stack.rescale(self.rna, channel_to_stretch=0)
image_contrasted = stack.maximum_projection(image_contrasted)
self.nuc_mip = stack.maximum_projection(self.nuc)
#Get results for field of view
self.fov_results = stack.extract_cell(cell_label=cell_label,
ndim=3,
nuc_label=nuc_label,
rna_coord=self.spots_no_ts,
others_coord={
"foci": self.non_ts_foci,
"transcription_site": self.ts
},
image=image_contrasted,
others_image={
"dapi": self.nuc_mip,
"smfish": self.rna_mip
},
remove_cropped_cell=False,
check_nuc_in_cell=False)
print("number of cells identified: {0}".format(len(self.fov_results)))
def _load_stack_no_recipe(paths, input_dimension=None):
"""Build a 5-d tensor from the same field of view (fov), without recipe.
Files with a path listed are stacked together, then empty dimensions are
added up to 5.
Parameters
----------
paths : List[str]
List of the file to stack.
input_dimension : str
Number of dimensions of the loaded files.
Returns
-------
tensor_5d : np.ndarray, np.uint
Tensor with shape (round, channel, z, y, x).
"""
# load an image and get the number of dimensions
if input_dimension is None:
testfile = stack.read_image(paths[0])
input_dimension = testfile.ndim
# get stacks
stacks = []
for path in paths:
s = stack.read_image(path)
stacks.append(s)
# we stack our files according to their initial dimension
if input_dimension == 2:
tensor_3d = np.stack(stacks, axis=0)
tensor_5d = tensor_3d[np.newaxis, np.newaxis, :, :, :]
elif input_dimension == 3:
tensor_4d = np.stack(stacks, axis=0)
tensor_5d = tensor_4d[np.newaxis, :, :, :, :]
elif input_dimension == 4:
tensor_5d = np.stack(stacks, axis=0)
elif input_dimension == 5 and len(stacks) == 1:
tensor_5d = stacks[0]
else:
raise ValueError("Files do not have the right number of dimensions: "
"{0}. The files we stack should have between 2 and "
"5 dimensions.".format(input_dimension))
return tensor_5d
def _build_stack_from_2d(recipe, input_folder, fov=0, nb_r=1, nb_c=1, nb_z=1):
"""Load and stack 2-d tensors.
Parameters
----------
recipe : dict
Map the images according to their field of view, their round,
their channel and their spatial dimensions. Only contain the keys
'fov', 'r', 'c', 'z', 'ext' or 'opt'.
input_folder : str
Path of the folder containing the images.
fov : int
Index of the fov to build.
nb_r : int
Number of round file to stack in order to get a 5-d tensor.
nb_c : int
Number of channel file to stack in order to get a 4-d tensor.
nb_z : int
Number of z file to stack in order to get a 3-d tensor.
Returns
-------
tensor_5d : np.ndarray
Tensor with shape (round, channel, z, y, x).
"""
# load and stack successively z, channel then round elements
tensors_4d = []
for r in range(nb_r):
# load and stack channel elements (3-d tensors)
tensors_3d = []
for c in range(nb_c):
# load and stack z elements (2-d tensors)
tensors_2d = []
for z in range(nb_z):
path = get_path_from_recipe(recipe,
input_folder,
fov=fov,
r=r,
c=c,
z=z)
tensor_2d = stack.read_image(path)
tensors_2d.append(tensor_2d)
# stack 2-d tensors in 3-d
tensor_3d = np.stack(tensors_2d, axis=0)
tensors_3d.append(tensor_3d)
# stack 3-d tensors in 4-d
tensor_4d = np.stack(tensors_3d, axis=0)
tensors_4d.append(tensor_4d)
# stack 4-d tensors in 5-d
tensor_5d = np.stack(tensors_4d, axis=0)
return tensor_5d
def __init__(self, fov_name, fish_img, dapi_img, args, nuc_mask=None, cell_mask=None):
self.voxel_size_z = args.voxel_size_z
self.voxel_size_xy = args.voxel_size_xy
self.nuc_mask = nuc_mask
self.cell_mask = cell_mask
self.psf_z, self.psf_xy = calculate_psf(self.voxel_size_z, self.voxel_size_xy,
args.Ex, args.Em, args.NA, args.RI,
args.microscope)
self.rna = stack.read_image(fish_img)
self.nuc = stack.read_image(dapi_img)
self.rna_mip = stack.maximum_projection(self.rna)
self.fov_name = fov_name
self.cluster_radius = args.cluster_radius
self.nb_in_cluster = args.nb_in_cluster
self.plotdir = os.path.join(args.outdir, 'plots')
self.datadir = os.path.join(args.outdir, 'results')
os.makedirs(self.plotdir, exist_ok = True)
os.makedirs(self.datadir, exist_ok = True)
def avg_area_whole_objects(infiles):
'''
Report average area of whole objects in mask files,
e.g. way to get average area across all nuclei in a dataset
'''
areas = []
for i in infiles:
label = stack.read_image(i)
whole_label = clear_border(label)
props = regionprops(whole_label)
areas.extend([i.area for i in props])
return np.mean(areas)
def summarize_experiments(indir, outname, spots_ext='_spots_and_foci.npy', foci_ext='_foci.npy',
nuc_ext='_dapi__mask__nuclei.png', nuc_count_mode='fractional',
res_subdir='results', nuc_subdir='segmentation-results'):
'''
Summarize results of all experiments in the dataset.
indir = path to main output folder
outname = path to output csv file, will be within the indir
spots_ext = extension of spots npy file (after foci assignment)
foci_ext = extension of foci file
nuc_ext = extension of the nuclear mask files
res_subdir = subdirectory for the npy files
nuc_subdir = subdirectory for the nuclear mask files
nuc_count_mode = how to count nuclei (see count_nuclei())
'''
#get average nuclear area across the dataset
search_path = f'{indir}/{nuc_subdir}/*{nuc_ext}'
mask_files = glob.glob(f'{indir}/{nuc_subdir}/*{nuc_ext}')
avg_nuc_area = avg_area_whole_objects(mask_files)
#regenerate the summary df using all nuclei as the nuclear masks
array_files = glob.glob(f'{indir}/{res_subdir}/*{spots_ext}')
res_dict = {}
for i in array_files:
expname = os.path.basename(i).split(spots_ext)[0]
res_dict[expname] = {}
foci_file = os.path.join(indir, res_subdir, f'{expname}{foci_ext}')
mask_file = os.path.join(indir, nuc_subdir, f'{expname}{nuc_ext}')
nuc_count = count_nuclei(mask_file, mode='fractional', avg_area=avg_nuc_area)
nuc_label = stack.read_image(mask_file)
rnas = np.load(i)
foci = np.load(foci_file)
ts, non_ts = stack.identify_objects_in_region(nuc_label, foci, 3)
nuc_rnas, cyt_rnas = stack.identify_objects_in_region(nuc_label, rnas, 3)
total = len(rnas)
ts = ts[:,3].sum()
nonts = total - ts
nuc = len(nuc_rnas)
cyt = len(cyt_rnas)
res_dict[expname]['total'] = total
res_dict[expname]['ts'] = ts
res_dict[expname]['non_ts'] = nonts
res_dict[expname]['nuclear'] = nuc
res_dict[expname]['cytoplasmic'] = cyt
res_dict[expname]['nuc_count'] = nuc_count
res_df = pd.DataFrame.from_dict(res_dict, orient='index')
res_df['ts/nuc'] = res_df['ts']/res_df['nuc_count']
res_df['non_ts/nuc'] = res_df['non_ts']/res_df['nuc_count']
res_df.to_csv(os.path.join(indir, f'{outname}_summary.csv'))
def test_image(shape, dtype, extension):
# build a temporary directory and save tensors inside
with tempfile.TemporaryDirectory() as tmp_dir:
test = np.zeros(shape, dtype=dtype)
path = os.path.join(tmp_dir, "test." + extension)
# error: boolean multidimensional image
if (extension in ["png", "jpg", "jpeg", "tif", "tiff"]
and len(test.shape) > 2 and test.dtype == bool):
with pytest.raises(ValueError):
stack.save_image(test, path)
# error: non-boolean multidimensional image with 'png', 'jpg' or 'jpeg'
elif (extension in ["png", "jpg", "jpeg"] and len(test.shape) > 2
and test.dtype != bool):
with pytest.raises(ValueError):
stack.save_image(test, path)
# error: boolean 2-d image with 'tig' and 'tiff'
elif (extension in ["tif", "tiff"] and len(test.shape) == 2
and test.dtype == bool):
with pytest.raises(ValueError):
stack.save_image(test, path)
# warning: 2-d image with 'png', 'jpg' or 'jpeg'
elif (extension in ["png", "jpg", "jpeg"] and len(test.shape) == 2):
with pytest.warns(UserWarning):
stack.save_image(test, path)
tensor = stack.read_image(path, sanity_check=True)
assert_array_equal(test, tensor)
# others valid images
else:
stack.save_image(test, path)
tensor = stack.read_image(path, sanity_check=True)
assert_array_equal(test, tensor)
assert test.dtype == tensor.dtype
# get sigma
sigma_z, sigma_yx = detection.get_sigma(resolution_z=300,
resolution_yx=103,
psf_z=350,
psf_yx=150)
sigma = (sigma_z, sigma_yx, sigma_yx)
nb_images = 0
for i, _ in enumerate(generator):
filename = filename_base + "_" + str(i)
print("\t", filename)
# LoG filter
path = os.path.join(log_filter_directory, filename + ".tiff")
cyt_filtered_log = stack.read_image(path)
# cyt maximum projection
path = os.path.join(projection_cyt_directory, filename + ".png")
cyt_mip = stack.read_image(path)
cyt_mip_contrast = stack.rescale(cyt_mip, channel_to_stretch=0)
# detect spot
mask_lm = detection.local_maximum_detection(cyt_filtered_log,
minimum_distance=2)
spots, radius, _ = detection.spots_thresholding(image=cyt_filtered_log,
sigma=sigma,
mask_lm=mask_lm,
threshold=threshold)
# save detected spots
nb_images = 0
for filename_png in projections:
if not re.match(pattern, filename_png):
print(
"\t '{0}' is not a valid file. Skipped.".format(filename_png))
continue
# filename
filename = str(filename_png.split(".")[0])
filename_base = str(filename.split("_")[:-1])
i = int(filename.split("_")[-1])
print("\t", filename)
# projection
path = os.path.join(projection_directory, filename_png)
nuc_projected = stack.read_image(path)
# mask
path = os.path.join(mask_directory, filename + ".tiff")
mask = stack.read_image(path)
# remove segmented nuclei
unsegmented_nuclei = segmentation.remove_segmented_nuc(
nuc_projected, mask)
path = os.path.join(removed_directory, filename_png)
stack.save_image(unsegmented_nuclei, path)
nb_images += 1
print("Done ({0} images)!".format(nb_images), "\n")
nb_images = 0
for i, _ in enumerate(generator):
filename = filename_base + "_" + str(i)
gene = experience["gene"]
author = experience["author"]
puro = experience["puro"]
drug = experience["other"]
paper = experience["paper"]
if image_to_dismiss(experience_directory, i):
print("\t", filename, "DISMISSED")
continue
print("\t", filename)
# masks
path = os.path.join(nuc_mask_directory, filename + ".png")
mask_nuc = stack.read_image(path)
path = os.path.join(cyt_mask_directory, filename + ".png")
mask_cyt = stack.read_image(path)
# cytoplasm maximum projection
path = os.path.join(cyt_projection_directory, filename + ".png")
cyt_mip = stack.read_image(path)
# spots and foci
path = os.path.join(detection_directory, filename + ".npz")
data = np.load(path)
spots_out_foci = data["spots_out_foci"]
spots_in_foci = data["spots_in_foci"]
foci = data["foci"]
# extract coordinates
nb_images = 0
for i, _ in enumerate(generator):
filename = filename_base + "_" + str(i)
print("\t", filename)
# spots
path = os.path.join(decomposition_directory, filename + ".npz")
data = np.load(path)
spots_out_cluster = data["spots_out_cluster"]
spots_in_cluster = data["spots_in_cluster"]
clusters = data["clusters"]
radius_spots = data["radius_spots"]
# cytoplasm maximum projection
path = os.path.join(cyt_projection_directory, filename + ".png")
cyt_mip = stack.read_image(path)
cyt_mip_contrast = stack.rescale(cyt_mip, channel_to_stretch=0)
# detect foci
spots = np.concatenate((spots_out_cluster, spots_in_cluster[:, :3]),
axis=0)
clustered_spots = detection.cluster_spots(spots=spots,
resolution_z=300,
resolution_yx=103,
radius=350,
nb_min_spots=5)
foci = detection.extract_foci(clustered_spots=clustered_spots)
# save foci
path = os.path.join(foci_directory, filename)
np.savez(path, clustered_spots=clustered_spots, foci=foci)
return_image=False)
nb_images = 0
for i, _ in enumerate(generator):
filename = filename_base + "_" + str(i)
print("\t", filename)
# spots
path = os.path.join(foci_directory, filename + ".npz")
data = np.load(path)
clustered_spots = data["clustered_spots"]
foci = data["foci"]
# nuclei masks
path = os.path.join(nuc_mask_directory, filename + ".png")
mask_nuc = stack.read_image(path)
nuc = mask_nuc > 0
# spots out of foci and inside foci
spots_out_foci = clustered_spots.copy()
spots_out_foci = spots_out_foci[spots_out_foci[:, 3] == -1, :]
spots_in_foci = clustered_spots.copy()
spots_in_foci = spots_in_foci[spots_in_foci[:, 3] != -1, :]
# remove foci inside nuclei
spots_in_foci_cleaned, foci_cleaned = stack.remove_transcription_site(
mask_nuc=nuc, spots_in_foci=spots_in_foci, foci=foci)
# save transcription site-free coordinates
path = os.path.join(transcription_site_directory, filename)
np.savez(path,
def _load_bad_input(recipe, input_directory, i_fov):
"""Load an image badly saved (MIP most of the times).
:param recipe: dict
Map the images according to their field of view, their round,
their channel and their spatial dimensions. Only contain the keys
'pattern', 'fov', 'r', 'c', 'z', 'ext' or 'opt'.
:param input_directory: str
Full path of the input directory.
:param i_fov: int
Index of the fov to generate.
:return:
image : np.ndarray, np.uint
Image with shape (r, c, z, y, x).
"""
recipe = stack.utils.fit_recipe(recipe)
if "CTNNB1_2019_EB" in input_directory and i_fov in [3]:
path = stack.utils.get_path_from_recipe(recipe,
input_directory,
fov=i_fov,
c=0)
nuc = stack.read_image(path)
path = stack.utils.get_path_from_recipe(recipe,
input_directory,
fov=i_fov,
c=1)
cyt = stack.read_image(path)
nb_z = cyt.shape[0]
nuc = nuc[12, :, :]
nuc = np.stack([nuc] * nb_z, axis=0)
image = np.stack([nuc, cyt], axis=0)
image = image[np.newaxis, ...]
elif "w7_bac_kif20b_Racha" in input_directory and i_fov in [6]:
path = stack.utils.get_path_from_recipe(recipe,
input_directory,
fov=i_fov,
c=0)
nuc = stack.read_image(path)
path = stack.utils.get_path_from_recipe(recipe,
input_directory,
fov=i_fov,
c=1)
cyt = stack.read_image(path)
nb_z = cyt.shape[0]
nuc = nuc[17, :, :]
nuc = np.stack([nuc] * nb_z, axis=0)
image = np.stack([nuc, cyt], axis=0)
image = image[np.newaxis, ...]
else:
path = stack.utils.get_path_from_recipe(recipe,
input_directory,
fov=i_fov,
c=0)
nuc = stack.read_image(path)
path = stack.utils.get_path_from_recipe(recipe,
input_directory,
fov=i_fov,
c=1)
cyt = stack.read_image(path)
nb_z = cyt.shape[0]
nuc = np.stack([nuc] * nb_z, axis=0)
image = np.stack([nuc, cyt], axis=0)
image = image[np.newaxis, ...]
stack.check_array(image, ndim=5, dtype=[np.uint8, np.uint16])
return image
# start analysis
experience = get_metadata_directory(experience_directory)
filename_base = generate_filename_base(experience)
generator = images_generator(base_directory,
experience_directory,
return_image=False)
nb_images = 0
for i, _ in enumerate(generator):
filename = filename_base + "_" + str(i)
print("\t", filename)
# cyt focus projection
path = os.path.join(projection_cyt_directory, filename + ".png")
cyt_projected = stack.read_image(path)
cyt_projected_contrast = stack.rescale(cyt_projected,
channel_to_stretch=0)
# nuclei labelled
path = os.path.join(mask_nuc_directory, filename + ".png")
nuc_labelled = stack.read_image(path)
# compute binary mask
mask = segmentation.build_cyt_binary_mask(cyt_projected,
threshold=threshold)
mask[nuc_labelled > 0] = True
# compute relief
relief = segmentation.build_cyt_relief(cyt_projected,
nuc_labelled=nuc_labelled,
nb_images = 0
for filename_tiff in masks_1:
if not re.match(pattern, filename_tiff):
print(
"\t '{0}' is not a valid file. Skipped.".format(filename_tiff))
continue
# filename
filename = str(filename_tiff.split(".")[0])
filename_base = str(filename.split("_")[:-1])
i = int(filename.split("_")[-1])
print("\t", filename)
# nuclei projection
path = os.path.join(projection_directory, filename + ".png")
nuc_focus = stack.read_image(path)
# mask 1
path = os.path.join(mask_directory_1, filename_tiff)
mask_1 = stack.read_image(path)
# mask 2
path = os.path.join(mask_directory_2, filename_tiff)
mask_2 = stack.read_image(path)
# mask
mask = segmentation.merge_labels(mask_1, mask_2)
# postprocess mask and mask_1
mask_1 = segmentation.dilate_erode_labels(mask_1)
mask = segmentation.dilate_erode_labels(mask)