def __init__(self, fov_name, fish_img, dapi_img, args, nuc_mask=None, cell_mask=None, show=False): self.voxel_size_z = args.voxel_size_z self.voxel_size_yx = args.voxel_size_yx self.nuc_mask = nuc_mask self.cell_mask = cell_mask if hasattr(args, 'manual_threshold'): self.manual_threshold = args.manual_threshold else: self.manual_threshold = None self.show = show self.psf_z, self.psf_yx = calculate_psf(self.voxel_size_z, self.voxel_size_yx, args.Ex, args.Em, args.NA, args.RI, args.microscope) self.rna = stack.read_image(fish_img) self.nuc = stack.read_image(dapi_img) self.rna_mip = stack.maximum_projection(self.rna) self.fov_name = fov_name #would be good to determine vmax automatically from intensity of single RNAs self.vmax = args.vmax self.foci_radius = args.foci_radius self.nb_in_foci = args.nb_in_foci self.plotdir = os.path.join(args.outdir, 'plots') self.datadir = os.path.join(args.outdir, 'results') os.makedirs(self.plotdir, exist_ok=True) os.makedirs(self.datadir, exist_ok=True)
def _get_input_dimension(recipe, input_folder): """ Load an arbitrary image to get the original dimension of the files. Parameters ---------- recipe : dict Map the images according to their field of view, their round, their channel and their spatial dimensions. Only contain the keys 'fov', 'r', 'c', 'z', 'ext' or 'opt'. input_folder : str Path of the folder containing the images. Returns ------- nb_dim : int Number of dimensions of the original file. """ # get a valid path from the recipe path = get_path_from_recipe(recipe, input_folder) # load the image and return the number of dimensions image = stack.read_image(path) nb_dim = image.ndim return nb_dim
def _build_stack_from_4d(recipe, input_folder, fov=0, nb_r=1): """Load and stack 4-d tensors. Parameters ---------- recipe : dict Map the images according to their field of view, their round, their channel and their spatial dimensions. Only contain the keys 'fov', 'r', 'c', 'z', 'ext' or 'opt'. input_folder : str Path of the folder containing the images. fov : int Index of the fov to build. nb_r : int Number of round file to stack in order to get a 5-d tensor. Returns ------- tensor_5d : np.ndarray Tensor with shape (round, channel, z, y, x). """ # load each file from a new round element and stack them tensors_4d = [] for r in range(nb_r): path = get_path_from_recipe(recipe, input_folder, fov=fov, r=r) tensor_4d = stack.read_image(path) tensors_4d.append(tensor_4d) # stack 4-d tensors in 5-d tensor_5d = np.stack(tensors_4d, axis=0) return tensor_5d
def _build_stack_from_5d(recipe, input_folder, fov=0): """Load directly a 5-d tensor. Parameters ---------- recipe : dict Map the images according to their field of view, their round, their channel and their spatial dimensions. Only contain the keys 'fov', 'r', 'c', 'z', 'ext' or 'opt'. input_folder : str Path of the folder containing the images. fov : int Index of the fov to build. Returns ------- tensor_5d : np.ndarray Tensor with shape (round, channel, z, y, x). """ # the recipe can only contain one file with a 5-d tensor per fov path = get_path_from_recipe(recipe, input_folder, fov=fov) tensor_5d = stack.read_image(path) return tensor_5d
def count_nuclei(mask_file, mode='whole', avg_area=None): ''' Return the nuclei count of the image according to the following modes whole = only count whole nuclei all = count all nuclei fractional = estimate fractional nuclei of ones touching the border avg_area = average area (calculated elsewhere, e.g. from a whole dataset) to use for calculating the fractional area ''' nuc_label = stack.read_image(mask_file) #separate into whole nuclei and fractional nuclei whole_label = clear_border(nuc_label) nuc_props = regionprops(nuc_label) whole_props = regionprops(whole_label) if mode == 'whole': return len(whole_props) elif mode == 'all': return len(nuc_props) elif mode == 'fractional': whole_labels = [i.label for i in whole_props] if not avg_area: avg_area = np.mean([i.area for i in whole_props]) touching_border = [i for i in nuc_props if i.label not in whole_labels] border_area = np.sum([i.area for i in touching_border]) extra_nuclei = border_area/avg_area return len(whole_props) + extra_nuclei
def assign_foci_ts(self): ''' Split foci into foci that overlap the nucleus (ts) and foci which do not overlap. Extract the results for the FOV. ''' nuc_label = stack.read_image(self.nuc_mask) if self.cell_mask is None: cell_label = np.ones(nuc_label.shape, dtype='int64') self.spots_no_ts, self.non_ts_foci, self.ts = stack.remove_transcription_site( self.spots_and_foci, self.foci, nuc_label, ndim=3) image_contrasted = stack.rescale(self.rna, channel_to_stretch=0) image_contrasted = stack.maximum_projection(image_contrasted) self.nuc_mip = stack.maximum_projection(self.nuc) #Get results for field of view self.fov_results = stack.extract_cell(cell_label=cell_label, ndim=3, nuc_label=nuc_label, rna_coord=self.spots_no_ts, others_coord={ "foci": self.non_ts_foci, "transcription_site": self.ts }, image=image_contrasted, others_image={ "dapi": self.nuc_mip, "smfish": self.rna_mip }, remove_cropped_cell=False, check_nuc_in_cell=False) print("number of cells identified: {0}".format(len(self.fov_results)))
def _load_stack_no_recipe(paths, input_dimension=None): """Build a 5-d tensor from the same field of view (fov), without recipe. Files with a path listed are stacked together, then empty dimensions are added up to 5. Parameters ---------- paths : List[str] List of the file to stack. input_dimension : str Number of dimensions of the loaded files. Returns ------- tensor_5d : np.ndarray, np.uint Tensor with shape (round, channel, z, y, x). """ # load an image and get the number of dimensions if input_dimension is None: testfile = stack.read_image(paths[0]) input_dimension = testfile.ndim # get stacks stacks = [] for path in paths: s = stack.read_image(path) stacks.append(s) # we stack our files according to their initial dimension if input_dimension == 2: tensor_3d = np.stack(stacks, axis=0) tensor_5d = tensor_3d[np.newaxis, np.newaxis, :, :, :] elif input_dimension == 3: tensor_4d = np.stack(stacks, axis=0) tensor_5d = tensor_4d[np.newaxis, :, :, :, :] elif input_dimension == 4: tensor_5d = np.stack(stacks, axis=0) elif input_dimension == 5 and len(stacks) == 1: tensor_5d = stacks[0] else: raise ValueError("Files do not have the right number of dimensions: " "{0}. The files we stack should have between 2 and " "5 dimensions.".format(input_dimension)) return tensor_5d
def _build_stack_from_2d(recipe, input_folder, fov=0, nb_r=1, nb_c=1, nb_z=1): """Load and stack 2-d tensors. Parameters ---------- recipe : dict Map the images according to their field of view, their round, their channel and their spatial dimensions. Only contain the keys 'fov', 'r', 'c', 'z', 'ext' or 'opt'. input_folder : str Path of the folder containing the images. fov : int Index of the fov to build. nb_r : int Number of round file to stack in order to get a 5-d tensor. nb_c : int Number of channel file to stack in order to get a 4-d tensor. nb_z : int Number of z file to stack in order to get a 3-d tensor. Returns ------- tensor_5d : np.ndarray Tensor with shape (round, channel, z, y, x). """ # load and stack successively z, channel then round elements tensors_4d = [] for r in range(nb_r): # load and stack channel elements (3-d tensors) tensors_3d = [] for c in range(nb_c): # load and stack z elements (2-d tensors) tensors_2d = [] for z in range(nb_z): path = get_path_from_recipe(recipe, input_folder, fov=fov, r=r, c=c, z=z) tensor_2d = stack.read_image(path) tensors_2d.append(tensor_2d) # stack 2-d tensors in 3-d tensor_3d = np.stack(tensors_2d, axis=0) tensors_3d.append(tensor_3d) # stack 3-d tensors in 4-d tensor_4d = np.stack(tensors_3d, axis=0) tensors_4d.append(tensor_4d) # stack 4-d tensors in 5-d tensor_5d = np.stack(tensors_4d, axis=0) return tensor_5d
def __init__(self, fov_name, fish_img, dapi_img, args, nuc_mask=None, cell_mask=None): self.voxel_size_z = args.voxel_size_z self.voxel_size_xy = args.voxel_size_xy self.nuc_mask = nuc_mask self.cell_mask = cell_mask self.psf_z, self.psf_xy = calculate_psf(self.voxel_size_z, self.voxel_size_xy, args.Ex, args.Em, args.NA, args.RI, args.microscope) self.rna = stack.read_image(fish_img) self.nuc = stack.read_image(dapi_img) self.rna_mip = stack.maximum_projection(self.rna) self.fov_name = fov_name self.cluster_radius = args.cluster_radius self.nb_in_cluster = args.nb_in_cluster self.plotdir = os.path.join(args.outdir, 'plots') self.datadir = os.path.join(args.outdir, 'results') os.makedirs(self.plotdir, exist_ok = True) os.makedirs(self.datadir, exist_ok = True)
def avg_area_whole_objects(infiles): ''' Report average area of whole objects in mask files, e.g. way to get average area across all nuclei in a dataset ''' areas = [] for i in infiles: label = stack.read_image(i) whole_label = clear_border(label) props = regionprops(whole_label) areas.extend([i.area for i in props]) return np.mean(areas)
def summarize_experiments(indir, outname, spots_ext='_spots_and_foci.npy', foci_ext='_foci.npy', nuc_ext='_dapi__mask__nuclei.png', nuc_count_mode='fractional', res_subdir='results', nuc_subdir='segmentation-results'): ''' Summarize results of all experiments in the dataset. indir = path to main output folder outname = path to output csv file, will be within the indir spots_ext = extension of spots npy file (after foci assignment) foci_ext = extension of foci file nuc_ext = extension of the nuclear mask files res_subdir = subdirectory for the npy files nuc_subdir = subdirectory for the nuclear mask files nuc_count_mode = how to count nuclei (see count_nuclei()) ''' #get average nuclear area across the dataset search_path = f'{indir}/{nuc_subdir}/*{nuc_ext}' mask_files = glob.glob(f'{indir}/{nuc_subdir}/*{nuc_ext}') avg_nuc_area = avg_area_whole_objects(mask_files) #regenerate the summary df using all nuclei as the nuclear masks array_files = glob.glob(f'{indir}/{res_subdir}/*{spots_ext}') res_dict = {} for i in array_files: expname = os.path.basename(i).split(spots_ext)[0] res_dict[expname] = {} foci_file = os.path.join(indir, res_subdir, f'{expname}{foci_ext}') mask_file = os.path.join(indir, nuc_subdir, f'{expname}{nuc_ext}') nuc_count = count_nuclei(mask_file, mode='fractional', avg_area=avg_nuc_area) nuc_label = stack.read_image(mask_file) rnas = np.load(i) foci = np.load(foci_file) ts, non_ts = stack.identify_objects_in_region(nuc_label, foci, 3) nuc_rnas, cyt_rnas = stack.identify_objects_in_region(nuc_label, rnas, 3) total = len(rnas) ts = ts[:,3].sum() nonts = total - ts nuc = len(nuc_rnas) cyt = len(cyt_rnas) res_dict[expname]['total'] = total res_dict[expname]['ts'] = ts res_dict[expname]['non_ts'] = nonts res_dict[expname]['nuclear'] = nuc res_dict[expname]['cytoplasmic'] = cyt res_dict[expname]['nuc_count'] = nuc_count res_df = pd.DataFrame.from_dict(res_dict, orient='index') res_df['ts/nuc'] = res_df['ts']/res_df['nuc_count'] res_df['non_ts/nuc'] = res_df['non_ts']/res_df['nuc_count'] res_df.to_csv(os.path.join(indir, f'{outname}_summary.csv'))
def test_image(shape, dtype, extension): # build a temporary directory and save tensors inside with tempfile.TemporaryDirectory() as tmp_dir: test = np.zeros(shape, dtype=dtype) path = os.path.join(tmp_dir, "test." + extension) # error: boolean multidimensional image if (extension in ["png", "jpg", "jpeg", "tif", "tiff"] and len(test.shape) > 2 and test.dtype == bool): with pytest.raises(ValueError): stack.save_image(test, path) # error: non-boolean multidimensional image with 'png', 'jpg' or 'jpeg' elif (extension in ["png", "jpg", "jpeg"] and len(test.shape) > 2 and test.dtype != bool): with pytest.raises(ValueError): stack.save_image(test, path) # error: boolean 2-d image with 'tig' and 'tiff' elif (extension in ["tif", "tiff"] and len(test.shape) == 2 and test.dtype == bool): with pytest.raises(ValueError): stack.save_image(test, path) # warning: 2-d image with 'png', 'jpg' or 'jpeg' elif (extension in ["png", "jpg", "jpeg"] and len(test.shape) == 2): with pytest.warns(UserWarning): stack.save_image(test, path) tensor = stack.read_image(path, sanity_check=True) assert_array_equal(test, tensor) # others valid images else: stack.save_image(test, path) tensor = stack.read_image(path, sanity_check=True) assert_array_equal(test, tensor) assert test.dtype == tensor.dtype
# get sigma sigma_z, sigma_yx = detection.get_sigma(resolution_z=300, resolution_yx=103, psf_z=350, psf_yx=150) sigma = (sigma_z, sigma_yx, sigma_yx) nb_images = 0 for i, _ in enumerate(generator): filename = filename_base + "_" + str(i) print("\t", filename) # LoG filter path = os.path.join(log_filter_directory, filename + ".tiff") cyt_filtered_log = stack.read_image(path) # cyt maximum projection path = os.path.join(projection_cyt_directory, filename + ".png") cyt_mip = stack.read_image(path) cyt_mip_contrast = stack.rescale(cyt_mip, channel_to_stretch=0) # detect spot mask_lm = detection.local_maximum_detection(cyt_filtered_log, minimum_distance=2) spots, radius, _ = detection.spots_thresholding(image=cyt_filtered_log, sigma=sigma, mask_lm=mask_lm, threshold=threshold) # save detected spots
nb_images = 0 for filename_png in projections: if not re.match(pattern, filename_png): print( "\t '{0}' is not a valid file. Skipped.".format(filename_png)) continue # filename filename = str(filename_png.split(".")[0]) filename_base = str(filename.split("_")[:-1]) i = int(filename.split("_")[-1]) print("\t", filename) # projection path = os.path.join(projection_directory, filename_png) nuc_projected = stack.read_image(path) # mask path = os.path.join(mask_directory, filename + ".tiff") mask = stack.read_image(path) # remove segmented nuclei unsegmented_nuclei = segmentation.remove_segmented_nuc( nuc_projected, mask) path = os.path.join(removed_directory, filename_png) stack.save_image(unsegmented_nuclei, path) nb_images += 1 print("Done ({0} images)!".format(nb_images), "\n")
nb_images = 0 for i, _ in enumerate(generator): filename = filename_base + "_" + str(i) gene = experience["gene"] author = experience["author"] puro = experience["puro"] drug = experience["other"] paper = experience["paper"] if image_to_dismiss(experience_directory, i): print("\t", filename, "DISMISSED") continue print("\t", filename) # masks path = os.path.join(nuc_mask_directory, filename + ".png") mask_nuc = stack.read_image(path) path = os.path.join(cyt_mask_directory, filename + ".png") mask_cyt = stack.read_image(path) # cytoplasm maximum projection path = os.path.join(cyt_projection_directory, filename + ".png") cyt_mip = stack.read_image(path) # spots and foci path = os.path.join(detection_directory, filename + ".npz") data = np.load(path) spots_out_foci = data["spots_out_foci"] spots_in_foci = data["spots_in_foci"] foci = data["foci"] # extract coordinates
nb_images = 0 for i, _ in enumerate(generator): filename = filename_base + "_" + str(i) print("\t", filename) # spots path = os.path.join(decomposition_directory, filename + ".npz") data = np.load(path) spots_out_cluster = data["spots_out_cluster"] spots_in_cluster = data["spots_in_cluster"] clusters = data["clusters"] radius_spots = data["radius_spots"] # cytoplasm maximum projection path = os.path.join(cyt_projection_directory, filename + ".png") cyt_mip = stack.read_image(path) cyt_mip_contrast = stack.rescale(cyt_mip, channel_to_stretch=0) # detect foci spots = np.concatenate((spots_out_cluster, spots_in_cluster[:, :3]), axis=0) clustered_spots = detection.cluster_spots(spots=spots, resolution_z=300, resolution_yx=103, radius=350, nb_min_spots=5) foci = detection.extract_foci(clustered_spots=clustered_spots) # save foci path = os.path.join(foci_directory, filename) np.savez(path, clustered_spots=clustered_spots, foci=foci)
return_image=False) nb_images = 0 for i, _ in enumerate(generator): filename = filename_base + "_" + str(i) print("\t", filename) # spots path = os.path.join(foci_directory, filename + ".npz") data = np.load(path) clustered_spots = data["clustered_spots"] foci = data["foci"] # nuclei masks path = os.path.join(nuc_mask_directory, filename + ".png") mask_nuc = stack.read_image(path) nuc = mask_nuc > 0 # spots out of foci and inside foci spots_out_foci = clustered_spots.copy() spots_out_foci = spots_out_foci[spots_out_foci[:, 3] == -1, :] spots_in_foci = clustered_spots.copy() spots_in_foci = spots_in_foci[spots_in_foci[:, 3] != -1, :] # remove foci inside nuclei spots_in_foci_cleaned, foci_cleaned = stack.remove_transcription_site( mask_nuc=nuc, spots_in_foci=spots_in_foci, foci=foci) # save transcription site-free coordinates path = os.path.join(transcription_site_directory, filename) np.savez(path,
def _load_bad_input(recipe, input_directory, i_fov): """Load an image badly saved (MIP most of the times). :param recipe: dict Map the images according to their field of view, their round, their channel and their spatial dimensions. Only contain the keys 'pattern', 'fov', 'r', 'c', 'z', 'ext' or 'opt'. :param input_directory: str Full path of the input directory. :param i_fov: int Index of the fov to generate. :return: image : np.ndarray, np.uint Image with shape (r, c, z, y, x). """ recipe = stack.utils.fit_recipe(recipe) if "CTNNB1_2019_EB" in input_directory and i_fov in [3]: path = stack.utils.get_path_from_recipe(recipe, input_directory, fov=i_fov, c=0) nuc = stack.read_image(path) path = stack.utils.get_path_from_recipe(recipe, input_directory, fov=i_fov, c=1) cyt = stack.read_image(path) nb_z = cyt.shape[0] nuc = nuc[12, :, :] nuc = np.stack([nuc] * nb_z, axis=0) image = np.stack([nuc, cyt], axis=0) image = image[np.newaxis, ...] elif "w7_bac_kif20b_Racha" in input_directory and i_fov in [6]: path = stack.utils.get_path_from_recipe(recipe, input_directory, fov=i_fov, c=0) nuc = stack.read_image(path) path = stack.utils.get_path_from_recipe(recipe, input_directory, fov=i_fov, c=1) cyt = stack.read_image(path) nb_z = cyt.shape[0] nuc = nuc[17, :, :] nuc = np.stack([nuc] * nb_z, axis=0) image = np.stack([nuc, cyt], axis=0) image = image[np.newaxis, ...] else: path = stack.utils.get_path_from_recipe(recipe, input_directory, fov=i_fov, c=0) nuc = stack.read_image(path) path = stack.utils.get_path_from_recipe(recipe, input_directory, fov=i_fov, c=1) cyt = stack.read_image(path) nb_z = cyt.shape[0] nuc = np.stack([nuc] * nb_z, axis=0) image = np.stack([nuc, cyt], axis=0) image = image[np.newaxis, ...] stack.check_array(image, ndim=5, dtype=[np.uint8, np.uint16]) return image
# start analysis experience = get_metadata_directory(experience_directory) filename_base = generate_filename_base(experience) generator = images_generator(base_directory, experience_directory, return_image=False) nb_images = 0 for i, _ in enumerate(generator): filename = filename_base + "_" + str(i) print("\t", filename) # cyt focus projection path = os.path.join(projection_cyt_directory, filename + ".png") cyt_projected = stack.read_image(path) cyt_projected_contrast = stack.rescale(cyt_projected, channel_to_stretch=0) # nuclei labelled path = os.path.join(mask_nuc_directory, filename + ".png") nuc_labelled = stack.read_image(path) # compute binary mask mask = segmentation.build_cyt_binary_mask(cyt_projected, threshold=threshold) mask[nuc_labelled > 0] = True # compute relief relief = segmentation.build_cyt_relief(cyt_projected, nuc_labelled=nuc_labelled,
nb_images = 0 for filename_tiff in masks_1: if not re.match(pattern, filename_tiff): print( "\t '{0}' is not a valid file. Skipped.".format(filename_tiff)) continue # filename filename = str(filename_tiff.split(".")[0]) filename_base = str(filename.split("_")[:-1]) i = int(filename.split("_")[-1]) print("\t", filename) # nuclei projection path = os.path.join(projection_directory, filename + ".png") nuc_focus = stack.read_image(path) # mask 1 path = os.path.join(mask_directory_1, filename_tiff) mask_1 = stack.read_image(path) # mask 2 path = os.path.join(mask_directory_2, filename_tiff) mask_2 = stack.read_image(path) # mask mask = segmentation.merge_labels(mask_1, mask_2) # postprocess mask and mask_1 mask_1 = segmentation.dilate_erode_labels(mask_1) mask = segmentation.dilate_erode_labels(mask)