def apply_classifier(classifier, raw_src, cnn_src, collect_cnn = False, size = (3,12,12), pad=False, cores=10, numZSlicesPerSplit=50, overlapping_planes = 15, verbose=True, save=True, maxip=0): ''' classifier = pretrained random forest or path to pretrained random forest raw_src = folder of tiffs of raw input data cnn_src = folder of tiffs from cnn output size pad = if True, pad the edges of objects determined. False: remove edge cases, usually better since they will be few in number cores = number of cores for parallelization, larger the number the less memory efficient numZSlicesPerSplit: chunk of zplanes to process at once. Adjust this and cores based on memory constraints. overlapping_planes: number of planes on each side to overlap by, this should be a comfortable amount larger than the maximum z distances of a single object save (optional): #optional save to prevent rerun of jobs collect_cnn = optional to include cnn data for random forest input Returns ---------------- a dictionary consisting of k=centers, v=[corresponding pixel indices determine by CNN, maximum intensity, list of maximum radius/plane] ''' #handle inputs threshold = 1 zyx_search_range = (2,10,10) zdim = len(listdirfull(cnn_src, keyword='.tif')) #optional save to prevent rerun of jobs if save: save = cnn_src+'_apply_classifier_tmp' makedir(save) #run if verbose: sys.stdout.write('\n Thesholding, determining connected pixels, identifying center of masses, applying classifier\n\n'); sys.stdout.flush(); st = time.time() rng = range(0, zdim, numZSlicesPerSplit); jobs = len(rng); iterlst=[(cnn_src, raw_src, collect_cnn, z, zdim, numZSlicesPerSplit, overlapping_planes, threshold, classifier, size, zyx_search_range, pad, job, jobs, verbose, save, maxip) for job, z in enumerate(rng)] #par vs not par if cores > 1: p = mp.Pool(cores) center_pixels_intensity_radius_lst = p.starmap(apply_classifier_helper, iterlst) p.terminate() else: center_pixels_intensity_radius_lst = [] for i in iterlst: center_pixels_intensity_radius_lst.append(apply_classifier_helper(i)) #optional reload: if save: center_pixels_intensity_radius_lst = [load_dictionary(xx) for xx in listdirfull(save)] shutil.rmtree(save) #unpack if verbose: sys.stdout.write('\n...finished, formatting dictionary...'); sys.stdout.flush() center_pixels_intensity_radius_dct = {}; [center_pixels_intensity_radius_dct.update(xx) for xx in center_pixels_intensity_radius_lst] if 'None' in center_pixels_intensity_radius_dct: del center_pixels_intensity_radius_dct['None'] if verbose: print ('Total time {} minutes'.format(round((time.time() - st) / 60))) if verbose: print('{} centers found.'.format(len(center_pixels_intensity_radius_dct))) return center_pixels_intensity_radius_dct
def generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=False): '''Function to take a csv file and generate an input to transformix Inputs ---------------- dataframe = preloaded pandas dataframe dst = destination to save files transformfiles = list of all elastix transform files used, and in order of the original transform**** lightsheet_parameter_file = .p file generated from lightsheet package ''' #set up locations transformed_dst = os.path.join(dst, 'transformed_points') makedir(transformed_dst) #make zyx numpy arry zyx = dataframe[['z', 'y', 'x']].values #adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample kwargs = load_dictionary(lightsheet_parameter_dictionary) vol = [xx for xx in kwargs['volumes'] if xx.ch_type == 'cellch'][0] fullsizedimensions = get_fullsizedims_from_kwargs( kwargs ) #don't get from kwargs['volumes'][0].fullsizedimensions it's bad! use this instead zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample zyx = points_resample( zyx, original_dims=fix_dimension_orientation(fullsizedimensions, **kwargs), resample_dims=tifffile.imread(vol.resampled_for_elastix_vol).shape, verbose=verbose)[:, :3] #make into transformix-friendly text file pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst) #copy over elastix files transformfiles = modify_transform_files(transformfiles, transformed_dst) change_transform_parameter_initial_transform(transformfiles[0], 'NoInitialTransform') #run transformix on points points_file = point_transformix(pretransform_text_file, transformfiles[-1], transformed_dst) #convert registered points into structure counts converted_points = unpack_pnts(points_file, transformed_dst) return converted_points
def original_directory_structure(src, transfertype="move"): """ Move files back src = "/home/wanglab/wang/pisano/ymaze/cfos_4x/20171129_ymaze_23/full_sizedatafld/20171129_ymaze_23_488_050na_z7d5um_50msec_10povlap_ch00/terastitcher_dct.p" """ for k, v in pth_update(load_dictionary(src)["terastitcher_dct"]).items(): try: if transfertype == "copy": shutil.copy(pth_update(v), pth_update(k)) if transfertype == "move": shutil.move(pth_update(v), pth_update(k)) except: print("?") return
def overlay_cells(src, center_intensity_radius, dilation_radius=5, load_range=False): ''' src = fullsizedata folder center_intensity_radius = path to center_intensity_radius.p file src = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20161214_db_bl6_crii_l_53hr/full_sizedatafld/20161214_db_bl6_crii_l_53hr_647_010na_z7d5um_75msec_5POVLP_ch00' center_intensity_radius = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20161214_db_bl6_crii_l_53hr/cells/20161214_db_bl6_crii_l_53hr_647_010na_z7d5um_75msec_5POVLP_ch00_centers_intensity_radius.p' load_range = (400,450) dilation_radius = 5 src = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20170115_tp_bl6_lob6a_1000r_02/full_sizedatafld/20170115_tp_bl6_lob6a_1000r_647_010na_z7d5um_125msec_10povlp_ch00' center_intensity_radius = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20170115_tp_bl6_lob6a_1000r_02/cells/20170115_tp_bl6_lob6a_1000r_647_010na_z7d5um_125msec_10povlp_ch00_centers_intensity_radius.p' src = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20170115_tp_bl6_lob6a_1000r_02/cells/test' center_intensity_radius = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20170115_tp_bl6_lob6a_1000r_02/cells/test_out.p' load_range = (0,100) ''' #load centers center_intensity_radius = load_dictionary(center_intensity_radius) if type( center_intensity_radius) == str else center_intensity_radius centers = sorted(center_intensity_radius.keys(), key=lambda x: x[0]) #adjust and compensate for load range if load_range: centers = [ tuple((c[0] - load_range[0], c[1], c[2])) for c in centers if c[0] in range(load_range[0], load_range[1]) ] #load vol = load_tiff_folder(src, threshold=0, load_range=load_range) vol0 = np.zeros_like(vol).astype('bool') for c in centers: vol0[c[0], c[1], c[2]] = True vol0 = dilate_with_element(vol0 * 1, ball(dilation_radius)).astype('uint8') # return vol, vol0
def overlay_qc(args): #unpacking this way for multiprocessing fld, folder_suffix, output_folder, verbose, doubletransform, make_volumes = args try: #get 3dunet cell dataframe csv file input_csv = listdirfull(os.path.join(fld, folder_suffix), ".csv") assert len(input_csv) == 1, "multiple csv files" dataframe = pd.read_csv(input_csv[0]) #location to save out dst = os.path.join(output_folder, os.path.basename(fld)); makedir(dst) #EXAMPLE USING LIGHTSHEET - assumes marking centers in the "raw" full sized cell channel. This will transform those #centers into "atlas" space (in this case the moving image) #in this case the "inverse transform has the atlas as the moving image in the first step, #and the autofluorescence channel as the moving image in the second step #NOTE - it seems that the registration of cell to auto is failing on occasion....thus get new files... ################################ cell_inverse_folder = listdirfull(os.path.join(fld, "elastix_inverse_transform"), "cellch")[0] a2r = listall(cell_inverse_folder, "atlas2reg_TransformParameters"); a2r.sort() r2s = listall(cell_inverse_folder, "reg2sig_TransformParameters"); r2s.sort() #possibly remove #IMPORTANT. the idea is to apply cfos->auto->atlas transformfiles = r2s + a2r if doubletransform else a2r #might get rid of r2s lightsheet_parameter_dictionary = os.path.join(fld, "param_dict.p") converted_points = generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=verbose) #load and convert to single voxel loc zyx = np.asarray([str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in np.nan_to_num(np.load(converted_points))]) from collections import Counter zyx_cnt = Counter(zyx) #check... if make_volumes: #manually call transformix kwargs = load_dictionary(lightsheet_parameter_dictionary) vol = [xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0].resampled_for_elastix_vol transformed_vol = os.path.join(dst, "transformed_volume"); makedir(transformed_vol) if not doubletransform: transformfiles = [os.path.join(fld, "elastix/TransformParameters.0.txt"), os.path.join(fld, "elastix/TransformParameters.1.txt")] transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files transformix_command_line_call(vol, transformed_vol, transformfiles[-1]) else: v=[xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0] #sig to reg tps = [listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.0")[0], listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.1")[0]] #reg to atlas transformfiles = tps+[os.path.join(fld, "elastix/TransformParameters.0.txt"), os.path.join(fld, "elastix/TransformParameters.1.txt")] transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files transformix_command_line_call(vol, transformed_vol, transformfiles[-1]) #cell_registered channel cell_reg = tifffile.imread(os.path.join(transformed_vol, "result.tif")) tifffile.imsave(os.path.join(transformed_vol, "result.tif"), cell_reg, compress=1) cell_cnn = np.zeros_like(cell_reg) tarr = []; badlist=[] for zyx,v in zyx_cnt.items(): z,y,x = [int(xx) for xx in zyx.replace("(","",).replace(")","").split(",")] tarr.append([z,y,x]) try: cell_cnn[z,y,x] = v*100 except: badlist.append([z,y,x]) #apply x y dilation r = 2 selem = ball(r)[int(r/2)] cell_cnn = cell_cnn.astype("uint8") cell_cnn = np.asarray([cv2.dilate(cell_cnn[i], selem, iterations = 1) for i in range(cell_cnn.shape[0])]) tarr=np.asarray(tarr) if len(badlist)>0: print("{} errors in mapping with cell_cnn shape {}, each max dim {}, \npossibly due to a registration overshoot \ or not using double transform\n\n{}".format(len(badlist), cell_cnn.shape, np.max(tarr,0), badlist)) merged = np.stack([cell_cnn, cell_reg, np.zeros_like(cell_reg)], -1) tifffile.imsave(os.path.join(transformed_vol, "merged.tif"), merged)#, compress=1) #out = np.concatenate([cell_cnn, cell_reg, ], 0) #####check at the resampled for elastix phase before transform...this mapping looks good... if make_volumes: #make zyx numpy arry zyx = dataframe[["z","y","x"]].values kwargs = load_dictionary(lightsheet_parameter_dictionary) vol = [xx for xx in kwargs["volumes"] if xx.ch_type =="cellch"][0] fullsizedimensions = get_fullsizedims_from_kwargs(kwargs) #don"t get from kwargs["volumes"][0].fullsizedimensions it"s bad! use this instead zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs), resample_dims = tifffile.imread(vol.resampled_for_elastix_vol).shape, verbose = verbose)[:, :3] #cell channel cell_ch = tifffile.imread(vol.resampled_for_elastix_vol) cell_cnn = np.zeros_like(cell_ch) tarr = []; badlist=[] for _zyx in zyx: z,y,x = [int(xx) for xx in _zyx] tarr.append([z,y,x]) try: cell_cnn[z,y,x] = 100 except: badlist.append([z,y,x]) tarr=np.asarray(tarr) merged = np.stack([cell_cnn, cell_ch, np.zeros_like(cell_ch)], -1) tifffile.imsave(os.path.join(transformed_vol, "resampled_merged.tif"), merged)#, compress=1) except Exception as e: print(e) with open(error_file, "a") as err_fl: err_fl.write("\n\n{} {}\n\n".format(fld, e))
Created on Wed Jun 19 13:28:24 2019 @author: wanglab """ from __future__ import division import os, numpy as np os.chdir("/jukebox/wang/zahra/lightsheet_copy") from tools.conv_net.functions.bipartite import pairwise_distance_metrics from tools.utils.io import listdirfull, load_np, makedir, load_dictionary, save_dictionary from tools.conv_net.input.read_roi import read_roi_zip if __name__ == "__main__": #load points dict points_dict = load_dictionary( "/home/wanglab/Documents/cfos_inputs/filename_points_dictionary.p") print(points_dict.keys()) #setup comparison b/w clearmap and annotator(s)? dafina and tiffancy (highest f1 between them) src = "/home/wanglab/mounts/wang/Jess/lightsheet_output/201904_ymaze_cfos/clearmap_accuracy_quantification" clearmap_dsets = dict([(os.path.basename(xx)[:-13] + ".npy", np.load(os.path.join(src, xx))) for xx in os.listdir(src) if xx[-4:] == ".npy"]) #initialise empty vectors tps = [] fps = [] fns = [] #set voxel cutoff value