示例#1
0
def apply_classifier(classifier, raw_src, cnn_src, collect_cnn = False, size = (3,12,12), pad=False, cores=10, numZSlicesPerSplit=50, overlapping_planes = 15, verbose=True, save=True, maxip=0):
    '''
    classifier = pretrained random forest or path to pretrained random forest
    raw_src = folder of tiffs of raw input data
    cnn_src = folder of tiffs from cnn output
    size
    pad = if True, pad the edges of objects determined. False: remove edge cases, usually better since they will be few in number
    cores = number of cores for parallelization, larger the number the less memory efficient
    numZSlicesPerSplit: chunk of zplanes to process at once. Adjust this and cores based on memory constraints.
    overlapping_planes: number of planes on each side to overlap by, this should be a comfortable amount larger than the maximum z distances of a single object
    save (optional): #optional save to prevent rerun of jobs
    collect_cnn = optional to include cnn data for random forest input
    
    Returns
    ----------------
    a dictionary consisting of k=centers, v=[corresponding pixel indices determine by CNN, maximum intensity, list of maximum radius/plane]
    
    '''
    #handle inputs
    threshold = 1
    zyx_search_range = (2,10,10)
    zdim = len(listdirfull(cnn_src, keyword='.tif'))
    
    #optional save to prevent rerun of jobs
    if save: 
        save = cnn_src+'_apply_classifier_tmp'
        makedir(save)
    
    #run
    if verbose: sys.stdout.write('\n   Thesholding, determining connected pixels, identifying center of masses, applying classifier\n\n'); sys.stdout.flush(); st = time.time()
    rng = range(0, zdim, numZSlicesPerSplit); jobs = len(rng);
    iterlst=[(cnn_src, raw_src, collect_cnn, z, zdim, numZSlicesPerSplit, overlapping_planes, threshold, classifier, size, zyx_search_range, pad, job, jobs, verbose, save, maxip) for job, z in enumerate(rng)]
    #par vs not par
    if cores > 1:
        p = mp.Pool(cores)
        center_pixels_intensity_radius_lst = p.starmap(apply_classifier_helper, iterlst)
        p.terminate()
    else:
        center_pixels_intensity_radius_lst = []
        for i in iterlst:
            center_pixels_intensity_radius_lst.append(apply_classifier_helper(i))
    #optional reload:
    if save:
        center_pixels_intensity_radius_lst = [load_dictionary(xx) for xx in listdirfull(save)]
        shutil.rmtree(save)

    #unpack
    if verbose: sys.stdout.write('\n...finished, formatting dictionary...'); sys.stdout.flush()
    center_pixels_intensity_radius_dct = {}; [center_pixels_intensity_radius_dct.update(xx) for xx in center_pixels_intensity_radius_lst]
    if 'None' in center_pixels_intensity_radius_dct: del center_pixels_intensity_radius_dct['None']
        
    if verbose: print ('Total time {} minutes'.format(round((time.time() - st) / 60)))
    if verbose: print('{} centers found.'.format(len(center_pixels_intensity_radius_dct)))

    return center_pixels_intensity_radius_dct
def generate_transformed_cellcount(dataframe,
                                   dst,
                                   transformfiles,
                                   lightsheet_parameter_dictionary,
                                   verbose=False):
    '''Function to take a csv file and generate an input to transformix
    
    Inputs
    ----------------
    dataframe = preloaded pandas dataframe
    dst = destination to save files
    transformfiles = list of all elastix transform files used, and in order of the original transform****
    lightsheet_parameter_file = .p file generated from lightsheet package
    '''
    #set up locations
    transformed_dst = os.path.join(dst, 'transformed_points')
    makedir(transformed_dst)

    #make zyx numpy arry
    zyx = dataframe[['z', 'y', 'x']].values

    #adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample
    kwargs = load_dictionary(lightsheet_parameter_dictionary)
    vol = [xx for xx in kwargs['volumes'] if xx.ch_type == 'cellch'][0]
    fullsizedimensions = get_fullsizedims_from_kwargs(
        kwargs
    )  #don't get from kwargs['volumes'][0].fullsizedimensions it's bad! use this instead
    zyx = fix_contour_orientation(zyx, verbose=verbose,
                                  **kwargs)  #now in orientation of resample
    zyx = points_resample(
        zyx,
        original_dims=fix_dimension_orientation(fullsizedimensions, **kwargs),
        resample_dims=tifffile.imread(vol.resampled_for_elastix_vol).shape,
        verbose=verbose)[:, :3]

    #make into transformix-friendly text file
    pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst)

    #copy over elastix files
    transformfiles = modify_transform_files(transformfiles, transformed_dst)
    change_transform_parameter_initial_transform(transformfiles[0],
                                                 'NoInitialTransform')

    #run transformix on points
    points_file = point_transformix(pretransform_text_file, transformfiles[-1],
                                    transformed_dst)

    #convert registered points into structure counts
    converted_points = unpack_pnts(points_file, transformed_dst)

    return converted_points
示例#3
0
def original_directory_structure(src, transfertype="move"):
    """
    Move files back
    src = "/home/wanglab/wang/pisano/ymaze/cfos_4x/20171129_ymaze_23/full_sizedatafld/20171129_ymaze_23_488_050na_z7d5um_50msec_10povlap_ch00/terastitcher_dct.p"
    
    """
    for k, v in pth_update(load_dictionary(src)["terastitcher_dct"]).items():
        try:
            if transfertype == "copy":
                shutil.copy(pth_update(v), pth_update(k))
            if transfertype == "move":
                shutil.move(pth_update(v), pth_update(k))
        except:
            print("?")

    return
示例#4
0
def overlay_cells(src,
                  center_intensity_radius,
                  dilation_radius=5,
                  load_range=False):
    '''
    
    src = fullsizedata folder
    center_intensity_radius = path to center_intensity_radius.p file 
    
    src = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20161214_db_bl6_crii_l_53hr/full_sizedatafld/20161214_db_bl6_crii_l_53hr_647_010na_z7d5um_75msec_5POVLP_ch00'
    center_intensity_radius = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20161214_db_bl6_crii_l_53hr/cells/20161214_db_bl6_crii_l_53hr_647_010na_z7d5um_75msec_5POVLP_ch00_centers_intensity_radius.p'
    load_range = (400,450)
    dilation_radius = 5
    
    src = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20170115_tp_bl6_lob6a_1000r_02/full_sizedatafld/20170115_tp_bl6_lob6a_1000r_647_010na_z7d5um_125msec_10povlp_ch00'
    center_intensity_radius = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20170115_tp_bl6_lob6a_1000r_02/cells/20170115_tp_bl6_lob6a_1000r_647_010na_z7d5um_125msec_10povlp_ch00_centers_intensity_radius.p'
    
    src = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20170115_tp_bl6_lob6a_1000r_02/cells/test'
    center_intensity_radius = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20170115_tp_bl6_lob6a_1000r_02/cells/test_out.p'
    load_range = (0,100)
    
    '''

    #load centers
    center_intensity_radius = load_dictionary(center_intensity_radius) if type(
        center_intensity_radius) == str else center_intensity_radius
    centers = sorted(center_intensity_radius.keys(), key=lambda x: x[0])

    #adjust and compensate for load range
    if load_range:
        centers = [
            tuple((c[0] - load_range[0], c[1], c[2])) for c in centers
            if c[0] in range(load_range[0], load_range[1])
        ]

    #load
    vol = load_tiff_folder(src, threshold=0, load_range=load_range)
    vol0 = np.zeros_like(vol).astype('bool')
    for c in centers:
        vol0[c[0], c[1], c[2]] = True

    vol0 = dilate_with_element(vol0 * 1, ball(dilation_radius)).astype('uint8')
    #

    return vol, vol0
示例#5
0
def overlay_qc(args):  
    #unpacking this way for multiprocessing
    fld, folder_suffix, output_folder, verbose, doubletransform, make_volumes = args
    try:
        #get 3dunet cell dataframe csv file
        input_csv = listdirfull(os.path.join(fld, folder_suffix), ".csv")
        assert len(input_csv) == 1, "multiple csv files"
        dataframe = pd.read_csv(input_csv[0])
        
        #location to save out
        dst = os.path.join(output_folder, os.path.basename(fld)); makedir(dst)
    
        #EXAMPLE USING LIGHTSHEET - assumes marking centers in the "raw" full sized cell channel. This will transform those 
        #centers into "atlas" space (in this case the moving image)
        #in this case the "inverse transform has the atlas as the moving image in the first step, 
        #and the autofluorescence channel as the moving image in the second step 
        #NOTE - it seems that the registration of cell to auto is failing on occasion....thus get new files...
        ################################
        cell_inverse_folder = listdirfull(os.path.join(fld, "elastix_inverse_transform"), "cellch")[0]
        a2r = listall(cell_inverse_folder, "atlas2reg_TransformParameters"); a2r.sort()
        r2s = listall(cell_inverse_folder, "reg2sig_TransformParameters"); r2s.sort() #possibly remove

        #IMPORTANT. the idea is to apply cfos->auto->atlas
        transformfiles = r2s + a2r if doubletransform else a2r #might get rid of r2s
    
        lightsheet_parameter_dictionary = os.path.join(fld, "param_dict.p")
            
        converted_points = generate_transformed_cellcount(dataframe, dst, transformfiles, 
                                                          lightsheet_parameter_dictionary, verbose=verbose)
    
        #load and convert to single voxel loc
        zyx = np.asarray([str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in np.nan_to_num(np.load(converted_points))])
        from collections import Counter
        zyx_cnt = Counter(zyx)
        
        #check...
        if make_volumes:
            #manually call transformix
            kwargs = load_dictionary(lightsheet_parameter_dictionary)
            vol = [xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0].resampled_for_elastix_vol
            transformed_vol = os.path.join(dst, "transformed_volume"); makedir(transformed_vol)
            if not doubletransform:
                transformfiles = [os.path.join(fld, "elastix/TransformParameters.0.txt"), os.path.join(fld, 
                                  "elastix/TransformParameters.1.txt")]
                transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files
                transformix_command_line_call(vol, transformed_vol, transformfiles[-1])
            else:
                v=[xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0]
                #sig to reg
                tps = [listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.0")[0], 
                       listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.1")[0]]
                #reg to atlas
                transformfiles = tps+[os.path.join(fld, "elastix/TransformParameters.0.txt"), 
                                      os.path.join(fld, "elastix/TransformParameters.1.txt")]
                transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files
                transformix_command_line_call(vol, transformed_vol, transformfiles[-1])
            

            #cell_registered channel
            cell_reg = tifffile.imread(os.path.join(transformed_vol, "result.tif"))
            tifffile.imsave(os.path.join(transformed_vol, "result.tif"), cell_reg, compress=1)
            cell_cnn = np.zeros_like(cell_reg)
            tarr = []; badlist=[]
            for zyx,v in zyx_cnt.items():
                z,y,x = [int(xx) for xx in zyx.replace("(","",).replace(")","").split(",")]
                tarr.append([z,y,x])
                try:
                    cell_cnn[z,y,x] = v*100
                except:
                    badlist.append([z,y,x])
                    
            #apply x y dilation
            r = 2
            selem = ball(r)[int(r/2)]
            cell_cnn = cell_cnn.astype("uint8")
            cell_cnn = np.asarray([cv2.dilate(cell_cnn[i], selem, iterations = 1) for i in range(cell_cnn.shape[0])])
            
            tarr=np.asarray(tarr)
            if len(badlist)>0: 
                print("{} errors in mapping with cell_cnn shape {}, each max dim {}, \npossibly due to a registration overshoot \
                      or not using double transform\n\n{}".format(len(badlist), cell_cnn.shape, np.max(tarr,0), badlist))
            merged = np.stack([cell_cnn, cell_reg, np.zeros_like(cell_reg)], -1)
            tifffile.imsave(os.path.join(transformed_vol, "merged.tif"), merged)#, compress=1)
            #out = np.concatenate([cell_cnn, cell_reg, ], 0)
        
        #####check at the resampled for elastix phase before transform...this mapping looks good...
        if make_volumes:
            #make zyx numpy arry
            zyx = dataframe[["z","y","x"]].values
            kwargs = load_dictionary(lightsheet_parameter_dictionary)
            vol = [xx for xx in kwargs["volumes"] if xx.ch_type =="cellch"][0]
            fullsizedimensions = get_fullsizedims_from_kwargs(kwargs) #don"t get from kwargs["volumes"][0].fullsizedimensions it"s bad! use this instead
            zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample
            zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs), 
                                  resample_dims = tifffile.imread(vol.resampled_for_elastix_vol).shape, verbose = verbose)[:, :3]
            
            #cell channel
            cell_ch = tifffile.imread(vol.resampled_for_elastix_vol)
            cell_cnn = np.zeros_like(cell_ch)
            tarr = []; badlist=[]
            for _zyx in zyx:
                z,y,x = [int(xx) for xx in _zyx]
                tarr.append([z,y,x])
                try:
                    cell_cnn[z,y,x] = 100
                except:
                    badlist.append([z,y,x])
            tarr=np.asarray(tarr)        
            merged = np.stack([cell_cnn, cell_ch, np.zeros_like(cell_ch)], -1)
            tifffile.imsave(os.path.join(transformed_vol, "resampled_merged.tif"), merged)#, compress=1)
            
    except Exception as e:
        print(e)
        with open(error_file, "a") as err_fl:
            err_fl.write("\n\n{} {}\n\n".format(fld, e))
Created on Wed Jun 19 13:28:24 2019

@author: wanglab
"""

from __future__ import division
import os, numpy as np
os.chdir("/jukebox/wang/zahra/lightsheet_copy")
from tools.conv_net.functions.bipartite import pairwise_distance_metrics
from tools.utils.io import listdirfull, load_np, makedir, load_dictionary, save_dictionary
from tools.conv_net.input.read_roi import read_roi_zip

if __name__ == "__main__":

    #load points dict
    points_dict = load_dictionary(
        "/home/wanglab/Documents/cfos_inputs/filename_points_dictionary.p")

    print(points_dict.keys())

    #setup comparison b/w clearmap and annotator(s)? dafina and tiffancy (highest f1 between them)
    src = "/home/wanglab/mounts/wang/Jess/lightsheet_output/201904_ymaze_cfos/clearmap_accuracy_quantification"
    clearmap_dsets = dict([(os.path.basename(xx)[:-13] + ".npy",
                            np.load(os.path.join(src, xx)))
                           for xx in os.listdir(src) if xx[-4:] == ".npy"])

    #initialise empty vectors
    tps = []
    fps = []
    fns = []

    #set voxel cutoff value