def point_transformix(txtflnm, transformfile, dst=False): '''apply elastix transform to points Inputs ------------- txtflnm = list of points that have resizingtransform transformfile = elastix transform file dst = optional folder Returns --------------- trnsfrm_out_file = pth to file containing post transformix points ''' sys.stdout.write('\n***********Starting Transformix***********') from subprocess import check_output #set paths if not dst: trnsfrm_outpath = os.path.join(os.path.dirname(transformfile), 'posttransformix') makedir(trnsfrm_outpath) if dst: trnsfrm_outpath = os.path.join(dst, 'posttransformix') makedir(trnsfrm_outpath) trnsfrm_out_file = os.path.join(trnsfrm_outpath, 'outputpoints.txt') #run transformix point transform call = 'transformix -def {} -out {} -tp {}'.format(txtflnm, trnsfrm_outpath, transformfile) print(check_output(call, shell=True)) writer(trnsfrm_outpath, '\n Transformix File Generated: {}'.format(trnsfrm_out_file)) return trnsfrm_out_file
def generate_masked_atlas(binarize = False, **kwargs): '''Wrapper to call mask_atlas and save in appropriate place Inputs ------------------- binarize: (optional) if true convert all nonzero pixels to 255 kwargs used: ouputdirectory AtlasFile maskatlas Returns: ------------------- path to newly saved masked atlas ''' #make folder to store masked atlas fld = os.path.join(kwargs['outputdirectory'], 'maskedatlas') makedir(fld) #generate masked_atlas masked_atlas = mask_atlas(**kwargs) if binarize: masked_atlas[masked_atlas>0]=255 masked_atlas = change_bitdepth(masked_atlas) #save masked_atlas masked_atlas_pth = os.path.join(fld, 'masked_atlas.tif') tifffile.imsave(masked_atlas_pth, masked_atlas) return masked_atlas_pth
def overlay(AtlasFile, svlc, nm_to_sv, arr, gaussianblur=False): '''function to plot coordinates using depth color overlay. Inputs: pth=pth to allen annoation AtlasFile=pth to atlas (grayscale) svlc=pth to folder to save nm_to_sv=text to identify this structure arr= np array of [zyx] PIXEL COORDINATES to plot gaussianblur (optional):False/True ''' im = sitk.GetArrayFromImage(sitk.ReadImage(AtlasFile)) im = np.zeros(im.shape).astype('uint8') for i in tuple(map(tuple, arr.astype('int'))): im[i] = 255 #optional if gaussianblur: from scipy.ndimage.filters import gaussian_filter im = gaussian_filter(im, (2,2,2)).astype('uint8') im[im>0]=255 im = gaussian_filter(im, (2,2,2)).astype('uint8') im[im>0]=255 makedir(svlc) svlc2=os.path.join(svlc, nm_to_sv) makedir(svlc2) impth=os.path.join(svlc2, str(nm_to_sv)+'.tif') tifffile.imsave(impth, im.astype('uint8')) allen_compare(AtlasFile, svlc, impth) return
def blend_lightsheets(flds, dst, cores, cleanup=False): """0=L, 1=R """ #make sure l and r are in appropriate positions if np.any(["right_lightsheet" in xx for xx in flds]) and np.any( ["left_lightsheet" in xx for xx in flds]): l = [xx for xx in flds if "left_lightsheet" in xx][0] r = [xx for xx in flds if "right_lightsheet" in xx][0] flds = [l, r] # st = time.time() name = os.path.basename(dst) ch = dst[-2:] sys.stdout.write("\nStarting blending of {}...".format(dst)) sys.stdout.flush() ydim0, xdim0 = sitk.GetArrayFromImage( sitk.ReadImage(listall(flds[0], keyword=".tif")[0])).shape ydim1, xdim1 = sitk.GetArrayFromImage( sitk.ReadImage(listall(flds[1], keyword=".tif")[0])).shape ydim = ydim0 if ydim0 < ydim1 else ydim1 xdim = xdim0 if xdim0 < xdim1 else xdim1 #alpha=np.tile(scipy.stats.logistic.cdf(np.linspace(-250, 250, num=xdim)), (ydim, 1)) fls0 = listall(flds[0], keyword=".tif") fls0.sort() fls1 = listall(flds[1], keyword=".tif") fls1.sort() fls = fls0 if len(fls0) < len(fls1) else fls1 makedir(dst) #makedir(os.path.join(dst, name)) iterlst = [{ "xdim": xdim, "ydim": ydim, "fl0": fls0[i], "channel": ch, "fl1": fls1[i], "dst": dst, "name": name, "zplane": i } for i, fl0 in enumerate(fls)] if cores >= 2: p = mp.Pool(cores) p.map(blend, iterlst) p.terminate() else: [blend(dct) for dct in iterlst] #if cleanup: [shutil.rmtree(xx) for xx in flds] sys.stdout.write("\n...finished in {} minutes.\n".format(np.round( (time.time() - st) / 60), decimals=2)) sys.stdout.flush() return
def apply_classifier(classifier, raw_src, cnn_src, collect_cnn = False, size = (3,12,12), pad=False, cores=10, numZSlicesPerSplit=50, overlapping_planes = 15, verbose=True, save=True, maxip=0): ''' classifier = pretrained random forest or path to pretrained random forest raw_src = folder of tiffs of raw input data cnn_src = folder of tiffs from cnn output size pad = if True, pad the edges of objects determined. False: remove edge cases, usually better since they will be few in number cores = number of cores for parallelization, larger the number the less memory efficient numZSlicesPerSplit: chunk of zplanes to process at once. Adjust this and cores based on memory constraints. overlapping_planes: number of planes on each side to overlap by, this should be a comfortable amount larger than the maximum z distances of a single object save (optional): #optional save to prevent rerun of jobs collect_cnn = optional to include cnn data for random forest input Returns ---------------- a dictionary consisting of k=centers, v=[corresponding pixel indices determine by CNN, maximum intensity, list of maximum radius/plane] ''' #handle inputs threshold = 1 zyx_search_range = (2,10,10) zdim = len(listdirfull(cnn_src, keyword='.tif')) #optional save to prevent rerun of jobs if save: save = cnn_src+'_apply_classifier_tmp' makedir(save) #run if verbose: sys.stdout.write('\n Thesholding, determining connected pixels, identifying center of masses, applying classifier\n\n'); sys.stdout.flush(); st = time.time() rng = range(0, zdim, numZSlicesPerSplit); jobs = len(rng); iterlst=[(cnn_src, raw_src, collect_cnn, z, zdim, numZSlicesPerSplit, overlapping_planes, threshold, classifier, size, zyx_search_range, pad, job, jobs, verbose, save, maxip) for job, z in enumerate(rng)] #par vs not par if cores > 1: p = mp.Pool(cores) center_pixels_intensity_radius_lst = p.starmap(apply_classifier_helper, iterlst) p.terminate() else: center_pixels_intensity_radius_lst = [] for i in iterlst: center_pixels_intensity_radius_lst.append(apply_classifier_helper(i)) #optional reload: if save: center_pixels_intensity_radius_lst = [load_dictionary(xx) for xx in listdirfull(save)] shutil.rmtree(save) #unpack if verbose: sys.stdout.write('\n...finished, formatting dictionary...'); sys.stdout.flush() center_pixels_intensity_radius_dct = {}; [center_pixels_intensity_radius_dct.update(xx) for xx in center_pixels_intensity_radius_lst] if 'None' in center_pixels_intensity_radius_dct: del center_pixels_intensity_radius_dct['None'] if verbose: print ('Total time {} minutes'.format(round((time.time() - st) / 60))) if verbose: print('{} centers found.'.format(len(center_pixels_intensity_radius_dct))) return center_pixels_intensity_radius_dct
def move_images_after_stitching(ts_out, dst, channel): #make sure destination exists makedir(dst) imgs = listall(ts_out, '.tif') imgs.sort() #parellelize iterlst = [(i, img, dst, channel) for i, img in enumerate(imgs)] p = mp.Pool(6) p.starmap(move_images_after_stitching_par, iterlst) return dst
def generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=False): '''Function to take a csv file and generate an input to transformix Inputs ---------------- dataframe = preloaded pandas dataframe dst = destination to save files transformfiles = list of all elastix transform files used, and in order of the original transform**** lightsheet_parameter_file = .p file generated from lightsheet package ''' #set up locations transformed_dst = os.path.join(dst, 'transformed_points') makedir(transformed_dst) #make zyx numpy arry zyx = dataframe[['z', 'y', 'x']].values #adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample kwargs = load_dictionary(lightsheet_parameter_dictionary) vol = [xx for xx in kwargs['volumes'] if xx.ch_type == 'cellch'][0] fullsizedimensions = get_fullsizedims_from_kwargs( kwargs ) #don't get from kwargs['volumes'][0].fullsizedimensions it's bad! use this instead zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample zyx = points_resample( zyx, original_dims=fix_dimension_orientation(fullsizedimensions, **kwargs), resample_dims=tifffile.imread(vol.resampled_for_elastix_vol).shape, verbose=verbose)[:, :3] #make into transformix-friendly text file pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst) #copy over elastix files transformfiles = modify_transform_files(transformfiles, transformed_dst) change_transform_parameter_initial_transform(transformfiles[0], 'NoInitialTransform') #run transformix on points points_file = point_transformix(pretransform_text_file, transformfiles[-1], transformed_dst) #convert registered points into structure counts converted_points = unpack_pnts(points_file, transformed_dst) return converted_points
def make_jobs(image_dictionary): '''Simple function to create job dct for parallelization ''' jobdct={} lslst = ['left_lightsheet', 'right_lightsheet'] for channel in image_dictionary['channels']: for lightsheet in range(image_dictionary['lightsheets']): if image_dictionary['lightsheets'] == 2: side=['_C00_', '_C01_'][lightsheet] dct = copy.deepcopy(image_dictionary) dct['zchanneldct']={k:{kk:[xx for xx in vv if side in xx]} for k,v in dct['zchanneldct'].iteritems() for kk,vv in v.iteritems()} name = os.path.basename(os.path.dirname((dct['zchanneldct'].values()[0].values()[0][0]))) out = os.path.join(dst, '{}_{}'.format(name, lslst[lightsheet])); makedir(out) jobdct['channel{}_lightsheet{}'.format(channel, lightsheet)] = copy.deepcopy({'job': 'channel{}_lightsheet{}'.format(channel, lightsheet), 'name': name, 'lightsheet': lslst[lightsheet], 'channel': channel, 'dst':dct['dst'], 'dct': copy.deepcopy(dct), 'out':out, 'cores':1}) return jobdct
def inj_and_cells(elastixfld, cellch, injch, AtlasFile, threshold=0.075): ###assumes sagittal files ###set paths for ###generate Allen Atlas background allen = os.path.join(elastixfld, 'allenatlas_inj') makedir(allen) allentiff = tifffile.imread(AtlasFile[:-4] + '.tif')[:, 390:, :] allentiffloc = os.path.join(allen, 'allen_for_inj.tif') tifffile.imsave(allentiffloc, allentiff) depth.colorcode(allentiffloc, allen) grayscale = [ os.path.join(allen, 'proj0.png'), os.path.join(allen, 'proj1.png'), os.path.join(allen, 'proj2.png') ] ####parsing out injection site ninjfld = os.path.join(elastixfld, 'injection_and_cells') makedir(ninjfld) injstack = tifffile.imread(injch)[:, 390:, :] ##assumes sagittal ABA 25um ninjstackloc = os.path.join(ninjfld, 'injectionsite.tif') #normalize and threshold mxx = injstack.max() injstack = (injstack - injstack.min()) / (injstack.max() - injstack.min()) injstack[injstack < threshold] = 0 injstack = injstack * mxx tifffile.imsave(ninjstackloc, injstack.astype('uint16')) ####applying depth coding depth.colorcode(ninjstackloc, ninjfld) ###apply overlay color = [ os.path.join(ninjfld, 'proj0.png'), os.path.join(ninjfld, 'proj1.png'), os.path.join(ninjfld, 'proj2.png') ] nametosave = [ os.path.join(ninjfld, "proj0_overlay.png"), os.path.join(ninjfld, "proj1_overlay.png"), os.path.join(ninjfld, "proj2_overlay.png") ] for x in range(3): print(grayscale[x], color[x], nametosave[x]) depth.overlay(grayscale[x], color[x], nametosave[x], alpha=0.95) depth.layout(nametosave[1], nametosave[0], nametosave[2], ninjfld) print(grayscale) print(color) print(nametosave) writer(elastixfld, "\nFinished inj_and_cells depth coloring overlay") return
def resize(src, resizefactor, cores): """ src = fullsizedata_fld """ #find files fls = listall(src, keyword=".tif") #calc resize y, x = tifffile.imread(fls[0], multifile=False).shape yr = int(y / resizefactor) xr = int(x / resizefactor) #set up dsts [ makedir( os.path.join(os.path.dirname(src), xx[:-4] + "resized_" + xx[-4:])) for xx in os.listdir(src) if ".txt" not in xx ] #parallelize iterlst = [copy.deepcopy({"fl": fl, "xr": xr, "yr": yr}) for fl in fls] p = mp.Pool(cores) p.map(resize_helper, iterlst) p.terminate() return
def isolate_and_overlay(pth, AtlasFile, svlc, nm_to_sv,*args): '''function to segment out allen brain structures and then depth color overlay. Inputs: pth=pth to allen annoation AtlasFile=pth to atlas (grayscale) svlc=pth to folder to save nm_to_sv=text to identify this structure *args=lst of structure ID's (pixel values) to keep ''' im=isolate_structures(pth, *args) makedir(svlc) svlc2=os.path.join(svlc, nm_to_sv) makedir(svlc2) impth=os.path.join(svlc2, str(nm_to_sv)+'.tif') tifffile.imsave(impth, im.astype('uint8')) allen_compare(AtlasFile, svlc, impth) return
def similarity_transform(fx, mv, dst, nm, level='fast', cleanup=True): '''function for similarity transform Inputs ------------------- fx = fixed path (usually Atlas for 'normal' noninverse transforms) mv = moving path (usually volume to register for 'normal' noninverse transforms) dst = location to save nm = 'name of file to save' level = 'fast', 'intermediate', 'slow' : links to parameter files of certain complexity cleanup = if False do not remove files Returns ------------- path to file path to transform file (if cleanup == False) ''' transform_to_use = { 'slow': '/jukebox/wang/pisano/Python/lightsheet/supp_files/similarity_transform_slow.txt', 'intermediate': '/jukebox/wang/pisano/Python/lightsheet/supp_files/similarity_transform_intermediate.txt', 'fast': '/jukebox/wang/pisano/Python/lightsheet/supp_files/similarity_transform_fast.txt' }[level] #make dir makedir(dst) out = os.path.join(dst, 'tmp') makedir(out) fl, tp = elastix_command_line_call(fx, mv, out=out, parameters=[transform_to_use]) #move and delete if nm[-4:] != '.tif': nm = nm + '.tif' dstfl = os.path.join(dst, nm) shutil.move(fl, dstfl) if cleanup: shutil.rmtree(out) print('saved as {}'.format(dstfl)) if cleanup: return dstfl if not cleanup: return dstfl, tp
def points_transform(src, dst, transformfile, verbose=False): '''Function to apply a tranform given a numpy. ***Assumes ZYX and that any orientation changes have already been done.*** src: numpy array or list of np arrays of dims nx3 dst: folder to save ''' src = np.asarray(src) assert src.shape[-1] == 3, 'src must be a nx3 array' #create txt file, with elastix header, then populate points makedir(dst) pnts_fld = os.path.join(dst, 'transformedpoints_pretransformix') makedir(pnts_fld) transforminput = os.path.join(pnts_fld, 'zyx_transformedpnts.txt') #prevent adding to an already made file removedir(transforminput) writer(pnts_fld, 'index\n{}\n'.format(len(src)), flnm=transforminput) if verbose: sys.stdout.write( '\nwriting centers to transfomix input points text file: {}....'. format(transforminput)) #this step converts from zyx to xyz***** stringtowrite = '\n'.join( ['\n'.join(['{} {} {}'.format(i[2], i[1], i[0])]) for i in src]) writer(pnts_fld, stringtowrite, flnm=transforminput) if verbose: sys.stdout.write('...done writing centers.') sys.stdout.flush() #elastix for inverse transform trnsfrm_out_file = point_transformix(txtflnm=transforminput, dst=dst, transformfile=transformfile) assert os.path.exists( trnsfrm_out_file), 'Error Transform file does not exist: {}'.format( trnsfrm_out_file) #if verbose: sys.stdout.write('\n***Elastix Inverse Transform File: {}***'.format(transformfile)) return trnsfrm_out_file
def blend_lightsheets_pre_stitching(left, right, dst, cores): st = time.time() name = os.path.basename(dst) ch = dst[-2:] sys.stdout.write("\nStarting blending of {}...".format(dst)) sys.stdout.flush() ydim0, xdim0 = sitk.GetArrayFromImage(sitk.ReadImage(left[0])).shape ydim1, xdim1 = sitk.GetArrayFromImage(sitk.ReadImage(right[0])).shape #checks assert ydim0 == ydim1 assert xdim0 == xdim1 assert len(left) == len(right) makedir(dst) iterlst = [{ "xdim": xdim0, "ydim": ydim0, "fl0": left[i], "channel": ch, "fl1": right[i], "dst": dst, "name": name, "zplane": i, "stitched": False } for i, l in enumerate(left)] if cores >= 2: p = mp.Pool(cores) p.map(blend, iterlst) p.terminate() else: [blend(dct) for dct in iterlst] #if cleanup: [shutil.rmtree(xx) for xx in flds] sys.stdout.write("\n...finished in {} minutes.\n".format(np.round( (time.time() - st) / 60), decimals=2)) sys.stdout.flush() return
def blend_lightsheets(name, flds, dst, cores): ''' ''' sys.stdout.write('\nStarting blending of {}...'.format(name)); sys.stdout.flush() ydim, xdim =tifffile.imread(listall(flds[0], keyword='.tif')[0]).shape alpha=np.tile(scipy.stats.logistic.cdf(np.linspace(-250, 250, num=xdim)), (ydim, 1)) fls0 = listall(flds[0], keyword='.tif'); fls0.sort() fls1 = listall(flds[1], keyword='.tif'); fls1.sort() assert set([os.path.basename(xx) for xx in fls0]) == set([os.path.basename(xx) for xx in fls1]), 'uneven number of z planes between L and R lightsheets' makedir(os.path.join(dst, name)) iterlst=[{'alpha':alpha, 'fl0':fl0, 'fl1':fls1[i], 'dst':dst, 'name':name, 'zplane':i} for i,fl0 in enumerate(fls0)] if cores>=2: p=mp.Pool(cores) p.map(blend, iterlst) p.terminate() else: [blend(dct) for dct in iterlst] [shutil.rmtree(xx) for xx in flds] sys.stdout.write('completed.\n'.format(name)); sys.stdout.flush() return
def generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=False): """Function to take a csv file and generate an input to transformix Inputs ---------------- dataframe = preloaded pandas dataframe dst = destination to save files transformfiles = list of all elastix transform files used, and in order of the original transform**** lightsheet_parameter_file = .p file generated from lightsheet package """ #set up locations transformed_dst = os.path.join(dst, "transformed_points"); makedir(transformed_dst) #make zyx numpy arry zyx = dataframe[["z","y","x"]].values #adjust for reorientation THEN rescaling, remember full size data needs dimension change releative to resample fullsizedimensions = get_fullsizedimensions(lightsheet_parameter_dictionary) kwargs = load_kwargs(lightsheet_parameter_dictionary) zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample resampled_dims, resampled_vol = get_resampledvol_n_dimensions(lightsheet_parameter_dictionary) zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs), resample_dims = resampled_dims, verbose = verbose)[:, :3] #make into transformix-friendly text file pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst) #copy over elastix files transformfiles = modify_transform_files(transformfiles, transformed_dst) change_transform_parameter_initial_transform(transformfiles[0], "NoInitialTransform") #run transformix on points points_file = point_transformix(pretransform_text_file, transformfiles[-1], transformed_dst) #convert registered points into structure counts converted_points = unpack_pnts(points_file, transformed_dst) return converted_points
def generate_transformed_cellcount(points, dst, transformfiles): """ makes an input to transformix """ #set up locations transformed_dst = os.path.join(dst, "transformed_points") makedir(transformed_dst) #make zyx numpy arry zyx = np.asarray(points) transformfiles = modify_transform_files(transformfiles, transformed_dst) change_transform_parameter_initial_transform(transformfiles[0], "NoInitialTransform") pretransform_text_file = create_text_file_for_elastix(zyx, transformed_dst) #run transformix on points points_file = point_transformix(pretransform_text_file, transformfiles[-1], transformed_dst) #convert registered points into structure counts converted_points = unpack_pnts(points_file, transformed_dst) return converted_points
def generate_cropped_atlas(**kwargs): '''Wrapper to call crop_atlas and save in appropriate place Inputs ------------------- kwargs used: ouputdirectory AtlasFile maskatlas Returns: ------------------- path to newly saved cropped atlas ''' #make folder to store masked atlas fld = os.path.join(kwargs['outputdirectory'], 'croppedatlas') makedir(fld) #generate masked_atlas cropped_atlas = crop_atlas(**kwargs) #save masked_atlas cropped_atlas_pth = os.path.join(fld, 'cropped_atlas.tif') tifffile.imsave(cropped_atlas_pth, cropped_atlas) return cropped_atlas_pth
ch = "Ex_642_Em_2" #for array job parallelization print(os.environ["SLURM_ARRAY_TASK_ID"]) jobid = int(os.environ["SLURM_ARRAY_TASK_ID"]) # jobid=0 #set brain name brain = os.path.join(src, brains[jobid]) start = time.time() #need to change this config depending on processing pipeline cellvol = fast_scandir(os.path.join(brain,ch,"stitched"))[-1] #link to parameters a2r0 = os.path.join(brain, "elastix_inverse_transform/TransformParameters.0.txt") a2r1 = os.path.join(brain, "elastix_inverse_transform/TransformParameters.1.txt") #set destination directory braindst = os.path.join(scratch_dir, os.path.basename(brain)) makedir(braindst) aldst = os.path.join(braindst, "transformed_annotations"); makedir(aldst) #transformix # transformfiles = modify_transform_files(transformfiles=[a2r0, a2r1, r2s0, r2s1], dst = aldst) transformfiles = modify_transform_files(transformfiles=[a2r0, a2r1], dst = aldst) [change_interpolation_order(xx,0) for xx in transformfiles] #change the parameter in the transform files that outputs 16bit images instead for fl in transformfiles:# Read in the file with open(fl, "r") as file: filedata = file.read() # Replace the target string filedata = filedata.replace('(ResultImagePixelType "short")', '(ResultImagePixelType "float")') # Write the file out again with open(fl, "w") as file: file.write(filedata) #run transformix
filtered = gfilt(clh**power, filter_kernel) #tif.imsave("/home/wanglab/Desktop/filtered_z200-300.tif", filtered[200:300].astype("float32")) # #plt.figure() #plt.imshow(filtered[300], "gist_yarg") thresholded = filtered > filtered.mean() + thresh * filtered.std() labelled, nlab = label(thresholded) #plt.figure() sizes = [np.sum(labelled == i) for i in range(1, nlab + 1)] vals = [i + 1 for i in np.argsort(sizes)[::-1]] arr = np.in1d(labelled, vals).reshape(labelled.shape) seg_dst = os.path.join(dst, "rawdata_segmentations") makedir(seg_dst) tif.imsave( os.path.join(seg_dst, "%s.tif" % os.path.basename(brain)), np.in1d(labelled, vals).reshape(labelled.shape).astype("uint16")) im = tif.imread(imgs[0]) arr_fullsz = np.zeros((len(imgs), im.shape[0], im.shape[1])) print(arr_fullsz.shape) print(arr.shape) ################################################################################################################################################################# arr_fullsz[:, 1700:, :] = arr #plt.imshow(arr_fullsz[300]) zf, yf, xf = 1, (1 / 3), (1 / 3)
#this isn't a perfect Quality control and you can expect some pixel shifts #qc_overlay_transform_type = 'affine_only_reg_to_sig';#both for reg+atlas, and only affine for sig and reg. This is what clearmap does #qc_overlay_transform_type = 'all' #both for auto+atlas and for sig+auto qc_overlay_transform_type = 'single' #don't consider reg with sig at all #%% ############################### #run, don't touch below ############################### df = pd.read_csv(path_to_csv_file) for i, row in df.iterrows(): #set up destination dst = os.path.join(row['dst'], dst_suffix) makedir(dst) #lightsheet package output folder fld = row['fld'] #path to mat file cell_centers_path = os.path.join(row['prefix_path_to_mat_file'], mat_file_suffix) print( '\n\n_________________\nFolder: {}\nDestination: {}\nMatfile: {}\n' .format(fld, dst, cell_centers_path)) #loads kwargs = load_kwargs(fld) regvol = [xx for xx in kwargs['volumes'] if xx.ch_type == 'regch'][0]
np.sum(xx[4:7]), np.sum(xx[7:10]), xx[10], xx[11], xx[12], np.sum(xx[13:16]) ] for xx in expr_all_as_frac_of_inj]) primary_pool = np.array([np.argmax(e) for e in frac_of_inj_pool]) #get n's after pooling primary_lob_n = np.array([ len(np.where(primary_pool == i)[0]) for i in range(max(primary_pool) + 1) ]) #normalization of inj site frac_of_inj_pool_norm = np.asarray( [brain / brain.sum() for brain in frac_of_inj_pool]) lr_brains = list(lr_dist.keys()) atl_dst = os.path.join(dst, "pma_to_aba") makedir(atl_dst) id_table = pd.read_excel(df_pth) #%% #------------------------------------------------------------------------------------------------------------------------------ #NOTE THAT ONLY HAVE TO DO THIS ONCE!!!! DO NOT NEED TO DO AGAIN UNLESS DOUBLE CHECKIHG #transform points to allen atlas space #get brains that we actually need to get cell counts from src = "/jukebox/wang/pisano/tracing_output/antero_4x_analysis/201903_antero_pooled_cell_counts_thalamus/transformed_points" post_transformed = [ os.path.join( src, os.path.join(xx, "transformed_points/posttransformed_zyx_voxels.npy")) for xx in lr_brains ] transformfiles = [
def pool_injections_for_analysis(**kwargs): inputlist = kwargs['inputlist'] dst = kwargs['dst']; makedir(dst) injscale = kwargs['injectionscale'] if 'injectionscale' in kwargs else 1 imagescale = kwargs['imagescale'] if 'imagescale' in kwargs else 1 axes = kwargs['reorientation'] if 'reorientation' in kwargs else ('0','1','2') cmap = kwargs['colormap'] if 'colormap' in kwargs else 'plasma' id_table = kwargs['id_table'] if 'id_table' in kwargs else '/jukebox/temp_wang/pisano/Python/lightsheet/supp_files/allen_id_table.xlsx' save_tif = kwargs['save_tif'] if 'save_tif' in kwargs else False num_sites_to_keep = kwargs['num_sites_to_keep'] if 'num_sites_to_keep' in kwargs else 1 nonzeros = [] ann = sitk.GetArrayFromImage(sitk.ReadImage(kwargs['annotation'])) if kwargs['crop']: ann = eval('ann{}'.format(kwargs['crop'])) allen_id_table=pd.read_excel(id_table) for i in range(len(inputlist)): impth = inputlist[i] animal = os.path.basename(os.path.dirname(os.path.dirname(os.path.dirname(impth)))) print('\n\n_______\n{}'.format(animal)) print(' loading:\n {}'.format(animal)) im = tifffile.imread(impth) if kwargs['crop']: im = eval('im{}'.format(kwargs['crop']))#; print im.shape #reorient to coronal? #segment arr = find_site(im, thresh=kwargs['threshold'], filter_kernel=kwargs['filter_kernel'], num_sites_to_keep=num_sites_to_keep)*injscale if save_tif: tifffile.imsave(os.path.join(dst,'{}'.format(os.path.dirname(impth))+'_inj.tif'), arr.astype('float32')) #optional 'save_individual' if kwargs['save_individual']: im = im*imagescale a=np.concatenate((np.max(im, axis=0), np.max(arr.astype('uint16'), axis=0)), axis=1) b=np.concatenate((np.fliplr(np.rot90(np.max(fix_orientation(im, axes=axes), axis=0),k=3)), np.fliplr(np.rot90(np.max(fix_orientation(arr.astype('uint16'), axes=axes), axis=0),k=3))), axis=1) plt.figure() plt.imshow(np.concatenate((b,a), axis=0), cmap=cmap, alpha=1); plt.axis('off') plt.savefig(os.path.join(dst,'{}'.format(animal)+'.pdf'), dpi=300, transparent=True) plt.close() #cell counts to csv print(' finding nonzero pixels for voxel counts...') nz = np.nonzero(arr) nonzeros.append(zip(*nz)) #<-for pooled image pos = transformed_pnts_to_allen_helper_func(np.asarray(zip(*[nz[2], nz[1], nz[0]])), ann) tdf = count_structure_lister(allen_id_table, *pos) if i == 0: df = tdf.copy() countcol = 'count' if 'count' in df.columns else 'cell_count' df.drop([countcol], axis=1, inplace=True) df[animal] = tdf[countcol] df.to_csv(os.path.join(dst,'voxel_counts.csv')) print('\n\nCSV file of cell counts, saved as {}\n\n\n'.format(os.path.join(dst,'voxel_counts.csv'))) #condense nonzero pixels nzs = [str(x) for xx in nonzeros for x in xx] #this list has duplicates if two brains had the same voxel w label c = Counter(nzs) array = np.zeros(im.shape) print('Collecting nonzero pixels for pooled image...') tick = 0 #generating pooled array where voxel value = total number of brains with that voxel as positive for k,v in c.iteritems(): k = [int(xx) for xx in k.replace('(','').replace(')','').split(',')] array[k[0], k[1], k[2]] = int(v) tick+=1 if tick % 50000 == 0: print(' {}'.format(tick)) #load atlas and generate final figure print('Generating final figure...') atlas = tifffile.imread(kwargs['atlas']) arr = fix_orientation(array, axes=axes) #cropping #if 'crop_atlas' not in kwargs: if kwargs['crop']: atlas = eval('atlas{}'.format(kwargs['crop'])) atlas = fix_orientation(atlas, axes=axes) my_cmap = eval('plt.cm.{}(np.arange(plt.cm.RdBu.N))'.format(cmap)) my_cmap[:1,:4] = 0.0 my_cmap = mpl.colors.ListedColormap(my_cmap) my_cmap.set_under('w') plt.figure() plt.imshow(np.max(atlas, axis=0), cmap='gray') plt.imshow(np.max(arr, axis=0), alpha=0.99, cmap=my_cmap); plt.colorbar(); plt.axis('off') dpi = int(kwargs['dpi']) if 'dpi' in kwargs else 300 plt.savefig(os.path.join(dst,'heatmap.pdf'), dpi=dpi, transparent=True); plt.close() print('Saved as {}'.format(os.path.join(dst,'heatmap.pdf'))) return df
def overlay_qc(args): #unpacking this way for multiprocessing fld, folder_suffix, output_folder, verbose, doubletransform, make_volumes = args try: #get 3dunet cell dataframe csv file input_csv = listdirfull(os.path.join(fld, folder_suffix), ".csv") assert len(input_csv) == 1, "multiple csv files" dataframe = pd.read_csv(input_csv[0]) #location to save out dst = os.path.join(output_folder, os.path.basename(fld)); makedir(dst) #EXAMPLE USING LIGHTSHEET - assumes marking centers in the "raw" full sized cell channel. This will transform those #centers into "atlas" space (in this case the moving image) #in this case the "inverse transform has the atlas as the moving image in the first step, #and the autofluorescence channel as the moving image in the second step #NOTE - it seems that the registration of cell to auto is failing on occasion....thus get new files... ################################ cell_inverse_folder = listdirfull(os.path.join(fld, "elastix_inverse_transform"), "cellch")[0] a2r = listall(cell_inverse_folder, "atlas2reg_TransformParameters"); a2r.sort() r2s = listall(cell_inverse_folder, "reg2sig_TransformParameters"); r2s.sort() #possibly remove #IMPORTANT. the idea is to apply cfos->auto->atlas transformfiles = r2s + a2r if doubletransform else a2r #might get rid of r2s lightsheet_parameter_dictionary = os.path.join(fld, "param_dict.p") converted_points = generate_transformed_cellcount(dataframe, dst, transformfiles, lightsheet_parameter_dictionary, verbose=verbose) #load and convert to single voxel loc zyx = np.asarray([str((int(xx[0]), int(xx[1]), int(xx[2]))) for xx in np.nan_to_num(np.load(converted_points))]) from collections import Counter zyx_cnt = Counter(zyx) #check... if make_volumes: #manually call transformix kwargs = load_dictionary(lightsheet_parameter_dictionary) vol = [xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0].resampled_for_elastix_vol transformed_vol = os.path.join(dst, "transformed_volume"); makedir(transformed_vol) if not doubletransform: transformfiles = [os.path.join(fld, "elastix/TransformParameters.0.txt"), os.path.join(fld, "elastix/TransformParameters.1.txt")] transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files transformix_command_line_call(vol, transformed_vol, transformfiles[-1]) else: v=[xx for xx in kwargs["volumes"] if xx.ch_type == "cellch"][0] #sig to reg tps = [listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.0")[0], listall(os.path.dirname(v.ch_to_reg_to_atlas), "/TransformParameters.1")[0]] #reg to atlas transformfiles = tps+[os.path.join(fld, "elastix/TransformParameters.0.txt"), os.path.join(fld, "elastix/TransformParameters.1.txt")] transformfiles = modify_transform_files(transformfiles, transformed_vol) #copy over elastix files transformix_command_line_call(vol, transformed_vol, transformfiles[-1]) #cell_registered channel cell_reg = tifffile.imread(os.path.join(transformed_vol, "result.tif")) tifffile.imsave(os.path.join(transformed_vol, "result.tif"), cell_reg, compress=1) cell_cnn = np.zeros_like(cell_reg) tarr = []; badlist=[] for zyx,v in zyx_cnt.items(): z,y,x = [int(xx) for xx in zyx.replace("(","",).replace(")","").split(",")] tarr.append([z,y,x]) try: cell_cnn[z,y,x] = v*100 except: badlist.append([z,y,x]) #apply x y dilation r = 2 selem = ball(r)[int(r/2)] cell_cnn = cell_cnn.astype("uint8") cell_cnn = np.asarray([cv2.dilate(cell_cnn[i], selem, iterations = 1) for i in range(cell_cnn.shape[0])]) tarr=np.asarray(tarr) if len(badlist)>0: print("{} errors in mapping with cell_cnn shape {}, each max dim {}, \npossibly due to a registration overshoot \ or not using double transform\n\n{}".format(len(badlist), cell_cnn.shape, np.max(tarr,0), badlist)) merged = np.stack([cell_cnn, cell_reg, np.zeros_like(cell_reg)], -1) tifffile.imsave(os.path.join(transformed_vol, "merged.tif"), merged)#, compress=1) #out = np.concatenate([cell_cnn, cell_reg, ], 0) #####check at the resampled for elastix phase before transform...this mapping looks good... if make_volumes: #make zyx numpy arry zyx = dataframe[["z","y","x"]].values kwargs = load_dictionary(lightsheet_parameter_dictionary) vol = [xx for xx in kwargs["volumes"] if xx.ch_type =="cellch"][0] fullsizedimensions = get_fullsizedims_from_kwargs(kwargs) #don"t get from kwargs["volumes"][0].fullsizedimensions it"s bad! use this instead zyx = fix_contour_orientation(zyx, verbose=verbose, **kwargs) #now in orientation of resample zyx = points_resample(zyx, original_dims = fix_dimension_orientation(fullsizedimensions, **kwargs), resample_dims = tifffile.imread(vol.resampled_for_elastix_vol).shape, verbose = verbose)[:, :3] #cell channel cell_ch = tifffile.imread(vol.resampled_for_elastix_vol) cell_cnn = np.zeros_like(cell_ch) tarr = []; badlist=[] for _zyx in zyx: z,y,x = [int(xx) for xx in _zyx] tarr.append([z,y,x]) try: cell_cnn[z,y,x] = 100 except: badlist.append([z,y,x]) tarr=np.asarray(tarr) merged = np.stack([cell_cnn, cell_ch, np.zeros_like(cell_ch)], -1) tifffile.imsave(os.path.join(transformed_vol, "resampled_merged.tif"), merged)#, compress=1) except Exception as e: print(e) with open(error_file, "a") as err_fl: err_fl.write("\n\n{} {}\n\n".format(fld, e))
except Exception as e: print(e) with open(error_file, "a") as err_fl: err_fl.write("\n\n{} {}\n\n".format(fld, e)) if __name__ == "__main__": #goal is to transform cooridnates, voxelize based on number of cells and overlay with reigstered cell signal channel... #set paths and make folders dst = "/jukebox/wang/zahra/h129_qc" input_folder = "/jukebox/wang/pisano/tracing_output/antero_4x/" folder_suffix = "3dunet_output/pooled_cell_measures" #thalamus output_folder = os.path.join(dst, "rtn_thal_transformed_points"); makedir(output_folder) qc_folder = os.path.join(dst, "rtn_thal_qc"); makedir(qc_folder) error_file = os.path.join(qc_folder, "errors.txt") #inputs brains = ["20160622_db_bl6_crii_52hr_01", "20160622_db_bl6_unk_01", "20160801_db_cri_02_1200rlow_52hr", "20160801_db_l7_cri_01_mid_64hr", "20160822_tp_bl6_crii_250r_01", "20160822_tp_bl6_crii_1250r_05", "20160822_tp_bl6_crii_1500r_06", "20160823_tp_bl6_cri_250r_01", "20160823_tp_bl6_cri_500r_02", "20160912_tp_bl6_lob7_750r_04", "20160916_tp_lob6_ml_01", "20160916_tp_lob6_ml_04", "20160916_tp_lob7_250r_05", "20160920_tp_bl6_lob7_250r_02", "20160920_tp_bl6_lob7_500r_03", "20160920_tp_bl6_lob7_ml_01", "20161129_db_bl6_lob6b_ml50r_53hr", "20161201_db_bl6_lob6b_500r_53d5hr", "20161203_tp_bl6_crii_250r_08", "20161203_tp_bl6_crii_750r_09", "20161203_tp_bl6_lob7_500r_05", "20161203_tp_bl6_lob7_1000r_06", "20161203_tp_bl6_lob7_1500r_07", "20161203_tp_bl6_lob7_ml_04", "20161205_tp_bl6_sim_250r_02", "20161205_tp_bl6_sim_750r_03", "20161205_tp_bl6_sim_1250r_04", "20161205_tp_bl6_sim_1750r_05","20161207_db_bl6_lob6a_500r_53hr", "20161207_db_bl6_lob6a_850r_53hr", "20161208_db_bl6_cri_50r_pv_53hr","20170115_tp_bl6_lob6b_500r_05", "20170115_tp_bl6_lob6b_1000r_06","20170116_tp_bl6_lob6b_lpv_07","20170130_tp_bl6_sim_rlat_05", "20170207_db_bl6_crii_1300r_02", "20170207_db_bl6_crii_rlat_03","20170212_tp_bl6_lob8_ml_05",
def ovly_3d(ch, svlc, tplst, valid_plns, outdr, vol_to_process, resizefactor, contour_class_lst, multiplane, ovly, core, cores): '''function to apply independent colors to contours that have been detected on multiple planes, and save images. This has been parallelized and calculates a range based on the number of cores. ''' ####assign random color to each group of contours, the break up groups, and sort by plane #use this for looking at multi contours only if multiplane != False: multilist = [x for x in tplst if len(x.plns) > 1] tplst = multilist contour_color_lst = [] for contours in tplst: color = (random.randint(50, 255), random.randint(50, 255), random.randint(50, 255)) for plns in contours.plns.itervalues(): contour_color_lst.append([plns, color]) ###break apart list #sort by z try: #sort by z contour_color_lst.sort() except ValueError: #done to suppress error pass ############################################load find full size files############################################ makedir(svlc) dr = vol_to_process.full_sizedatafld_vol #dr='/home/wanglab/wang/pisano/tracing_output/H129_EGFP_NLS_vc22_01_lob5_intensityrescale/full_sizedatafld/ch01' fl = [f for f in os.listdir(dr) if '.tif' in f] reg = re.compile(r'(.*C+)(?P<ch>[0-9]{1,2})(.*Z+)(?P<z>[0-9]{1,4})(.tif)' ) ###regex used since code changes file format matches = map(reg.match, fl) ##find index of z,y,x,ch in matches z_indx = matches[0].span('z') ####make dictionary where each pln is a key, values are all point that should be drawn zdct = {} for i in range(len(contour_color_lst)): pln = contour_color_lst[i] ##check try: zdct[str(pln[0][0]).zfill(4)].append(pln) except KeyError: zdct[str(pln[0][0]).zfill(4)] = [] zdct[str(pln[0][0]).zfill(4)].append(pln) ############parse jobs: chnkrng = chunkit(core, cores, zdct) ########### apply points to each pln if ovly == True: ###ovly 3d contours onto data flpth = ''.join(matches[0].groups()) y, x = tifffile.imread(os.path.join(dr, flpth)).shape dsf = resizefactor for i in range(chnkrng[0], chnkrng[1]): try: pln = zdct.keys()[i] cntrs = zdct[zdct.keys()[i]] ovly_helper(flpth, z_indx, dr, svlc, x, y, dsf, pln, cntrs) del cntrs, pln gc.collect() except: pass ###make zpln range and make empty planes on the zplns where no contours are found try: plnlst = [int(plns) for plns in zdct.iterkeys()] plnrng = range(min(plnlst), max(plnlst) + 1) nocontour_plns = set(plnrng).difference(set(plnlst)) ############parse jobs: chnkrng = chunkit(core, cores, nocontour_plns) print( '\n\ncontours not found on planes: {}\nmaking empty images for those planes...' .format(nocontour_plns)) for i in range(chnkrng[0], chnkrng[1]): pln = list(nocontour_plns)[i] blnk_ovly_helper(flpth, z_indx, dr, svlc, x, y, dsf, pln) del pln gc.collect() return except ValueError: #no contours found print('No contour found in ch{}, not making color stack...'.format( ch)) return elif ovly == False: ###make downsized 3d contours flpth = ''.join(matches[0].groups()) y, x = tifffile.imread(os.path.join(dr, flpth)).shape dsf = resizefactor for i in range(chnkrng[0], chnkrng[1]): try: pln = zdct.keys()[i] cntrs = zdct[zdct.keys()[i]] no_ovly_helper(flpth, z_indx, dr, svlc, x, y, dsf, pln, cntrs) del cntrs, pln gc.collect() except: pass ###make zpln range and make empty planes on the zplns where no contours are found try: plnlst = [int(plns) for plns in zdct.iterkeys()] plnrng = range(min(plnlst), max(plnlst) + 1) nocontour_plns = set(plnrng).difference(set(plnlst)) if len(nocontour_plns) != 0: print( '\n\ncontours not found on planes: {}\nmaking empty images for those planes...' .format(nocontour_plns)) ############parse jobs: chnkrng = chunkit(core, cores, nocontour_plns) for i in range(chnkrng[0], chnkrng[1]): try: pln = list(nocontour_plns)[i] blnk_helper(flpth, z_indx, dr, svlc, x, y, dsf, pln) del pln gc.collect() except IndexError: pass return except ValueError: #no contours found print('No contour found in ch{}, not making color stack...'.format( ch)) return
def identify_structures_w_contours(jobid, cores=5, make_color_images=False, overlay_on_original_data=False, consider_only_multipln_contours=False, **kwargs): '''function to take 3d detected contours and apply elastix transform ''' #######################inputs and setup################################################# ###inputs kwargs = load_kwargs(**kwargs) outdr = kwargs['outputdirectory'] vols = kwargs['volumes'] reg_vol = [xx for xx in vols if xx.ch_type == 'regch'][0] ###get rid of extra jobs if jobid >= len([xx for xx in vols if xx.ch_type != 'regch' ]): ###used to end jobs if too many are called print('jobid({}) >= volumes {}'.format( jobid, len([xx for xx in vols if xx.ch_type != 'regch']))) return ###volumes to process: each job represents a different contour volume vol_to_process = [xx for xx in vols if xx.ch_type != 'regch'][jobid] ch = vol_to_process.channel print(vol_to_process.ch_type) #find appropriate folders for contours if vol_to_process.ch_type == 'cellch': detect3dfld = reg_vol.celldetect3dfld coordinatesfld = reg_vol.cellcoordinatesfld elif vol_to_process.ch_type == 'injch': detect3dfld = reg_vol.injdetect3dfld coordinatesfld = reg_vol.injcoordinatesfld #set scale and atlas xscl, yscl, zscl = reg_vol.xyz_scale ###micron/pixel zmx, ymx, xmx = reg_vol.fullsizedimensions AtlasFile = reg_vol.atlasfile print('Using {} CORES'.format(cores)) try: p except NameError: p = mp.Pool(cores) resizefactor = kwargs['resizefactor'] brainname = reg_vol.brainname ############################################################################################################ #######################use regex to sort np files by ch and then by zpln#################################### ############################################################################################################ fl = [f for f in os.listdir(detect3dfld) if '.p' in f and 'ch' in f] #sorted for raw files reg = re.compile(r'(.*h+)(?P<ch>\d{2})(.*)(.p)') matches = map(reg.match, fl) ##load .np files sys.stdout.write( '\njobid({}), loading ch{} .p files to extract contour_class objects....' .format(jobid, ch)) contour_class_lst = [] for fl in [ os.path.join(detect3dfld, ''.join(xx.groups())) for xx in matches if xx.group('ch')[-2:] in ch ]: tmpkwargs = {} pckl = open(fl, 'rb') tmpkwargs.update(pickle.load(pckl)) pckl.close() if consider_only_multipln_contours == False: tmplst = tmpkwargs['single'] [tmplst.append(xx) for xx in tmpkwargs['multi']] elif consider_only_multipln_contours == True: tmplst = tmpkwargs['multi'] [contour_class_lst.append(xx) for xx in tmplst] sys.stdout.write('\ndone loading contour_class objects.\n') #check for successful loading if len(contour_class_lst) == 0: print('Length of contours in ch{} was {}, ending process...'.format( jobid, len(contour_class_lst))) try: p.terminate() except: 1 return ############################################################################################################ ##############################make color files############################################################## ############################################################################################################ if make_color_images == True: sys.stdout.write('\nmaking 3d planes...') sys.stdout.flush() valid_plns = range(0, zmx + 1) svlc = os.path.join(outdr, 'ch{}_3dcontours'.format(ch)) removedir(svlc) if overlay_on_original_data == False: ovly = False elif overlay_on_original_data == True: ovly = True iterlst = [] [ iterlst.append( (ch, svlc, contour_class_lst, valid_plns, outdr, vol_to_process, resizefactor, contour_class_lst, consider_only_multipln_contours, ovly, core, cores)) for core in range(cores) ] p.starmap(ovly_3d, iterlst) lst = os.listdir(svlc) lst1 = [os.path.join(svlc, xx) for xx in lst] lst1.sort() del lst del iterlst ###load ims and return dct of keys=str(zpln), values=np.array sys.stdout.write( '\n3d planes made, saved in {},\nnow compressing into single tifffile' .format(svlc)) imstack = tifffile.imread(lst1) del lst1 if len(imstack.shape) > 3: imstack = np.squeeze(imstack) try: ###check for orientation differences, i.e. from horiztonal scan to sagittal for atlas registration imstack = np.swapaxes(imstack, *kwargs['swapaxes']) except: pass tiffstackpth = os.path.join( outdr, '3D_contours_ch{}_{}'.format(ch, brainname)) tifffile.imsave(tiffstackpth, imstack.astype('uint16')) del imstack gc.collect() shutil.rmtree(svlc) sys.stdout.write('\ncolor image stack made for ch{}'.format(ch)) else: sys.stdout.write('\nmake_color_images=False, not creating images') ############################################################################################################ ######################apply point transform and make transformix input file################################# ############################################################################################################ ###find centers and add 1's to make nx4 array for affine matrix multiplication to account for downsizing ###everything is in PIXELS contourarr = np.empty((len(contour_class_lst), 3)) for i in range(len(contour_class_lst)): contourarr[i, ...] = contour_class_lst[ i].center ###full sized dimensions: if 3x3 tiles z(~2000),y(7680),x(6480) before any rotation try: contourarr = swap_cols( contourarr, *kwargs['swapaxes'] ) ###change columns to account for orientation changes between brain and atlas: if horizontal to sagittal==>x,y,z relative to horizontal; zyx relative to sagittal z, y, x = swap_cols(np.array([ vol_to_process.fullsizedimensions ]), *kwargs['swapaxes'])[ 0] ##convert full size cooridnates into sagittal atlas coordinates sys.stdout.write('\nSwapping Axes') except: ###if no swapaxes then just take normal z,y,x dimensions in original scan orientation z, y, x = vol_to_process.fullsizedimensions sys.stdout.write('\nNo Swapping of Axes') d1, d2 = contourarr.shape nx4centers = np.ones((d1, d2 + 1)) nx4centers[:, :-1] = contourarr ###find resampled elastix file dim print(os.listdir(outdr)) print([xx.channel for xx in vols if xx.ch_type == 'regch']) with tifffile.TiffFile([ os.path.join(outdr, f) for f in os.listdir(outdr) if 'resampledforelastix' in f and 'ch{}'.format( [xx.channel for xx in vols if xx.ch_type == 'regch'][0]) in f ][0]) as tif: zr = len(tif.pages) yr, xr = tif.pages[0].shape tif.close() ####create transformmatrix trnsfrmmatrix = np.identity(4) * ( zr / z, yr / y, xr / x, 1) ###downscale to "resampledforelastix size" sys.stdout.write('trnsfrmmatrix:\n{}\n'.format(trnsfrmmatrix)) #nx4 * 4x4 to give transform trnsfmdpnts = nx4centers.dot(trnsfrmmatrix) ##z,y,x sys.stdout.write('first three transformed pnts:\n{}\n'.format( trnsfmdpnts[0:3])) #create txt file, with elastix header, then populate points txtflnm = '{}_zyx_transformedpnts_ch{}.txt'.format(brainname, ch) pnts_fld = os.path.join(outdr, 'transformedpoints_pretransformix') makedir(pnts_fld) transforminput = os.path.join(pnts_fld, txtflnm) removedir(transforminput) ###prevent adding to an already made file writer(pnts_fld, 'index\n{}\n'.format(len(trnsfmdpnts)), flnm=txtflnm) sys.stdout.write( '\nwriting centers to transfomix input points text file: {}....'. format(transforminput)) stringtowrite = '\n'.join([ '\n'.join(['{} {} {}'.format(i[2], i[1], i[0])]) for i in trnsfmdpnts ]) ####this step converts from zyx to xyz***** writer(pnts_fld, stringtowrite, flnm=txtflnm) #[writer(pnts_fld, '{} {} {}\n'.format(i[2],i[1],i[0]), flnm=txtflnm, verbose=False) for i in trnsfmdpnts] ####this step converts from zyx to xyz***** sys.stdout.write('...done writing centers.') sys.stdout.flush() del trnsfmdpnts, trnsfrmmatrix, nx4centers, contourarr gc.collect() ############################################################################################################ ####################################elastix for inverse transform########################################### ############################################################################################################ transformfile = make_inverse_transform(vol_to_process, cores, **kwargs) assert os.path.exists(transformfile) sys.stdout.write( '\n***Elastix Inverse Transform File: {}***'.format(transformfile)) ############################################################################################################ ####################################transformix############################################################# ############################################################################################################ if make_color_images != False: #apply transform to 3d_tiffstack: transformimageinput = tiffstackpth elastixpth = os.path.join(outdr, 'elastix') trnsfrm_outpath = os.path.join( elastixpth, '3D_contours_ch{}_{}'.format(ch, brainname)) makedir(trnsfrm_outpath) writer(trnsfrm_outpath, '\nProcessing ch{} 3D...'.format(ch)) #transformfiles=[os.path.join(elastixpth, xx) for xx in os.listdir(os.path.join(outdr, 'elastix')) if "TransformParameters" in xx]; mxx=max([xx[-5] for xx in transformfiles]) #transformfile=os.path.join(elastixpth, 'TransformParameters.{}.txt'.format(mxx)) trnsfrm_out_file = os.path.join(trnsfrm_outpath, 'result.tif') #output of transformix transformimageinput_resized = transformimageinput[:-4] + '_resampledforelastix.tif' print('Resizing {}'.format(transformimageinput_resized)) resample_par(cores, transformimageinput, AtlasFile, svlocname=transformimageinput_resized, singletifffile=True, resamplefactor=1.7) sp.call([ 'transformix', '-in', transformimageinput_resized, '-out', trnsfrm_outpath, '-tp', transformfile ]) writer(trnsfrm_outpath, '\n Transformix File Generated: {}'.format(trnsfrm_out_file)) writer( trnsfrm_outpath, '\n Passing colorcode: {} file as {}'.format( ch, os.path.join(trnsfrm_outpath, 'depthcoded.png'))) ###depth coded image of transformix result; not functional yet #depth.colorcode(trnsfrm_out_file, trnsfrm_outpath) #getvoxels(trnsfrm_out_file, os.path.join(trnsfrm_outpath, 'zyx_voxels_{}.npy'.format(ch))) #allen_compare(AtlasFile, svlc, trnsfrm_outpath) ##if successful delete contour cooridnates and maybe contourdetect3d flds ############################################################ ##############apply transform to points##################### elastixpth = os.path.join(outdr, 'elastix_inverse_transform') trnsfrm_outpath = os.path.join(elastixpth, 'ch{}_3dpoints'.format(ch)) makedir(trnsfrm_outpath) writer(trnsfrm_outpath, '\n***********Starting Transformix for: {}***********'.format(ch)) sys.stdout.flush() #transformfiles=[os.path.join(elastixpth, xx) for xx in os.listdir(os.path.join(outdr, 'elastix')) if "TransformParameters" in xx]; mxx=max([xx[-5] for xx in transformfiles]) #transformfile=os.path.join(elastixpth, 'TransformParameters.{}.txt'.format(mxx)) trnsfrm_out_file = os.path.join( trnsfrm_outpath, 'outputpoints.txt') #MIGHT NEED TO CHANGE THIS sp.call([ 'transformix', '-def', transforminput, '-out', trnsfrm_outpath, '-tp', transformfile ]) #sp.call(['transformix', '-def', 'all', '-out', trnsfrm_outpath, '-tp', transformfile]) ##displacement field writer(trnsfrm_outpath, '\n Transformix File Generated: {}'.format(trnsfrm_out_file)) #################################################################################### ##############generate list and image overlaid onto allen atlas##################### #################################################################################### name = 'job{}_{}'.format(jobid, vol_to_process.ch_type) transformed_pnts_to_allen(trnsfrm_out_file, ch, cores, name=name, **kwargs) writer(outdr, '*************STEP 5*************\n Finished') print('end of script') try: p.terminate() except: 1 return
from tools.utils.io import listdirfull, makedir, load_memmap_arr, load_np, listall, load_kwargs from skimage.external import tifffile import skimage from tools.utils.directorydeterminer import directorydeterminer from scipy.ndimage.interpolation import zoom from tools.registration.transform_cell_counts import generate_transformed_cellcount, get_fullsizedims_from_kwargs, points_resample from tools.registration.transform_list_of_points import modify_transform_files from tools.imageprocessing.orientation import fix_contour_orientation, fix_dimension_orientation import matplotlib.gridspec as gridspec from tools.conv_net.functions.dilation import dilate_with_element from skimage.morphology import ball if __name__ == '__main__': #set up dst = '/home/wanglab/wang/pisano/tracing_output/qc/antero_no_jg'; makedir(dst) lst = [xx for xx in listdirfull('/home/wanglab/wang/pisano/tracing_output/antero_4x') if 'jg' not in os.path.basename(xx)] dst = '/home/wanglab/wang/pisano/tracing_output/qc/antero_only_jg'; makedir(dst) lst = [xx for xx in listdirfull('/home/wanglab/wang/pisano/tracing_output/antero_4x') if 'jg' in os.path.basename(xx)] cnn_transform_type = 'affine_only_reg_to_sig';#both for regwatlas, and only affine for sig adn reg #'all', 'single': don't consider reg with sig at all volume_transform_type = 'single';#both for regwatlas, and only affine for sig adn reg #'all', 'single': don't consider reg with sig at all verbose = True generate_registered_overlay = False #this looks bad generate_downsized_overlay = True #this looks better #fld = '/home/wanglab/wang/pisano/tracing_output/antero_4x/20170130_tp_bl6_sim_1750r_03' #loop for fld in lst: try: print fld
brain = os.path.join(src, brains[jobid]) start = time.time() #accessing parameter dictionary cellvol = "/jukebox/wang/pisano/tracing_output/bl6_ts/20150804_tp_bl6_ts04/full_sizedatafld/20150804_tp_bl6_ts04_488w_647_z3um_250msec_1hfds_ch01" a2r0 = "/jukebox/scratch/zmd/save/contra_ipsi_projection_studies_20191125/20150804_tp_bl6_ts04/elastix_inverse_transform/TransformParameters.0.txt" a2r1 = "/jukebox/scratch/zmd/save/contra_ipsi_projection_studies_20191125/20150804_tp_bl6_ts04/elastix_inverse_transform/TransformParameters.1.txt" # r2s0 = "/jukebox/wang/pisano/tracing_output/bl6_ts/20150804_tp_bl6_ts04/elastix_inverse_transform/cellch_20150804_tp_bl6_ts04_555_z3um_70msec_3hfds/20150804_tp_bl6_ts04_555_z3um_70msec_3hfds_resized_ch00_resampledforelastix_atlas2reg2sig/reg2sig_TransformParameters.0.txt" # r2s1 = "/jukebox/LightSheetTransfer/tp/20200701_12_55_28_20170207_db_bl6_crii_rpv_01/elastix_inverse_transform/TransformParameters.1.txt" #set destination directory braindst = os.path.join(scratch_dir, os.path.basename(brain)) makedir(braindst) aldst = os.path.join(braindst, "transformed_annotations") makedir(aldst) #transformix # transformfiles = modify_transform_files(transformfiles=[a2r0, a2r1, r2s0], dst = aldst) transformfiles = modify_transform_files(transformfiles=[a2r0, a2r1], dst=aldst) [change_interpolation_order(xx, 0) for xx in transformfiles] #change the parameter in the transform files that outputs 16bit images instead for fl in transformfiles: # Read in the file with open(fl, "r") as file: filedata = file.read() # Replace the target string
secondary = np.array([np.argsort(e)[-2] for e in expr_all_as_frac_of_inj]) #pooled injections ak_pool = np.array(["Lob. I-III, IV-V", "Lob. VIa, VIb, VII", "Lob. VIII, IX, X", "Simplex", "Crus I", "Crus II", "PM, CP"]) frac_of_inj_pool = np.array([[np.sum(xx[:4]),np.sum(xx[4:7]),np.sum(xx[7:10]), xx[10], xx[11], xx[12], np.sum(xx[13:16])] for xx in expr_all_as_frac_of_inj]) primary_pool = np.array([np.argmax(e) for e in frac_of_inj_pool]) #get n's after pooling primary_lob_n = np.array([len(np.where(primary_pool == i)[0]) for i in range(max(primary_pool)+1)]) #normalization of inj site frac_of_inj_pool_norm = np.asarray([brain/brain.sum() for brain in frac_of_inj_pool]) #%% lr_brains = list(lr_dist.keys()) atl_dst = os.path.join(dst, "pma_to_aba"); makedir(atl_dst) id_table = pd.read_excel(df_pth) #------------------------------------------------------------------------------------------------------------------------------ #NOTE THAT ONLY HAVE TO DO THIS ONCE!!!! DO NOT NEED TO DO AGAIN UNLESS DOUBLE CHECKIHG #transform points to allen atlas space #get brains that we actually need to get cell counts from src = "/jukebox/wang/pisano/tracing_output/antero_4x_analysis/201903_antero_pooled_cell_counts_thalamus/transformed_points" post_transformed = [os.path.join(src, os.path.join(xx, "transformed_points/posttransformed_zyx_voxels.npy")) for xx in lr_brains] transformfiles = ["/jukebox/wang/zahra/aba_to_pma/TransformParameters.0.txt", "/jukebox/wang/zahra/aba_to_pma/TransformParameters.1.txt"] #collect # for fl in post_transformed: # arr = np.load(fl)