def join_results_from_cluster_helper(source, x=all, y=all, z=all, sink=None, chunkSizeMax=100, chunkSizeMin=30, chunkOverlap=15, function=noProcessing, join=joinPoints, verbose=False, **parameter): #calc num of substacks subStacks = calculateSubStacks(source, x=x, y=y, z=z, processes=1, chunkSizeMax=chunkSizeMax, chunkSizeMin=chunkSizeMin, chunkOverlap=chunkOverlap, chunkOptimization=False, verbose=verbose) #load all cell detection job results if type(sink) == tuple: pckfld = os.path.join(sink[0][:sink[0].rfind("/")], "cell_detection") makedir(pckfld) elif type(sink) == str: pckfld = os.path.join(sink[:sink.rfind("/")], "cell_detection") makedir(pckfld) results = [] fls = listdirfull(pckfld) fls.sort() for fl in fls: if "~" not in fl: with open(fl, "rb") as pckl: results.append(pickle.load(pckl)) pckl.close() print(results[0]) #reformat results = [xx for yy in results for xx in yy] #join the results print("Length of results: {}".format(len(results))) results = join(results, subStacks=subStacks, **parameter) #write / or return return io.writePoints(sink, results)
def sweep_parameters_cluster(jobid, rBP_size_r, fEMP_hmax_r, fEMP_size_r, fEMP_threshold_r, fIP_method_r, fIP_size_r, dCSP_threshold_r, tick, optimization_chunk=4, pth=False, rescale=False, cleanup=True, **kwargs): """Function to sweep parameters final outputs will be saved in outputdirectory/parameter_sweep second copy will be saved in outputdirectory/parameter_sweep_jobid if cleanup=False Inputs: ---------------- jobid: chunk of tissue to run (usually int between 20-30) #pth (optional): if pth to output folder after running package, function will load the param file automatically rescale (optional): str of dtype to rescale to. E.g.: "uint8" cleanup = T/F removes subfolders after optimization_chunk = this was the old "jobid" in this case it is the chunk of volume to look at kwargs (if not pth): "params" from run_clearmap_cluster.py """ # make folder for final output: opt = kwargs["outputdirectory"] makedir(opt) out = opt + "/parameter_sweep" makedir(out) out0 = opt + "/parameter_sweep_jobid_{}".format(str(jobid).zfill(4)) makedir(out0) rBP_size, fEMP_hmax, fEMP_size, fEMP_threshold, fIP_method, fIP_size, dCSP_threshold = [ (rBP_size, fEMP_hmax, fEMP_size, fEMP_threshold, fIP_method, fIP_size, dCSP_threshold) for rBP_size, fEMP_hmax, fEMP_size, fEMP_threshold, fIP_method, fIP_size, dCSP_threshold in product( rBP_size_r, fEMP_hmax_r, fEMP_size_r, fEMP_threshold_r, fIP_method_r, fIP_size_r, dCSP_threshold_r) ][jobid] pth = out0 + "/parametersweep_rBP_size{}_fEMP_hmax{}_fEMP_size{}_fEMP_threshold{}_fIP_method{}_fIP_size{}_dCSP_threshold{}.tif".format( rBP_size, fEMP_hmax, fEMP_size, fEMP_threshold, fIP_method, fIP_size, dCSP_threshold) if not os.path.exists(pth): try: # set params for sweep # Remove the background with morphological opening (optimised for spherical objects), e.g. (7,7) kwargs["removeBackgroundParameter_size"] = (rBP_size, rBP_size) # (float or None) h parameter (for instance 20) for the initial h-Max transform, if None, do not perform a h-max transform kwargs["findExtendedMaximaParameter_hmax"] = fEMP_hmax # size in pixels (x,y) for the structure element of the morphological opening kwargs["findExtendedMaximaParameter_size"] = fEMP_size # (float or None) include only maxima larger than a threshold, if None keep all local maxima kwargs["findExtendedMaximaParameter_threshold"] = fEMP_threshold # (str, func, None) method to use to determine intensity (e.g. "Max" or "Mean") if None take intensities at the given pixels kwargs["findIntensityParameter_method"] = fIP_method # (tuple) size of the search box on which to perform the *method* kwargs["findIntensityParameter_size"] = (fIP_size, fIP_size, fIP_size) # (float or None) threshold to determine mask. Pixels below this are background if None no mask is generated kwargs["detectCellShapeParameter_threshold"] = dCSP_threshold # tmp nkwargs = load_kwargs(kwargs["outputdirectory"]) kwargs["outputdirectory"] = out0 nkwargs.update(kwargs) pckloc = out0 + "/param_dict.p" pckfl = open(pckloc, "wb") pickle.dump(nkwargs, pckfl) pckfl.close() # run cell detection sys.stdout.write( "\n\n\n *****Iteration {} of {}*****\n\n\n".format( jobid, tick)) sys.stdout.write( " Iteration parameters: {} {} {} {} {} {} {}" .format(kwargs["removeBackgroundParameter_size"], kwargs["findExtendedMaximaParameter_hmax"], kwargs["findExtendedMaximaParameter_size"], kwargs["findExtendedMaximaParameter_threshold"], kwargs["findIntensityParameter_method"], kwargs["findIntensityParameter_size"], kwargs["detectCellShapeParameter_threshold"])) celldetection_operations(optimization_chunk, testing=True, **kwargs) # list, load, and maxip raw = [ xx for xx in listdirfull(out0 + "/optimization/raw") if "~" not in xx and ".db" not in xx ] raw.sort() raw_im = np.squeeze(tifffile.imread(raw)) raw_mx = np.max(raw_im, axis=0) bkg = [ xx for xx in listdirfull(out0 + "/optimization/background") if "~" not in xx and "Thumbs.db" not in xx ] bkg.sort() bkg_im = tifffile.imread(bkg) bkg_mx = np.max(bkg_im, axis=0) cell = [ xx for xx in listdirfull(out0 + "/optimization/cell") if "~" not in xx and ".db" not in xx ] cell.sort() cell_im = tifffile.imread(cell) cell_mx = np.max(cell_im, axis=0) # optional rescale: if rescale: raw_mx = rescale_intensity(raw_mx, in_range=str(raw_mx.dtype), out_range=rescale).astype(rescale) bkg_mx = rescale_intensity(bkg_mx, in_range=str(bkg_mx.dtype), out_range=rescale).astype(rescale) cell_mx = rescale_intensity(cell_mx, in_range=str(cell_mx.dtype), out_range=rescale).astype(rescale) # concatenate and save out: bigim = np.concatenate((raw_mx, bkg_mx, cell_mx), axis=1) del bkg, bkg_im, bkg_mx, cell, cell_im, cell_mx if cleanup: removedir(out0) if not cleanup: tifffile.imsave(pth, bigim, compress=1) #save in main npth = out + "/jobid_{}_rBPSize{}_fEMPHmax{}_fEMPSize{}_fEMPThreshold{}_fIPMethod{}_fIPSize{}_dCSPThreshold{}.tif".format( str(jobid).zfill(4), rBP_size, fEMP_hmax, fEMP_size, fEMP_threshold, fIP_method, fIP_size, dCSP_threshold) tifffile.imsave(npth, bigim.astype("uint16"), compress=1) except Exception as e: print("Error on: {}\n\nerror={}".format(pth, e)) im = np.zeros((10, 10, 10)) tifffile.imsave(pth, im, compress=1) with open(os.path.join(out, "errored_files.txt"), "a") as fl: fl.write("\n\n{}\n{}\n".format(pth, kwargs)) fl.close return
def get_parameter_sweep_results(args, dst, resize=False, plot=False, hillshade=False, concatenate=False): ''' If passing several parameter sweeps results from different brains this assumes that each had the same sweep parameters. Inputs ---------- args = list of folders that are output of sweep_parameters function dst = pth to save folder resize = (optional; int) factor to resample by. smaller<1<larger plot (optional): save matplotlib figure hillshade (optional), if true apply hillshading concatenate (optional), if true, concatenate all into a single tifffile Outputs ---------- combined tiff of images ''' removedir(dst) makedir(dst) #get list of sublist of fls lst = [] for pth in args: fls = listdirfull(pth + '/optimization/parameter_sweep') fls.sort() lst.append(fls) if not resize: resize = 1 if tiff_stacks: tiff_stacks = dst + '/tiff_stacks' makedir(tiff_stacks) #load and concatenate files: sys.stdout.write('Loading files....') sys.stdout.flush() for xx in range(len(lst[0])): for yy in range(len(lst)): im = zoom(tifffile.imread(lst[yy][xx]), resize) if tiff_stacks: tifffile.imsave( tiff_stacks + lst[yy][xx][lst[yy][xx].rfind('/'):], im) if yy == 0: imm = im if yy > 0: imm = np.concatenate((imm, im), axis=0) if hillshading: if xx == 0: plt.ioff() xdim = imm.shape[1] hillshading.imshow_hs( imm[:, 0:xdim / 3], save=dst + '/' + 'raw_' + lst[yy][xx][lst[yy][xx].rfind('/') + 1:-4] + '_hillshading') plt.close() hillshading.imshow_hs(imm[:, xdim / 3:2 * xdim / 3], save=dst + '/' + 'bkgrdsubtract_' + lst[yy][xx][lst[yy][xx].rfind('/') + 1:-4] + '_hillshading') plt.close() hillshading.imshow_hs(imm[:, 2 * xdim / 3:3 * xdim / 3], save=dst + '/' + 'cells_' + lst[yy][xx][lst[yy][xx].rfind('/') + 1:-4] + '_hillshading') plt.close() if plot: if xx == 0: plt.ioff() xdim = imm.shape[1] plt.imshow(imm[:, 0:xdim / 3]) plt.savefig(dst + '/' + 'raw_' + lst[yy][xx][lst[yy][xx].rfind('/') + 1:-4], dpi=200) plt.close() plt.imshow(imm[:, xdim / 3:2 * xdim / 3]) plt.savefig(dst + '/' + 'bkgrdsubtract_' + lst[yy][xx][lst[yy][xx].rfind('/') + 1:-4], dpi=200) plt.close() plt.imshow(imm[:, 2 * xdim / 3:3 * xdim / 3]) plt.savefig(dst + '/' + 'cells_' + lst[yy][xx][lst[yy][xx].rfind('/') + 1:-4], dpi=200) plt.close() if concatenate: if xx == 0: bigim = np.zeros((len(lst[0]), imm.shape[0], imm.shape[1])) bigim[0, ...] = imm if xx > 0: bigim[xx, ...] = imm if xx % 10 == 0: sys.stdout.write('\n{} of {}'.format(xx, len(lst[0]))) sys.stdout.flush() #fix hillshade so that you can save out image. AND FIX SCALING (RESIZE OUTS 32 BIT) if not hillshade: tifffile.imsave(dst + '/parameter_sweep_output.tif', bigim.astype('uint16')) sys.stdout.write('Saved as {}'.format(dst + '/parameter_sweep_output.tif')) sys.stdout.flush() return
def sweep_parameters(jobid, pth=False, rescale=False, **kwargs): '''Function to sweep parameters Inputs: ---------------- jobid: chunk of tissue to run (usually int between 20-30) #pth (optional): if pth to output folder after running package, function will load the param file automatically rescale (optional): str of dtype to rescale to. E.g.: 'uint8' kwargs (if not pth): 'params' from run_clearmap_cluster.py ''' #if pth: sys.path.append(pth+'/clearmap_cluster'); from run_clearmap_cluster import params; kwargs = params #set param sweeps rBP_size_r = range(3, 11, 2) #[5, 11] #range(5,19,2) ###evens seem to not be good fEMP_hmax_r = [None] #[None, 5, 10, 20, 40] fEMP_size_r = [5] #range(3,8) fEMP_threshold_r = [None] #range(0,10) fIP_method_r = ['Max'] #['Max, 'Mean'] fIP_size_r = [5] #range(1,5) dCSP_threshold_r = range( 50, 230, 15 ) #[60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225]#range(50, 200, 10) # calculate number of iterations tick = 0 for rBP_size, fEMP_hmax, fEMP_size, fEMP_threshold, fIP_method, fIP_size, dCSP_threshold in product( rBP_size_r, fEMP_hmax_r, fEMP_size_r, fEMP_threshold_r, fIP_method_r, fIP_size_r, dCSP_threshold_r): tick += 1 sys.stdout.write('\n\nNumber of iterations is {}:'.format(tick)) #make folder for final output: opt = kwargs['outputdirectory'] + '/optimization' out = opt + '/parameter_sweep' makedir(out) makedir(opt) #; removedir(out); makedir(out) ntick = 0 for rBP_size, fEMP_hmax, fEMP_size, fEMP_threshold, fIP_method, fIP_size, dCSP_threshold in product( rBP_size_r, fEMP_hmax_r, fEMP_size_r, fEMP_threshold_r, fIP_method_r, fIP_size_r, dCSP_threshold_r): pth = out + '/parametersweep_rBP_size{}_fEMP_hmax{}_fEMP_size{}_fEMP_threshold{}_fIP_method{}_fIP_size{}_dCSP_threshold{}.tif'.format( rBP_size, fEMP_hmax, fEMP_size, fEMP_threshold, fIP_method, fIP_size, dCSP_threshold) if not os.path.exists(pth): try: #set params for sweep kwargs['removeBackgroundParameter_size'] = ( rBP_size, rBP_size ) #Remove the background with morphological opening (optimised for spherical objects), e.g. (7,7) kwargs[ 'findExtendedMaximaParameter_hmax'] = fEMP_hmax # (float or None) h parameter (for instance 20) for the initial h-Max transform, if None, do not perform a h-max transform kwargs[ 'findExtendedMaximaParameter_size'] = fEMP_size # size in pixels (x,y) for the structure element of the morphological opening kwargs[ 'findExtendedMaximaParameter_threshold'] = fEMP_threshold # (float or None) include only maxima larger than a threshold, if None keep all local maxima kwargs[ 'findIntensityParameter_method'] = fIP_method # (str, func, None) method to use to determine intensity (e.g. "Max" or "Mean") if None take intensities at the given pixels kwargs['findIntensityParameter_size'] = ( fIP_size, fIP_size, fIP_size ) # (tuple) size of the search box on which to perform the *method* kwargs[ 'detectCellShapeParameter_threshold'] = dCSP_threshold # (float or None) threshold to determine mask. Pixels below this are background if None no mask is generated #run cell detection ntick += 1 sys.stdout.write( '\n\n\n *****Iteration {} of {}*****\n\n\n'. format(ntick, tick)) sys.stdout.write( ' Iteration parameters: {} {} {} {} {} {} {}' .format(kwargs['removeBackgroundParameter_size'], kwargs['findExtendedMaximaParameter_hmax'], kwargs['findExtendedMaximaParameter_size'], kwargs['findExtendedMaximaParameter_threshold'], kwargs['findIntensityParameter_method'], kwargs['findIntensityParameter_size'], kwargs['detectCellShapeParameter_threshold'])) celldetection_operations(jobid, testing=True, **kwargs) #list, load, and maxip if ntick == 1: raw = [ xx for xx in listdirfull(opt + '/raw') if '~' not in xx and '.db' not in xx ] raw.sort() raw_im = np.squeeze(tifffile.imread(raw)) raw_mx = np.max(raw_im, axis=0) bkg = [ xx for xx in listdirfull(opt + '/background') if '~' not in xx and 'Thumbs.db' not in xx ] bkg.sort() bkg_im = tifffile.imread(bkg) bkg_mx = np.max(bkg_im, axis=0) cell = [ xx for xx in listdirfull(opt + '/cell') if '~' not in xx and '.db' not in xx ] cell.sort() cell_im = tifffile.imread(cell) cell_mx = np.max(cell_im, axis=0) #optional rescale: if rescale: raw_mx = rescale_intensity( raw_mx, in_range=str(raw_mx.dtype), out_range=rescale).astype(rescale) bkg_mx = rescale_intensity( bkg_mx, in_range=str(bkg_mx.dtype), out_range=rescale).astype(rescale) cell_mx = rescale_intensity( cell_mx, in_range=str(cell_mx.dtype), out_range=rescale).astype(rescale) #concatenate and save out: bigim = np.concatenate((raw_mx, bkg_mx, cell_mx), axis=1) del bkg, bkg_im, bkg_mx, cell, cell_im, cell_mx tifffile.imsave(pth, bigim, compress=1) except Exception, e: print('Error on: {}, error: {}'.format(pth, e)) im = np.zeros((10, 10, 10)) tifffile.imsave(pth, im, compress=1) with open(os.path.join(out, 'errored_files.txt'), 'a') as fl: fl.write('\n\n{}\n{}\n'.format(pth, kwargs)) fl.close
def set_parameters_for_clearmap(testing=False, **kwargs): """TP: Wrapped this into a function such that parallel processes can get at variables testing should be set to false unless testing where it will save optimization files for cell detection. Editting of this function is how one changes cell detection parameters """ #handle optional inputs, to change a parameter for your application, see run_clearmap_cluster.py. This is done before loading kwargs to ensure no overwrite during parameter sweep and/or cell detection testing rBP_size = kwargs[ "removeBackgroundParameter_size"] if "removeBackgroundParameter_size" in kwargs else ( 7, 7) fEMP_hmax = kwargs[ "findExtendedMaximaParameter_hmax"] if "findExtendedMaximaParameter_hmax" in kwargs else None # (float or None) h parameter (for instance 20) for the initial h-Max transform, if None, do not perform a h-max transform fEMP_size = kwargs[ "findExtendedMaximaParameter_size"] if "findExtendedMaximaParameter_size" in kwargs else 5 # size in pixels (x,y) for the structure element of the morphological opening fEMP_threshold = kwargs[ "findExtendedMaximaParameter_threshold"] if "findExtendedMaximaParameter_threshold" in kwargs else 0 # (float or None) include only maxima larger than a threshold, if None keep all local maxima fIP_method = kwargs[ "findIntensityParameter_method"] if "findIntensityParameter_method" in kwargs else "Max" ## (str, func, None) method to use to determine intensity (e.g. "Max" or "Mean") if None take intensities at the given pixels fIP_size = kwargs[ "findIntensityParameter_size"] if "findIntensityParameter_size" in kwargs else ( 3, 3, 3 ) # (tuple) size of the search box on which to perform the *method* dCSHP_threshold = kwargs[ "detectCellShapeParameter_threshold"] if "detectCellShapeParameter_threshold" in kwargs else 500 ## (float or None) threshold to determine mask. Pixels below this are background if None no mask is generated ##input data from preprocessing: with open(os.path.join(kwargs["outputdirectory"], "param_dict.p"), "rb") as pckl: kwargs.update(pickle.load(pckl)) pckl.close() kwargs = pth_update(kwargs) vols = kwargs["volumes"] cfosvol = [xx for xx in vols if "cellch" in xx.ch_type][0] #lightsheet mods try: autovol = [xx for xx in vols if "regch" in xx.ch_type][0] except Exception as e: print(e) autovol = [xx for xx in vols if "cellch" in xx.ch_type][0] ######################### Data parameters #Data File and Reference channel File, usually as a sequence of files from the microscope #Use \d{4} for 4 digits in the sequence for instance. As an example, if you have cfos-Z0001.ome.tif : #os.path.join() is used to join the savedirectory path and the data paths: cFosFile = os.path.join( cfosvol.full_sizedatafld_vol, cfosvol.brainname + "_C" + cfosvol.channel + "_Z\d{4}.tif") AutofluoFile = os.path.join( autovol.full_sizedatafld_vol, autovol.brainname + "_C" + autovol.channel + "_Z\d{4}.tif") OriginalResolution = kwargs["xyz_scale"] #Specify the range for the cell detection. This doesn"t affect the resampling and registration operations cFosFileRange = { "x": all, "y": all, "z": all } #Resolution of the Raw Data (in um / pixel) #OriginalResolution = (4.0625, 4.0625, 3); #OriginalResolution = (1.63, 1.63, 3); #multiplying everything by 2.5 #Orientation: 1,2,3 means the same orientation as the reference and atlas files. #Flip axis with - sign (eg. (-1,2,3) flips x). 3D Rotate by swapping numbers. (eg. (2,1,3) swaps x and y) #FinalOrientation = (1,2,3); FinalOrientation = kwargs[ "FinalOrientation"] if "FinalOrientation" in kwargs else (3, 2, 1) #Resolution of the Atlas (in um/ pixel) AtlasResolution = kwargs[ "AtlasResolution"] if "AtlasResolution" in kwargs else (25, 25, 25) #Path to registration parameters and atlases AtlasFile = kwargs[ "AtlasFile"] if "AtlasFile" in kwargs else "/jukebox/LightSheetTransfer/atlas/sagittal_atlas_20um_iso.tif" AnnotationFile = kwargs[ "annotationfile"] if "annotationfile" in kwargs else "/jukebox/LightSheetTransfer/atlas/annotation_sagittal_atlas_20um_iso.tif" ######################### Cell Detection Parameters using custom filters #Spot detection method: faster, but optimised for spherical objects. #You can also use "Ilastik" for more complex objects ImageProcessingMethod = "SpotDetection" #testing output: if testing: optdir = os.path.join(kwargs["outputdirectory"], "optimization") makedir(optdir) sys.stdout.write( "\nThis function is set for optimization of cell detection, optimization results in:\n {}\n\n" .format(optdir)) sys.stdout.flush() bgdir = os.path.join(optdir, "background") makedir(bgdir) bg = os.path.join(bgdir, "background\d{4}.ome.tif") exmaxdir = os.path.join(optdir, "extendmax") makedir(exmaxdir) ex = os.path.join(exmaxdir, "extendmax\d{4}.ome.tif") celldir = os.path.join(optdir, "cell") makedir(celldir) cell = os.path.join(celldir, "cell\d{4}.ome.tif") illum = None else: bg = None ex = None cell = None illum = None #For illumination correction (necessitates a specific calibration curve for your microscope) correctIlluminationParameter = { "flatfield": None, # (True or None) flat field intensities, if None do not correct image for illumination "background": None, # (None or array) background image as file name or array, if None background is assumed to be zero "scaling": None, # was "Mean" (str or None) scale the corrected result by this factor, if "max"/"mean" scale to keep max/mean invariant "save": illum, # (str or None) save the corrected image to file "verbose": True # (bool or int) print / plot information about this step } #Remove the background with morphological opening (optimised for spherical objects) removeBackgroundParameter = { "size": rBP_size, # size in pixels (x,y) for the structure element of the morphological opening #"size" : (7 * deltaresolution, 7 * deltaresolution), # size in pixels (x,y) for the structure element of the morphological opening #"size" : (5,5), # size in pixels (x,y) for the structure element of the morphological opening # 5 seems very good for 1.3x objective, 5.5 is bad, 4.5 is really bad "save": bg, # file name to save result of this operation "verbose": True # print / plot information about this step } #Difference of Gaussians filter: to enhance the edges. Useful if the objects have a non smooth texture (eg: amyloid deposits) filterDoGParameter = { "size": None, # (tuple or None) size for the DoG filter in pixels (x,y,z) if None, do not correct for any background "sigma": None, # (tuple or None) std of outer Gaussian, if None automatically determined from size "sigma2": None, # (tuple or None) std of inner Gaussian, if None automatically determined from size "save": None, # (str or None) file name to save result of this operation if None dont save to file "verbose": True # (bool or int) print / plot information about this step } #Extended maxima: if the peak intensity in the object is surrounded by smaller peaks: avoids overcounting "granular" looking objects findExtendedMaximaParameter = { "hMax": fEMP_hmax, # (float or None) h parameter (for instance 20) for the initial h-Max transform, if None, do not perform a h-max transform "size": fEMP_size, # (tuple) size for the structure element for the local maxima filter "threshold": fEMP_threshold, # (float or None) include only maxima larger than a threshold, if None keep all local maxima "save": ex, # (str or None) file name to save result of this operation if None dont save to file "verbose": True # (bool or int) print / plot information about this step } #If no cell shape detection and the maximum intensity is not at the gravity center of the object, look for a peak intensity around the center of mass. findIntensityParameter = { "method": fIP_method, # (str, func, None) method to use to determine intensity (e.g. "Max" or "Mean") if None take intensities at the given pixels "size": fIP_size # (tuple) size of the search box on which to perform the *method* } #Object volume detection. The object is painted by a watershed, until reaching the intensity threshold, based on the background subtracted image detectCellShapeParameter = { "threshold": dCSHP_threshold, #* deltaresolution, # (float or None) threshold to determine mask. Pixels below this are background if None no mask is generated. For 1.3x objective 500 is too large "save": cell, # (str or None) file name to save result of this operation if None dont save to file "verbose": True # (bool or int) print / plot information about this step if None take intensities at the given pixels } ##### ########### ###Nico"s parameters in paper w 4.0625: #background 7 #threhold cell detection intensity of 700 #cells within size o f20 to 500 voxels #density maps of voxelation of 15 ########### ## Parameters for cell detection using spot detection algorithm detectSpotsParameter = { "correctIlluminationParameter": correctIlluminationParameter, "removeBackgroundParameter": removeBackgroundParameter, "filterDoGParameter": filterDoGParameter, "findExtendedMaximaParameter": findExtendedMaximaParameter, "findIntensityParameter": findIntensityParameter, "detectCellShapeParameter": detectCellShapeParameter } #set directories if not testing: savedirectory = os.path.join(kwargs["outputdirectory"], "clearmap_cluster_output") makedir(savedirectory) elastixdirectory = os.path.join(kwargs["parameterfolder"]) else: savedirectory = kwargs["outputdirectory"] elastixdirectory = os.path.join(kwargs["parameterfolder"]) #################### Heat map generation ##Voxelization: file name for the output: VoxelizationFile = os.path.join(savedirectory, "points_voxelized.tif") # Parameter to calculate the density of the voxelization voxelizeParameter = { #Method to voxelize "method": "Spherical", # Spherical,"Rectangular, Gaussian" # Define bounds of the volume to be voxelized in pixels "size": (15, 15, 15), #shouldn"t need to change as this is now at the level of the isotropic atlas #"size" : (15 * deltaresolution,15 * deltaresolution,15 * deltaresolution), # Voxelization weigths (e/g intensities) "weights": None } ############################ Config parameters #Processes to use for Resampling (usually twice the number of physical processors) ResamplingParameter = { "processes": 16 } #Stack Processing Parameter for cell detection StackProcessingParameter = { #max number of parallel processes. Be careful of the memory footprint of each process! "processes": 1, #automatically set to 1 if using processMethod #chunk sizes: number of planes processed at once "chunkSizeMax": 30, "chunkSizeMin": 5, "chunkOverlap": 15, #optimize chunk size and number to number of processes to limit the number of cycles "chunkOptimization": True, #increase chunk size for optimization (True, False or all = automatic) "chunkOptimizationSize": all, "processMethod": "cluster" #"sequential", "parallel"=local parallelization, "cluster" = parellization using cluster } ######################## Run Parameters, usually you don"t need to change those ### Resample Fluorescent and CFos images # Autofluorescent cFos resampling for aquisition correction ResolutionAffineCFosAutoFluo = kwargs[ "ResolutionAffineCFosAutoFluo"] if "ResolutionAffineCFosAutoFluo" in kwargs else ( 16, 16, 16) CorrectionResamplingParameterCfos = ResamplingParameter.copy() CorrectionResamplingParameterCfos["source"] = cFosFile CorrectionResamplingParameterCfos["sink"] = os.path.join( savedirectory, "cfos_resampled.tif") CorrectionResamplingParameterCfos["resolutionSource"] = OriginalResolution CorrectionResamplingParameterCfos[ "resolutionSink"] = ResolutionAffineCFosAutoFluo CorrectionResamplingParameterCfos["orientation"] = FinalOrientation #Files for Auto-fluorescence for acquisition movements correction CorrectionResamplingParameterAutoFluo = CorrectionResamplingParameterCfos.copy( ) CorrectionResamplingParameterAutoFluo["source"] = AutofluoFile CorrectionResamplingParameterAutoFluo["sink"] = os.path.join( savedirectory, "autofluo_for_cfos_resampled.tif") #Files for Auto-fluorescence (Atlas Registration) RegistrationResamplingParameter = CorrectionResamplingParameterAutoFluo.copy( ) RegistrationResamplingParameter["sink"] = os.path.join( savedirectory, "autofluo_resampled.tif") RegistrationResamplingParameter["resolutionSink"] = AtlasResolution ### Align cFos and Autofluo affinepth = [ xx for xx in listdirfull(elastixdirectory) if "affine" in xx and "~" not in xx ][0] bsplinepth = [ xx for xx in listdirfull(elastixdirectory) if "bspline" in xx and "~" not in xx ][0] CorrectionAlignmentParameter = { #moving and reference images "movingImage": os.path.join(savedirectory, "autofluo_for_cfos_resampled.tif"), "fixedImage": os.path.join(savedirectory, "cfos_resampled.tif"), #elastix parameter files for alignment "affineParameterFile": affinepth, "bSplineParameterFile": bsplinepth, #directory of the alignment result "resultDirectory": os.path.join(savedirectory, "elastix_cfos_to_auto") } ### Align Autofluo and Atlas #directory of the alignment result RegistrationAlignmentParameter = CorrectionAlignmentParameter.copy() RegistrationAlignmentParameter["resultDirectory"] = os.path.join( savedirectory, "elastix_auto_to_atlas") #moving and reference images RegistrationAlignmentParameter["movingImage"] = AtlasFile RegistrationAlignmentParameter["fixedImage"] = os.path.join( savedirectory, "autofluo_resampled.tif") #elastix parameter files for alignment RegistrationAlignmentParameter["affineParameterFile"] = affinepth RegistrationAlignmentParameter["bSplineParameterFile"] = bsplinepth # result files for cell coordinates (csv, vtk or ims) SpotDetectionParameter = { "source": cFosFile, "sink": (os.path.join(savedirectory, "cells-allpoints.npy"), os.path.join(savedirectory, "intensities-allpoints.npy")), "detectSpotsParameter": detectSpotsParameter } SpotDetectionParameter = joinParameter(SpotDetectionParameter, cFosFileRange) ImageProcessingParameter = joinParameter(StackProcessingParameter, SpotDetectionParameter) FilteredCellsFile = (os.path.join(savedirectory, "cells.npy"), os.path.join(savedirectory, "intensities.npy")) TransformedCellsFile = os.path.join(savedirectory, "cells_transformed_to_Atlas.npy") ### Transform points from Original c-Fos position to autofluorescence ## Transform points from original to corrected # downscale points to referenece image size CorrectionResamplingPointsParameter = CorrectionResamplingParameterCfos.copy( ) CorrectionResamplingPointsParameter["pointSource"] = os.path.join( savedirectory, "cells.npy") CorrectionResamplingPointsParameter["dataSizeSource"] = cFosFile CorrectionResamplingPointsParameter["pointSink"] = None CorrectionResamplingPointsInverseParameter = CorrectionResamplingPointsParameter.copy( ) CorrectionResamplingPointsInverseParameter["dataSizeSource"] = cFosFile CorrectionResamplingPointsInverseParameter["pointSink"] = None ## Transform points from corrected to registered # downscale points to referenece image size RegistrationResamplingPointParameter = RegistrationResamplingParameter.copy( ) RegistrationResamplingPointParameter["dataSizeSource"] = cFosFile RegistrationResamplingPointParameter["pointSink"] = None ##TP bigdct = {} bigdct["detectSpotsParameter"] = detectSpotsParameter bigdct["voxelizeParameter"] = voxelizeParameter bigdct["ResamplingParameter"] = ResamplingParameter bigdct["StackProcessingParameter"] = StackProcessingParameter bigdct[ "CorrectionResamplingParameterCfos"] = CorrectionResamplingParameterCfos bigdct[ "CorrectionResamplingParameterAutoFluo"] = CorrectionResamplingParameterAutoFluo bigdct["RegistrationResamplingParameter"] = RegistrationResamplingParameter bigdct["CorrectionAlignmentParameter"] = CorrectionAlignmentParameter bigdct["RegistrationAlignmentParameter"] = RegistrationAlignmentParameter bigdct["SpotDetectionParameter"] = SpotDetectionParameter bigdct["ImageProcessingParameter"] = ImageProcessingParameter bigdct["FilteredCellsFile"] = FilteredCellsFile bigdct["TransformedCellsFile"] = TransformedCellsFile bigdct[ "CorrectionResamplingPointsParameter"] = CorrectionResamplingPointsParameter bigdct[ "CorrectionResamplingPointsInverseParameter"] = CorrectionResamplingPointsInverseParameter bigdct[ "RegistrationResamplingPointParameter"] = RegistrationResamplingPointParameter bigdct["AnnotationFile"] = AnnotationFile bigdct["VoxelizationFile"] = VoxelizationFile bigdct["ImageProcessingMethod"] = ImageProcessingMethod bigdct["AtlasFile"] = AtlasFile bigdct["OutputDirectory"] = kwargs["outputdirectory"] if testing: bigdct["OptimizationLocation"] = optdir #bigdct[] = #bigdct[] = return pth_update(bigdct)