def classifyPixel(project, source, sink=None, processingDirectory=None, cleanup=True): """Run pixel classification in headless moded using a trained project file Arguments: project (str): ilastik project .ilp file source (str or array): image source sink (str or array or None): image sink Returns: str or array: classified image sink """ #generate source image file if source is array if isinstance(source, str): inpfile = source else: #generate temporary file if processingDirectory is None: processingDirectory = tempfile.mkdtemp() inpfile = os.path.join(processingDirectory, 'ilastik.npy') io.writePoints(inpfile, source.transpose((2, 1, 0))) if isinstance(sink, str): outfile = sink else: #outdir = tempfile.mkdtemp(); #outfile = os.path.join(outdir, 'result\d*.tif'); outfile = tempfile.mktemp('.h5') ilinp = fileNameToIlastikInput(inpfile) ilout = fileNameToIlastikOuput(outfile) args = '--project="' + project + '" ' + ilout + ' ' + ilinp run(args) #clean up if cleanup and processingDirectory is not None: shutil.rmtree(processingDirectory) if not isinstance(sink, str): sink = readResultH5(outfile) if cleanup: os.remove(outfile) return sink
def classifyPixel(project, source, sink = None, processingDirectory = None, cleanup = True): """Run pixel classification in headless moded using a trained project file Arguments: project (str): ilastik project .ilp file source (str or array): image source sink (str or array or None): image sink Returns: str or array: classified image sink """ #generate source image file if source is array if isinstance(source, str): inpfile = source; else: #generate temporary file if processingDirectory is None: processingDirectory = tempfile.mkdtemp(); inpfile = os.path.join(processingDirectory, 'ilastik.npy') io.writePoints(inpfile, source.transpose((2,1,0))); if isinstance(sink, str): outfile = sink; else: #outdir = tempfile.mkdtemp(); #outfile = os.path.join(outdir, 'result\d*.tif'); outfile = tempfile.mktemp('.h5'); ilinp = fileNameToIlastikInput(inpfile); ilout = fileNameToIlastikOuput(outfile); args = '--project="' + project + '" ' + ilout + ' ' + ilinp; run(args); #clean up if cleanup and processingDirectory is not None: shutil.rmtree(processingDirectory); if not isinstance(sink, str): sink = readResultH5(outfile); if cleanup: os.remove(outfile); return sink;
def parallelProcessStack(source, x = all, y = all, z = all, sink = None, processes = 2, chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15, chunkOptimization = True, chunkOptimizationSize = all, function = noProcessing, join = joinPoints, verbose = False, **parameter): """Parallel process a image stack Main routine that distributes image processing on paralllel processes. Arguments: source (str): image source x,y,z (tuple or all): range specifications sink (str or None): destination for the result processes (int): number of parallel processes chunkSizeMax (int): maximal size of a sub-stack chunkSizeMin (int): minial size of a sub-stack chunkOverlap (int): minimal sub-stack overlap chunkOptimization (bool): optimize chunck sizes to best fit number of processes chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing function (function): the main image processing script join (function): the fuction to join the results from the image processing script verbose (bool): print information on sub-stack generation Returns: str or array: results of the image processing """ subStacks = calculateSubStacks(source, x = x, y = y, z = z, processes = processes, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap, chunkOptimization = chunkOptimization, chunkOptimizationSize = chunkOptimizationSize, verbose = verbose); nSubStacks = len(subStacks); if verbose: print "Number of SubStacks: %d" % nSubStacks; #for i in range(nSubStacks): # self.printSubStackInfo(subStacks[i]); argdata = []; for i in range(nSubStacks): argdata.append((function, parameter, subStacks[i], verbose)); #print argdata # process in parallel pool = Pool(processes = processes); results = pool.map(_processSubStack, argdata); #print '=========== results'; #print results; #join the results results = join(results, subStacks = subStacks, **parameter); #write / or return return io.writePoints(sink, results);
def parallelProcessStack(source, x = all, y = all, z = all, sink = None, processes = 2, chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15, chunkOptimization = True, chunkOptimizationSize = all, function = noProcessing, join = joinPoints, verbose = False, **parameter): """Parallel process a image stack Main routine that distributes image processing on paralllel processes. Arguments: source (str): image source x,y,z (tuple or all): range specifications sink (str or None): destination for the result processes (int): number of parallel processes chunkSizeMax (int): maximal size of a sub-stack chunkSizeMin (int): minial size of a sub-stack chunkOverlap (int): minimal sub-stack overlap chunkOptimization (bool): optimize chunck sizes to best fit number of processes chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing function (function): the main image processing script join (function): the fuction to join the results from the image processing script verbose (bool): print information on sub-stack generation Returns: str or array: results of the image processing """ subStacks = calculateSubStacks(source, x = x, y = y, z = z, processes = processes, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap, chunkOptimization = chunkOptimization, chunkOptimizationSize = chunkOptimizationSize, verbose = verbose); nSubStacks = len(subStacks); if verbose: print("Number of SubStacks: %d" % nSubStacks); #for i in range(nSubStacks): # self.printSubStackInfo(subStacks[i]); argdata = []; for i in range(nSubStacks): argdata.append((function, parameter, subStacks[i], verbose)); #print argdata # process in parallel pool = Pool(processes = processes); results = pool.map(_processSubStack, argdata); #print '=========== results'; #print results; #join the results results = join(results, subStacks = subStacks, **parameter); #write / or return return io.writePoints(sink, results);
def join_results_from_cluster_helper(source, x=all, y=all, z=all, sink=None, chunkSizeMax=100, chunkSizeMin=30, chunkOverlap=15, function=noProcessing, join=joinPoints, verbose=False, **parameter): #calc num of substacks subStacks = calculateSubStacks(source, x=x, y=y, z=z, processes=1, chunkSizeMax=chunkSizeMax, chunkSizeMin=chunkSizeMin, chunkOverlap=chunkOverlap, chunkOptimization=False, verbose=verbose) #load all cell detection job results if type(sink) == tuple: pckfld = os.path.join(sink[0][:sink[0].rfind("/")], "cell_detection") makedir(pckfld) elif type(sink) == str: pckfld = os.path.join(sink[:sink.rfind("/")], "cell_detection") makedir(pckfld) results = [] fls = listdirfull(pckfld) fls.sort() for fl in fls: if "~" not in fl: with open(fl, "rb") as pckl: results.append(pickle.load(pckl)) pckl.close() print(results[0]) #reformat results = [xx for yy in results for xx in yy] #join the results print("Length of results: {}".format(len(results))) results = join(results, subStacks=subStacks, **parameter) #write / or return return io.writePoints(sink, results)
def sequentiallyProcessStack(source, x = all, y = all, z = all, sink = None, chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15, function = noProcessing, join = joinPoints, verbose = False, **parameter): """Sequential image processing on a stack Main routine that sequentially processes a large image on sub-stacks. Arguments: source (str): image source x,y,z (tuple or all): range specifications sink (str or None): destination for the result processes (int): number of parallel processes chunkSizeMax (int): maximal size of a sub-stack chunkSizeMin (int): minial size of a sub-stack chunkOverlap (int): minimal sub-stack overlap chunkOptimization (bool): optimize chunck sizes to best fit number of processes chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing function (function): the main image processing script join (function): the fuction to join the results from the image processing script verbose (bool): print information on sub-stack generation Returns: str or array: results of the image processing """ #determine z ranges subStacks = calculateSubStacks(source, x = x, y = y, z = z, processes = 1, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap, chunkOptimization = False, verbose = verbose); nSubStacks = len(subStacks); #print nSubStacks; argdata = []; for i in range(nSubStacks): argdata.append((function, parameter, subStacks[i], verbose)); #run sequentially results = []; for i in range(nSubStacks): results.append(_processSubStack(argdata[i])); #join the results results = join(results, subStacks = subStacks, **parameter); #write / or return return io.writePoints(sink, results);
def output_analysis_helper(threshold=(20, 900), row=(3, 3), **params): ''' Function to change elastix result directory before running 'step 6' i.e. point transformix to atlas. ''' dct = pth_update(set_parameters_for_clearmap(**params)) dct['RegistrationAlignmentParameter']["resultDirectory"] = os.path.join( params["outputdirectory"], 'clearmap_cluster_output/elastix_auto_to_sim_atlas') points, intensities = io.readPoints( dct['ImageProcessingParameter']["sink"]) #Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row" #row = (0,0) : peak intensity from the raw data #row = (1,1) : peak intensity from the DoG filtered data #row = (2,2) : peak intensity from the background subtracted data #row = (3,3) : voxel size from the watershed points, intensities = thresholdPoints(points, intensities, threshold=threshold, row=row) #points, intensities = thresholdPoints(points, intensities, threshold = (20, 900), row = (2,2)); io.writePoints(dct['FilteredCellsFile'], (points, intensities)) # Transform point coordinates ############################# points = io.readPoints( dct['CorrectionResamplingPointsParameter']["pointSource"]) points = resamplePoints(**dct['CorrectionResamplingPointsParameter']) points = transformPoints( points, transformDirectory=dct['CorrectionAlignmentParameter'] ["resultDirectory"], indices=False, resultDirectory=None) dct['CorrectionResamplingPointsInverseParameter']["pointSource"] = points points = resamplePointsInverse( **dct['CorrectionResamplingPointsInverseParameter']) dct['RegistrationResamplingPointParameter']["pointSource"] = points points = resamplePoints(**dct['RegistrationResamplingPointParameter']) points = transformPoints( points, transformDirectory=dct['RegistrationAlignmentParameter'] ["resultDirectory"], indices=False, resultDirectory=None) io.writePoints(dct['TransformedCellsFile'], points) # Heat map generation ##################### points = io.readPoints(dct['TransformedCellsFile']) intensities = io.readPoints(dct['FilteredCellsFile'][1]) #Without weigths: vox = voxelize(points, dct['AtlasFile'], **dct['voxelizeParameter']) if not isinstance(vox, basestring): io.writeData(os.path.join(dct['OutputDirectory'], 'cells_heatmap.tif'), vox.astype('int32')) #With weigths from the intensity file (here raw intensity): dct['voxelizeParameter']["weights"] = intensities[:, 0].astype(float) vox = voxelize(points, dct['AtlasFile'], **dct['voxelizeParameter']) if not isinstance(vox, basestring): io.writeData( os.path.join(dct['OutputDirectory'], 'cells_heatmap_weighted.tif'), vox.astype('int32')) #Table generation: ################## #With integrated weigths from the intensity file (here raw intensity): ids, counts = countPointsInRegions(points, labeledImage=dct['AnnotationFile'], intensities=intensities, intensityRow=0) table = np.zeros(ids.shape, dtype=[('id', 'int64'), ('counts', 'f8'), ('name', 'a256')]) table["id"] = ids table["counts"] = counts table["name"] = labelToName(ids) io.writeTable( os.path.join(dct['OutputDirectory'], 'Annotated_counts_intensities.csv'), table) #Without weigths (pure cell number): ids, counts = countPointsInRegions(points, labeledImage=dct['AnnotationFile'], intensities=None) table = np.zeros(ids.shape, dtype=[('id', 'int64'), ('counts', 'f8'), ('name', 'a256')]) table["id"] = ids table["counts"] = counts table["name"] = labelToName(ids) io.writeTable(os.path.join(dct['OutputDirectory'], 'Annotated_counts.csv'), table) print('Analysis Completed') return
def transformPoints(source, sink=None, transformParameterFile=None, transformDirectory=None, indices=True, resultDirectory=None, tmpFile=None): """Transform coordinates math:`x` via elastix estimated transformation to :math:`T(x)` Note the transformation is from the fixed image coorindates to the moving image coordiantes. Arguments: source (str): source of the points sink (str or None): sink for transformed points transformParameterFile (str or None): parameter file for the primary transformation, if None, the file is determined from the transformDirectory. transformDirectory (str or None): result directory of elastix alignment, if None the transformParameterFile has to be given. indices (bool): if True use points as pixel coordinates otherwise spatial coordinates. resultDirectory (str or None): elastic result directory tmpFile (str or None): file name for the elastix point file. Returns: array or str: array or file name of transformed points """ global TransformixBinary checkElastixInitialized() global ElastixSettings if tmpFile == None: tmpFile = os.path.join(tempfile.gettempdir(), 'elastix_input.txt') # write text file if isinstance(source, basestring): #check if we have elastix signature with open(source) as f: line = f.readline() f.close() if line[:5] == 'point' or line[:5] != 'index': txtfile = source else: points = io.readPoints(source) #points = points[:,[1,0,2]]; txtfile = tmpFile writePoints(txtfile, points) elif isinstance(source, numpy.ndarray): txtfile = tmpFile #points = source[:,[1,0,2]]; writePoints(txtfile, source) else: raise RuntimeError('transformPoints: source not string or array!') if resultDirectory == None: outdirname = os.path.join(tempfile.gettempdir(), 'elastix_output') else: outdirname = resultDirectory if not os.path.exists(outdirname): os.makedirs(outdirname) if transformParameterFile == None: if transformDirectory == None: RuntimeError( 'neither alignment directory and transformation parameter file specified!' ) transformparameterdir = transformDirectory transformparameterfile = getTransformParameterFile( transformparameterdir) else: transformparameterdir = os.path.split(transformParameterFile) transformparameterdir = transformparameterdir[0] transformparameterfile = transformParameterFile #transform #make path in parameterfiles absolute setPathTransformParameterFiles(transformparameterdir) #run transformix cmd = TransformixBinary + ' -threads 8 -def ' + txtfile + ' -out ' + outdirname + ' -tp ' + transformparameterfile res = os.system(cmd) if res != 0: raise RuntimeError('failed executing ' + cmd) #read data / file if sink == []: return io.path.join(outdirname, 'outputpoints.txt') else: #read coordinates transpoints = parseElastixOutputPoints(os.path.join( outdirname, 'outputpoints.txt'), indices=indices) #correct x,y,z to y,x,z #transpoints = transpoints[:,[1,0,2]]; #cleanup for f in os.listdir(outdirname): os.remove(os.path.join(outdirname, f)) os.rmdir(outdirname) return io.writePoints(sink, transpoints)
def sequentiallyProcessStack(source, x = all, y = all, z = all, sink = None, chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15, function = noProcessing, join = joinPoints, verbose = False, **parameter): """Sequential image processing on a stack Main routine that sequentially processes a large image on sub-stacks. Arguments: source (str): image source x,y,z (tuple or all): range specifications sink (str or None): destination for the result processes (int): number of parallel processes chunkSizeMax (int): maximal size of a sub-stack chunkSizeMin (int): minial size of a sub-stack chunkOverlap (int): minimal sub-stack overlap chunkOptimization (bool): optimize chunck sizes to best fit number of processes chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing function (function): the main image processing script join (function): the fuction to join the results from the image processing script verbose (bool): print information on sub-stack generation Returns: str or array: results of the image processing """ #determine z ranges subStacks = calculateSubStacks(source, x = x, y = y, z = z, processes = 1, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap, chunkOptimization = False, verbose = verbose); nSubStacks = len(subStacks); #print nSubStacks; argdata = []; for i in range(nSubStacks): argdata.append((function, parameter, subStacks[i], verbose)); #run sequentially results = []; for i in range(nSubStacks): results.append(_processSubStack(argdata[i])); #join the results results = join(results, subStacks = subStacks, **parameter); #write / or return return io.writePoints(sink, results); ### Pickle does not like classes: ## sub stack information #class SubStack(object): # """Class containing all info of a sub stack usefull for the image processing and result joining functions""" # # # sub stack id # stackId = None; # # # number of stacks # nStacks = None; # # # tuple of x,y,z range of this sub stack # z = all; # x = all; # y = all; # # #original source # source = None; # # # tuple of center point of the overlaping regions # zCenters = None; # # # tuple of z indices that would generate full image without overlaps # zCenterIndices = None; # # # tuple of z indices in the sub image as returned by readData that would generate full image without overlaps # zSubCenterIndices = None; # # # def __init__(slf, stackId = 0, nStacks = 1, source = None, x = all, y = all, z = all, zCenters = all, zCenterIndices = all): # slf.stackId = stackId; # slf.nStacks = nStacks; # slf.source = source; # slf.x = x; # slf.y = y; # slf.z = z; # slf.zCenters = zCenters; # slf.zCenterIndices = zCenterIndices; # if not zCenterIndices is all and not z is all: # slf.zSubCenterIndices = (c - z[0] for c in zCenterIndices); # else: # slf.zSubCenterIndices = all; # # #def printSubStackInfo(slf, out = sys.stdout): # out.write("Sub Stack: %d / %d\n" % (slf.stackId, slf.nStacks)); # out.write("source: %s\n" % slf.source); # out.write("x,y,z: %s, %s, %s\n" % (str(slf.x), str(slf.y), str(slf.z))); # out.write("zCenters: %s\n" % str(slf.zCenters)); # out.write("zCenterIndices: %s\n" % str(slf.zCenterIndices));
def transformPoints(source, sink = None, transformParameterFile = None, transformDirectory = None, indices = True, resultDirectory = None, tmpFile = None): """Transform coordinates math:`x` via elastix estimated transformation to :math:`T(x)` Note the transformation is from the fixed image coorindates to the moving image coordiantes. Arguments: source (str): source of the points sink (str or None): sink for transformed points transformParameterFile (str or None): parameter file for the primary transformation, if None, the file is determined from the transformDirectory. transformDirectory (str or None): result directory of elastix alignment, if None the transformParameterFile has to be given. indices (bool): if True use points as pixel coordinates otherwise spatial coordinates. resultDirectory (str or None): elastic result directory tmpFile (str or None): file name for the elastix point file. Returns: array or str: array or file name of transformed points """ global TransformixBinary; checkElastixInitialized(); global ElastixSettings; if tmpFile == None: tmpFile = os.path.join(tempfile.tempdir, 'elastix_input.txt'); # write text file if isinstance(source, basestring): #check if we have elastix signature with open(source) as f: line = f.readline(); f.close(); if line[:5] == 'point' or line[:5] != 'index': txtfile = source; else: points = io.readPoints(source); #points = points[:,[1,0,2]]; txtfile = tmpFile; writePoints(txtfile, points); elif isinstance(source, numpy.ndarray): txtfile = tmpFile; #points = source[:,[1,0,2]]; writePoints(txtfile, source); else: raise RuntimeError('transformPoints: source not string or array!'); if resultDirectory == None: outdirname = os.path.join(tempfile.tempdir, 'elastix_output'); else: outdirname = resultDirectory; if not os.path.exists(outdirname): os.makedirs(outdirname); if transformParameterFile == None: if transformDirectory == None: RuntimeError('neither alignment directory and transformation parameter file specified!'); transformparameterdir = transformDirectory transformparameterfile = getTransformParameterFile(transformparameterdir); else: transformparameterdir = os.path.split(transformParameterFile); transformparameterdir = transformparameterdir[0]; transformparameterfile = transformParameterFile; #transform #make path in parameterfiles absolute setPathTransformParameterFiles(transformparameterdir); #run transformix cmd = TransformixBinary + ' -def ' + txtfile + ' -out ' + outdirname + ' -tp ' + transformparameterfile; res = os.system(cmd); if res != 0: raise RuntimeError('failed executing ' + cmd); #read data / file if sink == []: return io.path.join(outdirname, 'outputpoints.txt') else: #read coordinates transpoints = parseElastixOutputPoints(os.path.join(outdirname, 'outputpoints.txt'), indices = indices); #correct x,y,z to y,x,z #transpoints = transpoints[:,[1,0,2]]; #cleanup for f in os.listdir(outdirname): os.remove(os.path.join(outdirname, f)); os.rmdir(outdirname) return io.writePoints(sink, transpoints);
#pdb.set_trace() #io.writeData(os.path.join(homeDir, 'Results/OverlayWatershed.tif'), overlay_Img); overlay_Img = plt.fredOverlayPoints(cfos_fn, points, pointColor=[200, 0, 0]) io.writeData(os.path.join(resultDir, fName + '_PointsOriginalImg.tif'), overlay_Img) overlay_Img = plt.overlayPoints(imgD, points, pointColor=[200, 0, 0]) io.writeData(os.path.join(resultDir, fName + '_PointsFilterDoG.tif'), overlay_Img) #Convert points from 3d to 2d points = numpy.delete(points, 2, axis=1) io.writePoints(points_fn, points) ##############################################################333 imgR = io.readData(auto_fn) imgR = imgR[..., numpy.newaxis] imgR = imgR.astype('int16') imgR, scaleFactor = resampleData(imgR, auto_R_fn, orientation=(1, 2, 3), dataSizeSink=None, resolutionSource=pixelRes, resolutionSink=(25, 25, 1), processingDirectory=None, processes=4, cleanup=True,
if result == "ENDPROCESS": print("Jobid > # of jobs required, ending job") except: print("Jobid > # of jobs required, ending job") print("\n finished step 4 - cell detection \n") ###########################################STEP 5######################################################## out = join_results_from_cluster_helper(**dct["ImageProcessingParameter"]) print("\n finished step 5 \n") ###########################################STEP 6######################################################## threshold = (500,3000); row = (2,2) points, intensities = io.readPoints(dct["ImageProcessingParameter"]["sink"]) #Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row" #row = (0,0) : peak intensity from the raw data #row = (1,1) : peak intensity from the DoG filtered data #row = (2,2) : peak intensity from the background subtracted data #row = (3,3) : voxel size from the watershed points, intensities = thresholdPoints(points, intensities, threshold = threshold, row = row) #change dst to match parameters sweeped dst = (os.path.join(brain, "clearmap_cluster_output/cells_rBP%s_fIPmethod%s_fIPsize%s_dCSP%s.npy" % (rBP_size, fIP_method, fIP_size, dCSP_threshold)), os.path.join(brain, "clearmap_cluster_output/intensities_rBP%s_fIPmethod%s_fIPsize%s_dCSP%s.npy" % (rBP_size, fIP_method, fIP_size, dCSP_threshold))) io.writePoints(dst, (points, intensities)); print("\n finished step 6 \n")
#io.writeData(os.path.join(homeDir, 'Results/OverlayWatershed.tif'), overlay_Img); overlay_Img = plt.fredOverlayPoints(input_fn, points, pointColor=[200, 0, 0]) io.writeData(os.path.join(homeDir, 'Results/PointsOriginalImg.tif'), overlay_Img) overlay_Img = plt.fredOverlayPoints(os.path.join(homeDir, 'Results/FilterDoG.tif'), points, pointColor=[200, 0, 0]) io.writeData(os.path.join(homeDir, 'Results/PointsFilterDoG.tif'), overlay_Img) ################################################################################################# #Convert points from 3d to 2d points = numpy.delete(points, 2, axis=1) io.writePoints(points_fn, points) #Resampled Img imgR = io.readData(auto_fn) imgR = imgR[..., numpy.newaxis] imgR = imgR.astype('int16') imgR, scaleFactor = resampleData(imgR, auto_R_fn, orientation=(1, 2, 3), dataSizeSink=None, resolutionSource=(2.3719, 2.3719, 1), resolutionSink=(25, 25, 1), processingDirectory=None, processes=4, cleanup=True, verbose=True, interpolation='linear')
def flatfieldLineFromRegression(data, sink = None, method = 'polynomial', reverse = None, verbose = False): """Create flat field line fit from a list of positions and intensities The fit is either to be assumed to be a Gaussian: .. math: I(x) = a \\exp^{- (x- x_0)^2 / (2 \\sigma)) + b" or follows a order 6 radial polynomial .. math: I(x) = a + b (x- x_0)^2 + c (x- x_0)^4 + d (x- x_0)^6 Arguments: data (array): intensity data as vector of intensities or (n,2) dim array of positions d=0 and intensities measurements d=1:-1 sink (str or None): destination to write the result of the fit method (str): method to fit intensity data, 'Gaussian' or 'Polynomial' reverse (bool): reverse the line fit after fitting verbose (bool): print and plot information for the fit Returns: array: fitted intensities on points """ data = io.readPoints(data); # split data if len(data.shape) == 1: x = numpy.arange(0, data.shape[0]); y = data; elif len(data.shape) == 2: x = data[:,0] y = data[:,1:-1]; else: raise RuntimeError('flatfieldLineFromRegression: input data not a line or array of x,i data'); #calculate mean of the intensity measurements ym = numpy.mean(y, axis = 1); if verbose: plt.figure() for i in range(1,data.shape[1]): plt.plot(x, data[:,i]); plt.plot(x, ym, 'k'); if method.lower() == 'polynomial': ## fit r^6 mean = sum(ym * x)/sum(ym) def f(x,m,a,b,c,d): return a + b * (x-m)**2 + c * (x-m)**4 + d * (x-m)**6; popt, pcov = curve_fit(f, x, ym, p0 = (mean, 1, 1, 1, .1)); m = popt[0]; a = popt[1]; b = popt[2]; c = popt[3]; d = popt[4]; if verbose: print "polynomial fit: %f + %f (x- %f)^2 + %f (x- %f)^4 + %f (x- %f)^6" % (a, b, m, c, m, d, m); def fopt(x): return f(x, m = m, a = a, b = b, c = c, d = d); flt = map(fopt, range(0, int(x[-1]))); else: ## Gaussian fit mean = sum(ym * x)/sum(ym) sigma = sum(ym * (x-mean)**2)/(sum(ym)) def f(x, a, m, s, b): return a * numpy.exp(- (x - m)**2 / 2 / s) + b; popt, pcov = curve_fit(f, x, ym, p0 = (1000, mean, sigma, 400)); a = popt[0]; m = popt[1]; s = popt[2]; b = popt[3]; if verbose: print "Gaussian fit: %f exp(- (x- %f)^2 / (2 %f)) + %f" % (a, m, s, b); def fopt(x): return f(x, a = a, m = m, s = s, b = b); if reverse: flt.reverse(); if verbose: plt.plot(x, flt); plt.title('flatfieldLineFromRegression') return io.writePoints(sink, flt);
def output_analysis( threshold=(20, 900), row=(3, 3), check_cell_detection=False, **params): """Wrapper for analysis: Inputs ------------------- Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row" Row: row = (0,0) : peak intensity from the raw data row = (1,1) : peak intensity from the DoG filtered data row = (2,2) : peak intensity from the background subtracted data row = (3,3) : voxel size from the watershed Check Cell detection: (For the testing phase only, remove when running on the full size dataset) """ dct = pth_update(set_parameters_for_clearmap(**params)) points, intensities = io.readPoints( dct["ImageProcessingParameter"]["sink"]) #Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row" #row = (0,0) : peak intensity from the raw data #row = (1,1) : peak intensity from the DoG filtered data #row = (2,2) : peak intensity from the background subtracted data #row = (3,3) : voxel size from the watershed points, intensities = thresholdPoints(points, intensities, threshold=threshold, row=row) #points, intensities = thresholdPoints(points, intensities, threshold = (20, 900), row = (2,2)); io.writePoints(dct["FilteredCellsFile"], (points, intensities)) ## Check Cell detection (For the testing phase only, remove when running on the full size dataset) ####################### # if check_cell_detection: # import ClearMap.Visualization.Plot as plt # pointSource= os.path.join(BaseDirectory, FilteredCellsFile[0]); # data = plt.overlayPoints(cFosFile, pointSource, pointColor = None, **cFosFileRange); # io.writeData(os.path.join(BaseDirectory, "cells_check.tif"), data); # Transform point coordinates ############################# points = io.readPoints( dct["CorrectionResamplingPointsParameter"]["pointSource"]) points = resamplePoints(**dct["CorrectionResamplingPointsParameter"]) points = transformPoints( points, transformDirectory=dct["CorrectionAlignmentParameter"] ["resultDirectory"], indices=False, resultDirectory=None) dct["CorrectionResamplingPointsInverseParameter"]["pointSource"] = points points = resamplePointsInverse( **dct["CorrectionResamplingPointsInverseParameter"]) dct["RegistrationResamplingPointParameter"]["pointSource"] = points points = resamplePoints(**dct["RegistrationResamplingPointParameter"]) points = transformPoints( points, transformDirectory=dct["RegistrationAlignmentParameter"] ["resultDirectory"], indices=False, resultDirectory=None) io.writePoints(dct["TransformedCellsFile"], points) # Heat map generation ##################### points = io.readPoints(dct["TransformedCellsFile"]) intensities = io.readPoints(dct["FilteredCellsFile"][1]) #Without weigths: vox = voxelize(points, dct["AtlasFile"], **dct["voxelizeParameter"]) if not isinstance(vox, str): io.writeData(os.path.join(dct["OutputDirectory"], "cells_heatmap.tif"), vox.astype("int32")) #With weigths from the intensity file (here raw intensity): dct["voxelizeParameter"]["weights"] = intensities[:, 0].astype(float) vox = voxelize(points, dct["AtlasFile"], **dct["voxelizeParameter"]) if not isinstance(vox, str): io.writeData( os.path.join(dct["OutputDirectory"], "cells_heatmap_weighted.tif"), vox.astype("int32")) #Table generation: ################## #With integrated weigths from the intensity file (here raw intensity): try: ids, counts = countPointsInRegions(points, labeledImage=dct["AnnotationFile"], intensities=intensities, intensityRow=0) table = numpy.zeros(ids.shape, dtype=[("id", "int64"), ("counts", "f8"), ("name", "a256")]) table["id"] = ids table["counts"] = counts table["name"] = labelToName(ids) io.writeTable( os.path.join(dct["OutputDirectory"], "Annotated_counts_intensities.csv"), table) #Without weigths (pure cell number): ids, counts = countPointsInRegions(points, labeledImage=dct["AnnotationFile"], intensities=None) table = numpy.zeros(ids.shape, dtype=[("id", "int64"), ("counts", "f8"), ("name", "a256")]) table["id"] = ids table["counts"] = counts table["name"] = labelToName(ids) io.writeTable( os.path.join(dct["OutputDirectory"], "Annotated_counts.csv"), table) except: print("Table not generated.\n") print("Analysis Completed") return
ImageProcessingParameter = joinParameter(StackProcessingParameter, SpotDetectionParameter) #Cell detection: ################ detectCells(**ImageProcessingParameter) #Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row" #row = (0,0) : peak intensity from the raw data #row = (1,1) : peak intensity from the DoG filtered data #row = (2,2) : peak intensity from the background subtracted data #row = (3,3) : voxel size from the watershed points, intensities = thresholdPoints(points, intensities, threshold=(20, 900), row=(3, 3)) io.writePoints(FilteredCellsFile, (points, intensities)) # Transform point coordinates ############################# points = io.readPoints(CorrectionResamplingPointsParameter["pointSource"]) points = resamplePoints(**CorrectionResamplingPointsParameter) points = transformPoints( points, transformDirectory=CorrectionAlignmentParameter["resultDirectory"], indices=False, resultDirectory=None) CorrectionResamplingPointsInverseParameter["pointSource"] = points points = resamplePointsInverse(**CorrectionResamplingPointsInverseParameter) RegistrationResamplingPointParameter["pointSource"] = points points = resamplePoints(**RegistrationResamplingPointParameter) points = transformPoints( points,
def flatfieldLineFromRegression(data, sink=None, method='polynomial', reverse=None, verbose=False): """Create flat field line fit from a list of positions and intensities The fit is either to be assumed to be a Gaussian: .. math: I(x) = a \\exp^{- (x- x_0)^2 / (2 \\sigma)) + b" or follows a order 6 radial polynomial .. math: I(x) = a + b (x- x_0)^2 + c (x- x_0)^4 + d (x- x_0)^6 Arguments: data (array): intensity data as vector of intensities or (n,2) dim array of positions d=0 and intensities measurements d=1:-1 sink (str or None): destination to write the result of the fit method (str): method to fit intensity data, 'Gaussian' or 'Polynomial' reverse (bool): reverse the line fit after fitting verbose (bool): print and plot information for the fit Returns: array: fitted intensities on points """ data = io.readPoints(data) # split data if len(data.shape) == 1: x = numpy.arange(0, data.shape[0]) y = data elif len(data.shape) == 2: x = data[:, 0] y = data[:, 1:-1] else: raise RuntimeError( 'flatfieldLineFromRegression: input data not a line or array of x,i data' ) #calculate mean of the intensity measurements ym = numpy.mean(y, axis=1) if verbose: plt.figure() for i in range(1, data.shape[1]): plt.plot(x, data[:, i]) plt.plot(x, ym, 'k') if method.lower() == 'polynomial': ## fit r^6 mean = sum(ym * x) / sum(ym) def f(x, m, a, b, c, d): return a + b * (x - m)**2 + c * (x - m)**4 + d * (x - m)**6 popt, pcov = curve_fit(f, x, ym, p0=(mean, 1, 1, 1, .1)) m = popt[0] a = popt[1] b = popt[2] c = popt[3] d = popt[4] if verbose: print "polynomial fit: %f + %f (x- %f)^2 + %f (x- %f)^4 + %f (x- %f)^6" % ( a, b, m, c, m, d, m) def fopt(x): return f(x, m=m, a=a, b=b, c=c, d=d) flt = map(fopt, range(0, int(x[-1]))) else: ## Gaussian fit mean = sum(ym * x) / sum(ym) sigma = sum(ym * (x - mean)**2) / (sum(ym)) def f(x, a, m, s, b): return a * numpy.exp(-(x - m)**2 / 2 / s) + b popt, pcov = curve_fit(f, x, ym, p0=(1000, mean, sigma, 400)) a = popt[0] m = popt[1] s = popt[2] b = popt[3] if verbose: print "Gaussian fit: %f exp(- (x- %f)^2 / (2 %f)) + %f" % (a, m, s, b) def fopt(x): return f(x, a=a, m=m, s=s, b=b) if reverse: flt.reverse() if verbose: plt.plot(x, flt) plt.title('flatfieldLineFromRegression') return io.writePoints(sink, flt)