Esempio n. 1
0
def classifyPixel(project,
                  source,
                  sink=None,
                  processingDirectory=None,
                  cleanup=True):
    """Run pixel classification in headless moded using a trained project file
  
  Arguments:
    project (str): ilastik project .ilp file
    source (str or array): image source
    sink (str or array or None): image sink
  
  Returns:
    str or array: classified image sink
  """

    #generate source image file if source is array
    if isinstance(source, str):
        inpfile = source
    else:
        #generate temporary file
        if processingDirectory is None:
            processingDirectory = tempfile.mkdtemp()

        inpfile = os.path.join(processingDirectory, 'ilastik.npy')
        io.writePoints(inpfile, source.transpose((2, 1, 0)))

    if isinstance(sink, str):
        outfile = sink
    else:
        #outdir  = tempfile.mkdtemp();
        #outfile = os.path.join(outdir, 'result\d*.tif');
        outfile = tempfile.mktemp('.h5')

    ilinp = fileNameToIlastikInput(inpfile)
    ilout = fileNameToIlastikOuput(outfile)

    args = '--project="' + project + '" ' + ilout + ' ' + ilinp
    run(args)

    #clean up
    if cleanup and processingDirectory is not None:
        shutil.rmtree(processingDirectory)

    if not isinstance(sink, str):
        sink = readResultH5(outfile)
        if cleanup:
            os.remove(outfile)

    return sink
Esempio n. 2
0
def classifyPixel(project, source, sink = None, processingDirectory = None, cleanup = True):
  """Run pixel classification in headless moded using a trained project file
  
  Arguments:
    project (str): ilastik project .ilp file
    source (str or array): image source
    sink (str or array or None): image sink
  
  Returns:
    str or array: classified image sink
  """
  
  #generate source image file if source is array
  if isinstance(source, str):
    inpfile = source;
  else:
    #generate temporary file
    if processingDirectory is None:
        processingDirectory = tempfile.mkdtemp();
    
    inpfile = os.path.join(processingDirectory, 'ilastik.npy')
    io.writePoints(inpfile, source.transpose((2,1,0)));
  
  if isinstance(sink, str):
    outfile = sink;
  else:
    #outdir  = tempfile.mkdtemp();
    #outfile = os.path.join(outdir, 'result\d*.tif');
    outfile = tempfile.mktemp('.h5');
  
  ilinp = fileNameToIlastikInput(inpfile);
  ilout = fileNameToIlastikOuput(outfile);

  args = '--project="' + project + '" ' + ilout + ' ' + ilinp;
  run(args);
  
  #clean up 
  if cleanup and processingDirectory is not None:
    shutil.rmtree(processingDirectory);
  
  if not isinstance(sink, str):
    sink = readResultH5(outfile);
    if cleanup:
      os.remove(outfile);
  
  return sink;
Esempio n. 3
0
def parallelProcessStack(source, x = all, y = all, z = all, sink = None,
                         processes = 2, chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15,
                         chunkOptimization = True, chunkOptimizationSize = all, 
                         function = noProcessing, join = joinPoints, verbose = False, **parameter):
    """Parallel process a image stack
    
    Main routine that distributes image processing on paralllel processes.
       
    Arguments:
        source (str): image source
        x,y,z (tuple or all): range specifications
        sink (str or None): destination for the result
        processes (int): number of parallel processes
        chunkSizeMax (int): maximal size of a sub-stack
        chunkSizeMin (int): minial size of a sub-stack
        chunkOverlap (int): minimal sub-stack overlap
        chunkOptimization (bool): optimize chunck sizes to best fit number of processes
        chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing
        function (function): the main image processing script
        join (function): the fuction to join the results from the image processing script
        verbose (bool): print information on sub-stack generation
        
    Returns:
        str or array: results of the image processing
    """     
    
    subStacks = calculateSubStacks(source, x = x, y = y, z = z, 
                                   processes = processes, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap,
                                   chunkOptimization = chunkOptimization, chunkOptimizationSize = chunkOptimizationSize, verbose = verbose);
                                   
    nSubStacks = len(subStacks);
    if verbose:
        print "Number of SubStacks: %d" % nSubStacks;
                                       
    #for i in range(nSubStacks):
    #    self.printSubStackInfo(subStacks[i]);
    
    argdata = [];
    for i in range(nSubStacks):
        argdata.append((function, parameter, subStacks[i], verbose));    
    #print argdata
    
    # process in parallel
    pool = Pool(processes = processes);    
    results = pool.map(_processSubStack, argdata);
    
    #print '=========== results';
    #print results;
        
    #join the results
    results = join(results, subStacks = subStacks, **parameter);
    
    #write / or return 
    return io.writePoints(sink, results);
def parallelProcessStack(source, x = all, y = all, z = all, sink = None,
                         processes = 2, chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15,
                         chunkOptimization = True, chunkOptimizationSize = all, 
                         function = noProcessing, join = joinPoints, verbose = False, **parameter):
    """Parallel process a image stack
    
    Main routine that distributes image processing on paralllel processes.
       
    Arguments:
        source (str): image source
        x,y,z (tuple or all): range specifications
        sink (str or None): destination for the result
        processes (int): number of parallel processes
        chunkSizeMax (int): maximal size of a sub-stack
        chunkSizeMin (int): minial size of a sub-stack
        chunkOverlap (int): minimal sub-stack overlap
        chunkOptimization (bool): optimize chunck sizes to best fit number of processes
        chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing
        function (function): the main image processing script
        join (function): the fuction to join the results from the image processing script
        verbose (bool): print information on sub-stack generation
        
    Returns:
        str or array: results of the image processing
    """     
    
    subStacks = calculateSubStacks(source, x = x, y = y, z = z, 
                                   processes = processes, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap,
                                   chunkOptimization = chunkOptimization, chunkOptimizationSize = chunkOptimizationSize, verbose = verbose);
                                   
    nSubStacks = len(subStacks);
    if verbose:
        print("Number of SubStacks: %d" % nSubStacks);
                                       
    #for i in range(nSubStacks):
    #    self.printSubStackInfo(subStacks[i]);
    
    argdata = [];
    for i in range(nSubStacks):
        argdata.append((function, parameter, subStacks[i], verbose));    
    #print argdata
    
    # process in parallel
    pool = Pool(processes = processes);    
    results = pool.map(_processSubStack, argdata);
    
    #print '=========== results';
    #print results;
        
    #join the results
    results = join(results, subStacks = subStacks, **parameter);
    
    #write / or return 
    return io.writePoints(sink, results);
Esempio n. 5
0
def join_results_from_cluster_helper(source,
                                     x=all,
                                     y=all,
                                     z=all,
                                     sink=None,
                                     chunkSizeMax=100,
                                     chunkSizeMin=30,
                                     chunkOverlap=15,
                                     function=noProcessing,
                                     join=joinPoints,
                                     verbose=False,
                                     **parameter):

    #calc num of substacks
    subStacks = calculateSubStacks(source,
                                   x=x,
                                   y=y,
                                   z=z,
                                   processes=1,
                                   chunkSizeMax=chunkSizeMax,
                                   chunkSizeMin=chunkSizeMin,
                                   chunkOverlap=chunkOverlap,
                                   chunkOptimization=False,
                                   verbose=verbose)

    #load all cell detection job results
    if type(sink) == tuple:
        pckfld = os.path.join(sink[0][:sink[0].rfind("/")], "cell_detection")
        makedir(pckfld)
    elif type(sink) == str:
        pckfld = os.path.join(sink[:sink.rfind("/")], "cell_detection")
        makedir(pckfld)

    results = []
    fls = listdirfull(pckfld)
    fls.sort()
    for fl in fls:
        if "~" not in fl:
            with open(fl, "rb") as pckl:
                results.append(pickle.load(pckl))
                pckl.close()
    print(results[0])
    #reformat
    results = [xx for yy in results for xx in yy]
    #join the results
    print("Length of results: {}".format(len(results)))
    results = join(results, subStacks=subStacks, **parameter)

    #write / or return
    return io.writePoints(sink, results)
Esempio n. 6
0
def sequentiallyProcessStack(source, x = all, y = all, z = all, sink = None,
                             chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15,
                             function = noProcessing, join = joinPoints, verbose = False, **parameter):
    """Sequential image processing on a stack

    Main routine that sequentially processes a large image on sub-stacks.

    Arguments:
        source (str): image source
        x,y,z (tuple or all): range specifications
        sink (str or None): destination for the result
        processes (int): number of parallel processes
        chunkSizeMax (int): maximal size of a sub-stack
        chunkSizeMin (int): minial size of a sub-stack
        chunkOverlap (int): minimal sub-stack overlap
        chunkOptimization (bool): optimize chunck sizes to best fit number of processes
        chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing
        function (function): the main image processing script
        join (function): the fuction to join the results from the image processing script
        verbose (bool): print information on sub-stack generation

    Returns:
        str or array: results of the image processing
    """
    #determine z ranges

    subStacks = calculateSubStacks(source, x = x, y = y, z = z,
                                   processes = 1, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap,
                                   chunkOptimization = False, verbose = verbose);

    nSubStacks = len(subStacks);
    #print nSubStacks;

    argdata = [];
    for i in range(nSubStacks):
        argdata.append((function, parameter, subStacks[i], verbose));

    #run sequentially
    results = [];
    for i in range(nSubStacks):
        results.append(_processSubStack(argdata[i]));

    #join the results
    results = join(results, subStacks = subStacks, **parameter);

    #write / or return
    return io.writePoints(sink, results);
def output_analysis_helper(threshold=(20, 900), row=(3, 3), **params):
    '''
    Function to change elastix result directory before running 'step 6' i.e. point transformix to atlas.
    '''
    dct = pth_update(set_parameters_for_clearmap(**params))

    dct['RegistrationAlignmentParameter']["resultDirectory"] = os.path.join(
        params["outputdirectory"],
        'clearmap_cluster_output/elastix_auto_to_sim_atlas')

    points, intensities = io.readPoints(
        dct['ImageProcessingParameter']["sink"])

    #Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row"
    #row = (0,0) : peak intensity from the raw data
    #row = (1,1) : peak intensity from the DoG filtered data
    #row = (2,2) : peak intensity from the background subtracted data
    #row = (3,3) : voxel size from the watershed
    points, intensities = thresholdPoints(points,
                                          intensities,
                                          threshold=threshold,
                                          row=row)
    #points, intensities = thresholdPoints(points, intensities, threshold = (20, 900), row = (2,2));
    io.writePoints(dct['FilteredCellsFile'], (points, intensities))

    # Transform point coordinates
    #############################
    points = io.readPoints(
        dct['CorrectionResamplingPointsParameter']["pointSource"])
    points = resamplePoints(**dct['CorrectionResamplingPointsParameter'])
    points = transformPoints(
        points,
        transformDirectory=dct['CorrectionAlignmentParameter']
        ["resultDirectory"],
        indices=False,
        resultDirectory=None)
    dct['CorrectionResamplingPointsInverseParameter']["pointSource"] = points
    points = resamplePointsInverse(
        **dct['CorrectionResamplingPointsInverseParameter'])
    dct['RegistrationResamplingPointParameter']["pointSource"] = points
    points = resamplePoints(**dct['RegistrationResamplingPointParameter'])
    points = transformPoints(
        points,
        transformDirectory=dct['RegistrationAlignmentParameter']
        ["resultDirectory"],
        indices=False,
        resultDirectory=None)
    io.writePoints(dct['TransformedCellsFile'], points)

    # Heat map generation
    #####################
    points = io.readPoints(dct['TransformedCellsFile'])
    intensities = io.readPoints(dct['FilteredCellsFile'][1])

    #Without weigths:
    vox = voxelize(points, dct['AtlasFile'], **dct['voxelizeParameter'])
    if not isinstance(vox, basestring):
        io.writeData(os.path.join(dct['OutputDirectory'], 'cells_heatmap.tif'),
                     vox.astype('int32'))

    #With weigths from the intensity file (here raw intensity):
    dct['voxelizeParameter']["weights"] = intensities[:, 0].astype(float)
    vox = voxelize(points, dct['AtlasFile'], **dct['voxelizeParameter'])
    if not isinstance(vox, basestring):
        io.writeData(
            os.path.join(dct['OutputDirectory'], 'cells_heatmap_weighted.tif'),
            vox.astype('int32'))

    #Table generation:
    ##################
    #With integrated weigths from the intensity file (here raw intensity):
    ids, counts = countPointsInRegions(points,
                                       labeledImage=dct['AnnotationFile'],
                                       intensities=intensities,
                                       intensityRow=0)
    table = np.zeros(ids.shape,
                     dtype=[('id', 'int64'), ('counts', 'f8'),
                            ('name', 'a256')])
    table["id"] = ids
    table["counts"] = counts
    table["name"] = labelToName(ids)
    io.writeTable(
        os.path.join(dct['OutputDirectory'],
                     'Annotated_counts_intensities.csv'), table)

    #Without weigths (pure cell number):
    ids, counts = countPointsInRegions(points,
                                       labeledImage=dct['AnnotationFile'],
                                       intensities=None)
    table = np.zeros(ids.shape,
                     dtype=[('id', 'int64'), ('counts', 'f8'),
                            ('name', 'a256')])
    table["id"] = ids
    table["counts"] = counts
    table["name"] = labelToName(ids)
    io.writeTable(os.path.join(dct['OutputDirectory'], 'Annotated_counts.csv'),
                  table)

    print('Analysis Completed')

    return
Esempio n. 8
0
def transformPoints(source,
                    sink=None,
                    transformParameterFile=None,
                    transformDirectory=None,
                    indices=True,
                    resultDirectory=None,
                    tmpFile=None):
    """Transform coordinates math:`x` via elastix estimated transformation to :math:`T(x)`

    Note the transformation is from the fixed image coorindates to the moving image coordiantes.

    Arguments:
        source (str): source of the points
        sink (str or None): sink for transformed points
        transformParameterFile (str or None): parameter file for the primary transformation, if None, the file is determined from the transformDirectory.
        transformDirectory (str or None): result directory of elastix alignment, if None the transformParameterFile has to be given.
        indices (bool): if True use points as pixel coordinates otherwise spatial coordinates.
        resultDirectory (str or None): elastic result directory
        tmpFile (str or None): file name for the elastix point file.

    Returns:
        array or str: array or file name of transformed points
    """

    global TransformixBinary

    checkElastixInitialized()
    global ElastixSettings

    if tmpFile == None:
        tmpFile = os.path.join(tempfile.gettempdir(), 'elastix_input.txt')

    # write text file
    if isinstance(source, basestring):

        #check if we have elastix signature
        with open(source) as f:
            line = f.readline()
            f.close()

            if line[:5] == 'point' or line[:5] != 'index':
                txtfile = source
            else:
                points = io.readPoints(source)
                #points = points[:,[1,0,2]];
                txtfile = tmpFile
                writePoints(txtfile, points)

    elif isinstance(source, numpy.ndarray):
        txtfile = tmpFile
        #points = source[:,[1,0,2]];
        writePoints(txtfile, source)

    else:
        raise RuntimeError('transformPoints: source not string or array!')

    if resultDirectory == None:
        outdirname = os.path.join(tempfile.gettempdir(), 'elastix_output')
    else:
        outdirname = resultDirectory

    if not os.path.exists(outdirname):
        os.makedirs(outdirname)

    if transformParameterFile == None:
        if transformDirectory == None:
            RuntimeError(
                'neither alignment directory and transformation parameter file specified!'
            )
        transformparameterdir = transformDirectory
        transformparameterfile = getTransformParameterFile(
            transformparameterdir)
    else:
        transformparameterdir = os.path.split(transformParameterFile)
        transformparameterdir = transformparameterdir[0]
        transformparameterfile = transformParameterFile

    #transform
    #make path in parameterfiles absolute
    setPathTransformParameterFiles(transformparameterdir)

    #run transformix
    cmd = TransformixBinary + ' -threads 8 -def ' + txtfile + ' -out ' + outdirname + ' -tp ' + transformparameterfile
    res = os.system(cmd)

    if res != 0:
        raise RuntimeError('failed executing ' + cmd)

    #read data / file
    if sink == []:
        return io.path.join(outdirname, 'outputpoints.txt')

    else:
        #read coordinates
        transpoints = parseElastixOutputPoints(os.path.join(
            outdirname, 'outputpoints.txt'),
                                               indices=indices)

        #correct x,y,z to y,x,z
        #transpoints = transpoints[:,[1,0,2]];

        #cleanup
        for f in os.listdir(outdirname):
            os.remove(os.path.join(outdirname, f))
        os.rmdir(outdirname)

        return io.writePoints(sink, transpoints)
def sequentiallyProcessStack(source, x = all, y = all, z = all, sink = None,
                             chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15,
                             function = noProcessing, join = joinPoints, verbose = False, **parameter):
    """Sequential image processing on a stack
    
    Main routine that sequentially processes a large image on sub-stacks.
       
    Arguments:
        source (str): image source
        x,y,z (tuple or all): range specifications
        sink (str or None): destination for the result
        processes (int): number of parallel processes
        chunkSizeMax (int): maximal size of a sub-stack
        chunkSizeMin (int): minial size of a sub-stack
        chunkOverlap (int): minimal sub-stack overlap
        chunkOptimization (bool): optimize chunck sizes to best fit number of processes
        chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing
        function (function): the main image processing script
        join (function): the fuction to join the results from the image processing script
        verbose (bool): print information on sub-stack generation
        
    Returns:
        str or array: results of the image processing
    """     
    #determine z ranges  
    
    subStacks = calculateSubStacks(source, x = x, y = y, z = z, 
                                   processes = 1, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap,  
                                   chunkOptimization = False, verbose = verbose);
    
    nSubStacks = len(subStacks);
    #print nSubStacks;    
    
    argdata = [];
    for i in range(nSubStacks):
        argdata.append((function, parameter, subStacks[i], verbose));    
    
    #run sequentially
    results = [];
    for i in range(nSubStacks):
        results.append(_processSubStack(argdata[i]));
    
    #join the results
    results = join(results, subStacks = subStacks, **parameter);
    
    #write / or return 
    return io.writePoints(sink, results);










### Pickle does not like classes:

## sub stack information
#class SubStack(object):
#    """Class containing all info of a sub stack usefull for the image processing and result joining functions"""
#    
#    # sub stack id
#    stackId = None;
#    
#    # number of stacks
#    nStacks = None;
#    
#    # tuple of x,y,z range of this sub stack
#    z = all; 
#    x = all;
#    y = all;
#    
#    #original source
#    source = None;    
#    
#    # tuple of center point of the overlaping regions
#    zCenters = None;
#    
#    # tuple of z indices that would generate full image without overlaps
#    zCenterIndices = None;
#    
#    # tuple of z indices in the sub image as returned by readData that would generate full image without overlaps
#    zSubCenterIndices = None;
#    
#    
#    def __init__(slf, stackId = 0, nStacks = 1, source = None, x = all, y = all, z = all, zCenters = all, zCenterIndices = all):
#        slf.stackId = stackId;
#        slf.nStacks = nStacks;
#        slf.source = source; 
#        slf.x = x;
#        slf.y = y;
#        slf.z = z;
#        slf.zCenters = zCenters;
#        slf.zCenterIndices = zCenterIndices;
#        if not zCenterIndices is all and not z is all:
#            slf.zSubCenterIndices = (c - z[0] for c in zCenterIndices);
#        else:
#            slf.zSubCenterIndices = all;
#       
#    
#def printSubStackInfo(slf, out = sys.stdout):
#    out.write("Sub Stack: %d / %d\n" % (slf.stackId, slf.nStacks));
#    out.write("source:         %s\n" %       slf.source);
#    out.write("x,y,z:          %s, %s, %s\n" % (str(slf.x), str(slf.y), str(slf.z)));
#    out.write("zCenters:       %s\n" %       str(slf.zCenters));   
#    out.write("zCenterIndices: %s\n" %       str(slf.zCenterIndices));
Esempio n. 10
0
def transformPoints(source, sink = None, transformParameterFile = None, transformDirectory = None, indices = True, resultDirectory = None, tmpFile = None):
    """Transform coordinates math:`x` via elastix estimated transformation to :math:`T(x)`

    Note the transformation is from the fixed image coorindates to the moving image coordiantes.
    
    Arguments:
        source (str): source of the points
        sink (str or None): sink for transformed points
        transformParameterFile (str or None): parameter file for the primary transformation, if None, the file is determined from the transformDirectory.
        transformDirectory (str or None): result directory of elastix alignment, if None the transformParameterFile has to be given.
        indices (bool): if True use points as pixel coordinates otherwise spatial coordinates.
        resultDirectory (str or None): elastic result directory
        tmpFile (str or None): file name for the elastix point file.
        
    Returns:
        array or str: array or file name of transformed points
    """
        
    global TransformixBinary;    
    
    checkElastixInitialized();    
    global ElastixSettings;

    if tmpFile == None:
        tmpFile = os.path.join(tempfile.tempdir, 'elastix_input.txt');

    # write text file
    if isinstance(source, basestring):
        
        #check if we have elastix signature                 
        with open(source) as f:
            line = f.readline();
            f.close();
            
            if line[:5] == 'point' or line[:5] != 'index':
                txtfile = source;
            else:                
                points = io.readPoints(source);
                #points = points[:,[1,0,2]];
                txtfile = tmpFile;
                writePoints(txtfile, points); 
    
    elif isinstance(source, numpy.ndarray):
        txtfile = tmpFile;
        #points = source[:,[1,0,2]];
        writePoints(txtfile, source);
        
    else:
        raise RuntimeError('transformPoints: source not string or array!');
    
    
    if resultDirectory == None:
        outdirname = os.path.join(tempfile.tempdir, 'elastix_output');
    else:
        outdirname = resultDirectory;
        
    if not os.path.exists(outdirname):
        os.makedirs(outdirname);
        
    
    if transformParameterFile == None:
        if transformDirectory == None:
            RuntimeError('neither alignment directory and transformation parameter file specified!'); 
        transformparameterdir = transformDirectory
        transformparameterfile = getTransformParameterFile(transformparameterdir);
    else:
        transformparameterdir = os.path.split(transformParameterFile);
        transformparameterdir  = transformparameterdir[0];
        transformparameterfile = transformParameterFile;
    
    #transform
    #make path in parameterfiles absolute
    setPathTransformParameterFiles(transformparameterdir);
    
    #run transformix   
    cmd = TransformixBinary + ' -def ' + txtfile + ' -out ' + outdirname + ' -tp ' + transformparameterfile;
    res = os.system(cmd);
    
    if res != 0:
        raise RuntimeError('failed executing ' + cmd);
    
    
    #read data / file 
    if sink == []:
        return io.path.join(outdirname, 'outputpoints.txt')
    
    else:
        #read coordinates
        transpoints = parseElastixOutputPoints(os.path.join(outdirname, 'outputpoints.txt'), indices = indices);

        #correct x,y,z to y,x,z
        #transpoints = transpoints[:,[1,0,2]];     
        
        #cleanup
        for f in os.listdir(outdirname):
            os.remove(os.path.join(outdirname, f));
        os.rmdir(outdirname)
        
        return io.writePoints(sink, transpoints);
Esempio n. 11
0
    #pdb.set_trace()
    #io.writeData(os.path.join(homeDir, 'Results/OverlayWatershed.tif'), overlay_Img);

    overlay_Img = plt.fredOverlayPoints(cfos_fn,
                                        points,
                                        pointColor=[200, 0, 0])
    io.writeData(os.path.join(resultDir, fName + '_PointsOriginalImg.tif'),
                 overlay_Img)

    overlay_Img = plt.overlayPoints(imgD, points, pointColor=[200, 0, 0])
    io.writeData(os.path.join(resultDir, fName + '_PointsFilterDoG.tif'),
                 overlay_Img)

    #Convert points from 3d to 2d
    points = numpy.delete(points, 2, axis=1)
    io.writePoints(points_fn, points)

    ##############################################################333

    imgR = io.readData(auto_fn)
    imgR = imgR[..., numpy.newaxis]
    imgR = imgR.astype('int16')
    imgR, scaleFactor = resampleData(imgR,
                                     auto_R_fn,
                                     orientation=(1, 2, 3),
                                     dataSizeSink=None,
                                     resolutionSource=pixelRes,
                                     resolutionSink=(25, 25, 1),
                                     processingDirectory=None,
                                     processes=4,
                                     cleanup=True,
Esempio n. 12
0
             if result == "ENDPROCESS": print("Jobid > # of jobs required, ending job")
         except:
             print("Jobid > # of jobs required, ending job")
     
     print("\n           finished step 4 - cell detection \n")                                     
 ###########################################STEP 5########################################################
     
     out = join_results_from_cluster_helper(**dct["ImageProcessingParameter"])    
     print("\n           finished step 5 \n")   
 ###########################################STEP 6########################################################
     
     threshold = (500,3000); row = (2,2)
     points, intensities = io.readPoints(dct["ImageProcessingParameter"]["sink"])
     #Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row"
     #row = (0,0) : peak intensity from the raw data
     #row = (1,1) : peak intensity from the DoG filtered data
     #row = (2,2) : peak intensity from the background subtracted data
     #row = (3,3) : voxel size from the watershed
     points, intensities = thresholdPoints(points, intensities, threshold = threshold, 
                         row = row)
     #change dst to match parameters sweeped
     dst = (os.path.join(brain, 
     "clearmap_cluster_output/cells_rBP%s_fIPmethod%s_fIPsize%s_dCSP%s.npy" % (rBP_size,
         fIP_method, fIP_size, dCSP_threshold)),
     os.path.join(brain, 
     "clearmap_cluster_output/intensities_rBP%s_fIPmethod%s_fIPsize%s_dCSP%s.npy" % (rBP_size,
         fIP_method, fIP_size, dCSP_threshold)))
     
     io.writePoints(dst, (points, intensities));
     print("\n           finished step 6 \n")   
                                 
Esempio n. 13
0
#io.writeData(os.path.join(homeDir, 'Results/OverlayWatershed.tif'), overlay_Img);

overlay_Img = plt.fredOverlayPoints(input_fn, points, pointColor=[200, 0, 0])
io.writeData(os.path.join(homeDir, 'Results/PointsOriginalImg.tif'),
             overlay_Img)

overlay_Img = plt.fredOverlayPoints(os.path.join(homeDir,
                                                 'Results/FilterDoG.tif'),
                                    points,
                                    pointColor=[200, 0, 0])
io.writeData(os.path.join(homeDir, 'Results/PointsFilterDoG.tif'), overlay_Img)
#################################################################################################

#Convert points from 3d to 2d
points = numpy.delete(points, 2, axis=1)
io.writePoints(points_fn, points)
#Resampled Img
imgR = io.readData(auto_fn)
imgR = imgR[..., numpy.newaxis]
imgR = imgR.astype('int16')
imgR, scaleFactor = resampleData(imgR,
                                 auto_R_fn,
                                 orientation=(1, 2, 3),
                                 dataSizeSink=None,
                                 resolutionSource=(2.3719, 2.3719, 1),
                                 resolutionSink=(25, 25, 1),
                                 processingDirectory=None,
                                 processes=4,
                                 cleanup=True,
                                 verbose=True,
                                 interpolation='linear')
def flatfieldLineFromRegression(data, sink = None, method = 'polynomial', reverse = None, verbose = False):
    """Create flat field line fit from a list of positions and intensities
    
    The fit is either to be assumed to be a Gaussian:
    
    .. math:
        I(x) = a \\exp^{- (x- x_0)^2 / (2 \\sigma)) + b"
        
    or follows a order 6 radial polynomial
        
    .. math:
        I(x) = a + b (x- x_0)^2 + c (x- x_0)^4 + d (x- x_0)^6
    
    Arguments:
        data (array): intensity data as vector of intensities or (n,2) dim array of positions d=0 and intensities measurements d=1:-1 
        sink (str or None): destination to write the result of the fit
        method (str): method to fit intensity data, 'Gaussian' or 'Polynomial'
        reverse (bool): reverse the line fit after fitting
        verbose (bool): print and plot information for the fit
        
    Returns:
        array: fitted intensities on points
    """
    
    data = io.readPoints(data);

    # split data
    if len(data.shape) == 1:
        x = numpy.arange(0, data.shape[0]);
        y = data;
    elif len(data.shape) == 2:
        x = data[:,0]
        y = data[:,1:-1];
    else:
        raise RuntimeError('flatfieldLineFromRegression: input data not a line or array of x,i data');
    
    #calculate mean of the intensity measurements
    ym = numpy.mean(y, axis = 1);

    if verbose:
        plt.figure()
        for i in range(1,data.shape[1]):
            plt.plot(x, data[:,i]);
        plt.plot(x, ym, 'k');
    
    
    if method.lower() == 'polynomial':
        ## fit r^6
        mean = sum(ym * x)/sum(ym)

        def f(x,m,a,b,c,d):
            return a + b * (x-m)**2 + c * (x-m)**4 + d * (x-m)**6;
        
        popt, pcov = curve_fit(f, x, ym, p0 = (mean, 1, 1, 1, .1));
        m = popt[0]; a = popt[1]; b = popt[2];
        c = popt[3]; d = popt[4];

        if verbose:
            print "polynomial fit: %f + %f (x- %f)^2 + %f (x- %f)^4 + %f (x- %f)^6" % (a, b, m, c, m, d, m);

        def fopt(x):
            return f(x, m = m, a = a, b = b, c = c, d = d);
        
        flt = map(fopt, range(0, int(x[-1])));
    
    else: 
        ## Gaussian fit
        
        mean = sum(ym * x)/sum(ym)
        sigma = sum(ym * (x-mean)**2)/(sum(ym))
        
        def f(x, a, m, s, b):
            return a * numpy.exp(- (x - m)**2 / 2 / s) + b;
            
        
        popt, pcov = curve_fit(f, x, ym, p0 = (1000, mean, sigma, 400));
        a = popt[0]; m = popt[1]; s = popt[2]; b = popt[3];

        if verbose:
            print "Gaussian fit: %f exp(- (x- %f)^2 / (2 %f)) + %f" % (a, m, s, b);

        def fopt(x):
            return f(x, a = a, m = m, s = s, b = b);
    
    if reverse:
        flt.reverse();
    
    if verbose:
        plt.plot(x, flt);
        plt.title('flatfieldLineFromRegression')
    
    return io.writePoints(sink, flt);
Esempio n. 15
0
def output_analysis(
        threshold=(20, 900), row=(3, 3), check_cell_detection=False, **params):
    """Wrapper for analysis:

    Inputs
    -------------------
    Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row"
    Row:
        row = (0,0) : peak intensity from the raw data
        row = (1,1) : peak intensity from the DoG filtered data
        row = (2,2) : peak intensity from the background subtracted data
        row = (3,3) : voxel size from the watershed

    Check Cell detection: (For the testing phase only, remove when running on the full size dataset)
    """
    dct = pth_update(set_parameters_for_clearmap(**params))

    points, intensities = io.readPoints(
        dct["ImageProcessingParameter"]["sink"])

    #Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row"
    #row = (0,0) : peak intensity from the raw data
    #row = (1,1) : peak intensity from the DoG filtered data
    #row = (2,2) : peak intensity from the background subtracted data
    #row = (3,3) : voxel size from the watershed
    points, intensities = thresholdPoints(points,
                                          intensities,
                                          threshold=threshold,
                                          row=row)
    #points, intensities = thresholdPoints(points, intensities, threshold = (20, 900), row = (2,2));
    io.writePoints(dct["FilteredCellsFile"], (points, intensities))

    ## Check Cell detection (For the testing phase only, remove when running on the full size dataset)
    #######################
    #    if check_cell_detection:
    #        import ClearMap.Visualization.Plot as plt
    #        pointSource= os.path.join(BaseDirectory, FilteredCellsFile[0]);
    #        data = plt.overlayPoints(cFosFile, pointSource, pointColor = None, **cFosFileRange);
    #        io.writeData(os.path.join(BaseDirectory, "cells_check.tif"), data);

    # Transform point coordinates
    #############################
    points = io.readPoints(
        dct["CorrectionResamplingPointsParameter"]["pointSource"])
    points = resamplePoints(**dct["CorrectionResamplingPointsParameter"])
    points = transformPoints(
        points,
        transformDirectory=dct["CorrectionAlignmentParameter"]
        ["resultDirectory"],
        indices=False,
        resultDirectory=None)
    dct["CorrectionResamplingPointsInverseParameter"]["pointSource"] = points
    points = resamplePointsInverse(
        **dct["CorrectionResamplingPointsInverseParameter"])
    dct["RegistrationResamplingPointParameter"]["pointSource"] = points
    points = resamplePoints(**dct["RegistrationResamplingPointParameter"])
    points = transformPoints(
        points,
        transformDirectory=dct["RegistrationAlignmentParameter"]
        ["resultDirectory"],
        indices=False,
        resultDirectory=None)
    io.writePoints(dct["TransformedCellsFile"], points)

    # Heat map generation
    #####################
    points = io.readPoints(dct["TransformedCellsFile"])
    intensities = io.readPoints(dct["FilteredCellsFile"][1])

    #Without weigths:
    vox = voxelize(points, dct["AtlasFile"], **dct["voxelizeParameter"])
    if not isinstance(vox, str):
        io.writeData(os.path.join(dct["OutputDirectory"], "cells_heatmap.tif"),
                     vox.astype("int32"))

    #With weigths from the intensity file (here raw intensity):
    dct["voxelizeParameter"]["weights"] = intensities[:, 0].astype(float)
    vox = voxelize(points, dct["AtlasFile"], **dct["voxelizeParameter"])
    if not isinstance(vox, str):
        io.writeData(
            os.path.join(dct["OutputDirectory"], "cells_heatmap_weighted.tif"),
            vox.astype("int32"))

    #Table generation:
    ##################
    #With integrated weigths from the intensity file (here raw intensity):
    try:
        ids, counts = countPointsInRegions(points,
                                           labeledImage=dct["AnnotationFile"],
                                           intensities=intensities,
                                           intensityRow=0)
        table = numpy.zeros(ids.shape,
                            dtype=[("id", "int64"), ("counts", "f8"),
                                   ("name", "a256")])
        table["id"] = ids
        table["counts"] = counts
        table["name"] = labelToName(ids)
        io.writeTable(
            os.path.join(dct["OutputDirectory"],
                         "Annotated_counts_intensities.csv"), table)

        #Without weigths (pure cell number):
        ids, counts = countPointsInRegions(points,
                                           labeledImage=dct["AnnotationFile"],
                                           intensities=None)
        table = numpy.zeros(ids.shape,
                            dtype=[("id", "int64"), ("counts", "f8"),
                                   ("name", "a256")])
        table["id"] = ids
        table["counts"] = counts
        table["name"] = labelToName(ids)
        io.writeTable(
            os.path.join(dct["OutputDirectory"], "Annotated_counts.csv"),
            table)
    except:
        print("Table not generated.\n")

    print("Analysis Completed")

    return
ImageProcessingParameter = joinParameter(StackProcessingParameter,
                                         SpotDetectionParameter)
#Cell detection:
################
detectCells(**ImageProcessingParameter)

#Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row"
#row = (0,0) : peak intensity from the raw data
#row = (1,1) : peak intensity from the DoG filtered data
#row = (2,2) : peak intensity from the background subtracted data
#row = (3,3) : voxel size from the watershed
points, intensities = thresholdPoints(points,
                                      intensities,
                                      threshold=(20, 900),
                                      row=(3, 3))
io.writePoints(FilteredCellsFile, (points, intensities))
# Transform point coordinates
#############################
points = io.readPoints(CorrectionResamplingPointsParameter["pointSource"])
points = resamplePoints(**CorrectionResamplingPointsParameter)
points = transformPoints(
    points,
    transformDirectory=CorrectionAlignmentParameter["resultDirectory"],
    indices=False,
    resultDirectory=None)
CorrectionResamplingPointsInverseParameter["pointSource"] = points
points = resamplePointsInverse(**CorrectionResamplingPointsInverseParameter)
RegistrationResamplingPointParameter["pointSource"] = points
points = resamplePoints(**RegistrationResamplingPointParameter)
points = transformPoints(
    points,
def flatfieldLineFromRegression(data,
                                sink=None,
                                method='polynomial',
                                reverse=None,
                                verbose=False):
    """Create flat field line fit from a list of positions and intensities
    
    The fit is either to be assumed to be a Gaussian:
    
    .. math:
        I(x) = a \\exp^{- (x- x_0)^2 / (2 \\sigma)) + b"
        
    or follows a order 6 radial polynomial
        
    .. math:
        I(x) = a + b (x- x_0)^2 + c (x- x_0)^4 + d (x- x_0)^6
    
    Arguments:
        data (array): intensity data as vector of intensities or (n,2) dim array of positions d=0 and intensities measurements d=1:-1 
        sink (str or None): destination to write the result of the fit
        method (str): method to fit intensity data, 'Gaussian' or 'Polynomial'
        reverse (bool): reverse the line fit after fitting
        verbose (bool): print and plot information for the fit
        
    Returns:
        array: fitted intensities on points
    """

    data = io.readPoints(data)

    # split data
    if len(data.shape) == 1:
        x = numpy.arange(0, data.shape[0])
        y = data
    elif len(data.shape) == 2:
        x = data[:, 0]
        y = data[:, 1:-1]
    else:
        raise RuntimeError(
            'flatfieldLineFromRegression: input data not a line or array of x,i data'
        )

    #calculate mean of the intensity measurements
    ym = numpy.mean(y, axis=1)

    if verbose:
        plt.figure()
        for i in range(1, data.shape[1]):
            plt.plot(x, data[:, i])
        plt.plot(x, ym, 'k')

    if method.lower() == 'polynomial':
        ## fit r^6
        mean = sum(ym * x) / sum(ym)

        def f(x, m, a, b, c, d):
            return a + b * (x - m)**2 + c * (x - m)**4 + d * (x - m)**6

        popt, pcov = curve_fit(f, x, ym, p0=(mean, 1, 1, 1, .1))
        m = popt[0]
        a = popt[1]
        b = popt[2]
        c = popt[3]
        d = popt[4]

        if verbose:
            print "polynomial fit: %f + %f (x- %f)^2 + %f (x- %f)^4 + %f (x- %f)^6" % (
                a, b, m, c, m, d, m)

        def fopt(x):
            return f(x, m=m, a=a, b=b, c=c, d=d)

        flt = map(fopt, range(0, int(x[-1])))

    else:
        ## Gaussian fit

        mean = sum(ym * x) / sum(ym)
        sigma = sum(ym * (x - mean)**2) / (sum(ym))

        def f(x, a, m, s, b):
            return a * numpy.exp(-(x - m)**2 / 2 / s) + b

        popt, pcov = curve_fit(f, x, ym, p0=(1000, mean, sigma, 400))
        a = popt[0]
        m = popt[1]
        s = popt[2]
        b = popt[3]

        if verbose:
            print "Gaussian fit: %f exp(- (x- %f)^2 / (2 %f)) + %f" % (a, m, s,
                                                                       b)

        def fopt(x):
            return f(x, a=a, m=m, s=s, b=b)

    if reverse:
        flt.reverse()

    if verbose:
        plt.plot(x, flt)
        plt.title('flatfieldLineFromRegression')

    return io.writePoints(sink, flt)
Esempio n. 18
0
def sequentiallyProcessStack(source, x = all, y = all, z = all, sink = None,
                             chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15,
                             function = noProcessing, join = joinPoints, verbose = False, **parameter):
    """Sequential image processing on a stack
    
    Main routine that sequentially processes a large image on sub-stacks.
       
    Arguments:
        source (str): image source
        x,y,z (tuple or all): range specifications
        sink (str or None): destination for the result
        processes (int): number of parallel processes
        chunkSizeMax (int): maximal size of a sub-stack
        chunkSizeMin (int): minial size of a sub-stack
        chunkOverlap (int): minimal sub-stack overlap
        chunkOptimization (bool): optimize chunck sizes to best fit number of processes
        chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing
        function (function): the main image processing script
        join (function): the fuction to join the results from the image processing script
        verbose (bool): print information on sub-stack generation
        
    Returns:
        str or array: results of the image processing
    """     
    #determine z ranges  
    
    subStacks = calculateSubStacks(source, x = x, y = y, z = z, 
                                   processes = 1, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap,  
                                   chunkOptimization = False, verbose = verbose);
    
    nSubStacks = len(subStacks);
    #print nSubStacks;    
    
    argdata = [];
    for i in range(nSubStacks):
        argdata.append((function, parameter, subStacks[i], verbose));    
    
    #run sequentially
    results = [];
    for i in range(nSubStacks):
        results.append(_processSubStack(argdata[i]));
    
    #join the results
    results = join(results, subStacks = subStacks, **parameter);
    
    #write / or return 
    return io.writePoints(sink, results);










### Pickle does not like classes:

## sub stack information
#class SubStack(object):
#    """Class containing all info of a sub stack usefull for the image processing and result joining functions"""
#    
#    # sub stack id
#    stackId = None;
#    
#    # number of stacks
#    nStacks = None;
#    
#    # tuple of x,y,z range of this sub stack
#    z = all; 
#    x = all;
#    y = all;
#    
#    #original source
#    source = None;    
#    
#    # tuple of center point of the overlaping regions
#    zCenters = None;
#    
#    # tuple of z indices that would generate full image without overlaps
#    zCenterIndices = None;
#    
#    # tuple of z indices in the sub image as returned by readData that would generate full image without overlaps
#    zSubCenterIndices = None;
#    
#    
#    def __init__(slf, stackId = 0, nStacks = 1, source = None, x = all, y = all, z = all, zCenters = all, zCenterIndices = all):
#        slf.stackId = stackId;
#        slf.nStacks = nStacks;
#        slf.source = source; 
#        slf.x = x;
#        slf.y = y;
#        slf.z = z;
#        slf.zCenters = zCenters;
#        slf.zCenterIndices = zCenterIndices;
#        if not zCenterIndices is all and not z is all:
#            slf.zSubCenterIndices = (c - z[0] for c in zCenterIndices);
#        else:
#            slf.zSubCenterIndices = all;
#       
#    
#def printSubStackInfo(slf, out = sys.stdout):
#    out.write("Sub Stack: %d / %d\n" % (slf.stackId, slf.nStacks));
#    out.write("source:         %s\n" %       slf.source);
#    out.write("x,y,z:          %s, %s, %s\n" % (str(slf.x), str(slf.y), str(slf.z)));
#    out.write("zCenters:       %s\n" %       str(slf.zCenters));   
#    out.write("zCenterIndices: %s\n" %       str(slf.zCenterIndices));