Beispiel #1
0
def moveTeraStitcherStackToFileList(source, sink, deleteDirectory = True, verbose = True):
  """Moves image files from TeraSticher file structure to a list of files
  
  Arguments:
    source (str): base directory of the TeraStitcher files
    sink (str): regular expression of the files to copy to
    verbose (bool): show progress

  Returns:
    str: sink regular expression
  """
  
  fns = glob.glob(os.path.join(source, '*/*/*'));
  fns = natsort.natsorted(fns);
  
  io.createDirectory(sink);
  for i,f in enumerate(fns):
    fn = filelist.fileExpressionToFileName(sink, i);
    if verbose:
      print '%s -> %s' % (f,fn)
    shutil.move(f, fn);
    
  if deleteDirectory:
    p,_ = os.path.split(fns[0]);
    p = p.split(os.path.sep);
    p = p[:-2];
    p = os.path.sep.join(p);
    shutil.rmtree(p);
  
  return sink;
Beispiel #2
0
def fileNameToIlastikOuput(filename):
  """Converts *ClearMap* file name to an argument string for use with Ilastik headless mode
  
  Arguments:
    filename (str): image file name or expression
  
  Returns:
    str: Ilastik headless ouput specifications
    
  Note:
    The output is formated accroding to the Ilastik pixel calssification output specifications
  """

  if not isValidOutputFileName(filename):
    raise RuntimeError('Ilastik: file format not compatibel with Ilastik output');
  
  if io.isFileExpression(filename):
    o = '--output_format="' + io.fileExtension(filename) + ' sequence" '+ \
        '--output_filename_format="' + filelist.fileExpressionToFileName(filename, '{slice_index}') + '"';
    return o;
    
  else: # single file
    extensionToOuput  = {'bmp' : 'bmp', 'gif' : 'gif', 'hdr' : 'hrd', 
                         'jpg' : 'jpg', 'jpeg': 'jpeg','pbm' : 'pbm', 
                         'pgm' : 'pgm', 'png' : 'png', 'pnm' : 'pnm', 'ppm' : 'ppm', 
                         'ras' : 'ras', 'tif' : 'tif', 'tiff': 'tiff','xv'  : 'xv',
                         'h5'  : 'hdf5' , 'npy' : 'numpy'};
    ext = extensionToOuput[io.fileExtension(filename)];
    o = '--output_format="' + ext +'" ' + \
        '--output_filename_format="' + filename + '"';
    return o;
def moveTeraStitcherStackToFileList(source, sink, deleteDirectory=True, verbose=True):
    """Moves image files from TeraSticher file structure to a list of files
  
  Arguments:
    source (str): base directory of the TeraStitcher files
    sink (str): regular expression of the files to copy to
    verbose (bool): show progress

  Returns:
    str: sink regular expression
  """

    fns = glob.glob(os.path.join(source, "*/*/*"))
    fns = natsort.natsorted(fns)

    io.createDirectory(sink)
    for i, f in enumerate(fns):
        fn = filelist.fileExpressionToFileName(sink, i)
        if verbose:
            print "%s -> %s" % (f, fn)
        shutil.move(f, fn)

    if deleteDirectory:
        p, _ = os.path.split(fns[0])
        p = p.split(os.path.sep)
        p = p[:-2]
        p = os.path.sep.join(p)
        shutil.rmtree(p)

    return sink
def fileNameToIlastikOuput(filename):
    """Converts *ClearMap* file name to an argument string for use with Ilastik headless mode
  
  Arguments:
    filename (str): image file name or expression
  
  Returns:
    str: Ilastik headless ouput specifications
    
  Note:
    The output is formated accroding to the Ilastik pixel calssification output specifications
  """

    if not isValidOutputFileName(filename):
        raise RuntimeError("Ilastik: file format not compatibel with Ilastik output")

    if io.isFileExpression(filename):
        o = (
            '--output_format="'
            + io.fileExtension(filename)
            + ' sequence" '
            + '--output_filename_format="'
            + filelist.fileExpressionToFileName(filename, "{slice_index}")
            + '"'
        )
        return o

    else:  # single file
        extensionToOuput = {
            "bmp": "bmp",
            "gif": "gif",
            "hdr": "hrd",
            "jpg": "jpg",
            "jpeg": "jpeg",
            "pbm": "pbm",
            "pgm": "pgm",
            "png": "png",
            "pnm": "pnm",
            "ppm": "ppm",
            "ras": "ras",
            "tif": "tif",
            "tiff": "tiff",
            "xv": "xv",
            "h5": "hdf5",
            "npy": "numpy",
        }
        ext = extensionToOuput[io.fileExtension(filename)]
        o = '--output_format="' + ext + '" ' + '--output_filename_format="' + filename + '"'
        return o
Beispiel #5
0
def location_to_module(location):
  """Returns the IO module associated with a location string.
  
  Arguments
  ---------
  location : object
    Location of the source.
      
  Returns
  -------
  module : module
    The module that handles the IO of the source specified by its location.
  """
  if fl.is_file_list(location):
    return fl;
  else:
    return filename_to_module(location);
Beispiel #6
0
def fileNameToIlastikOuput(filename):
    """Converts *ClearMap* file name to an argument string for use with Ilastik headless mode
  
  Arguments:
    filename (str): image file name or expression
  
  Returns:
    str: Ilastik headless ouput specifications
    
  Note:
    The output is formated accroding to the Ilastik pixel calssification output specifications
  """

    if not isValidOutputFileName(filename):
        raise RuntimeError(
            'Ilastik: file format not compatibel with Ilastik output')

    if io.isFileExpression(filename):
        o = '--output_format="' + io.fileExtension(filename) + ' sequence" '+ \
            '--output_filename_format="' + filelist.fileExpressionToFileName(filename, '{slice_index}') + '"'
        return o

    else:  # single file
        extensionToOuput = {
            'bmp': 'bmp',
            'gif': 'gif',
            'hdr': 'hrd',
            'jpg': 'jpg',
            'jpeg': 'jpeg',
            'pbm': 'pbm',
            'pgm': 'pgm',
            'png': 'png',
            'pnm': 'pnm',
            'ppm': 'ppm',
            'ras': 'ras',
            'tif': 'tif',
            'tiff': 'tiff',
            'xv': 'xv',
            'h5': 'hdf5',
            'npy': 'numpy'
        }
        ext = extensionToOuput[io.fileExtension(filename)]
        o = '--output_format="' + ext +'" ' + \
            '--output_filename_format="' + filename + '"'
        return o
Beispiel #7
0
def file_list(expression = None, file_list = None, sort = True, verbose = False):
  """Returns the list of files that match the tag expression.
  
  Arguments
  ---------
  expression :str
    The regular expression the file names should match.
  sort : bool
    If True, sort files naturally.
  verbose : bool
    If True, print warning if no files exists.
  
  Returns
  -------
  file_list : list of str
    The list of files that matched the expression.
  """
  return fl._file_list(expression=expression, file_list=file_list, sort=sort, verbose=verbose)
Beispiel #8
0
def fileNameToIlastikInput(filename):
  """Converts *ClearMap* file name to a string for use with Ilastik input
  
  Arguments:
    filename (str): image file name or expression
  
  Returns:
    str: Ilastik conform file name
    
  Note:
    file expressions in *ClearMap* are regular expressions but shell expressions in Ilastik.
  """

  if not isValidInputFileName(filename):
    raise RuntimeError('Ilastik: file format not compatibel with Ilastik');
  
  if io.isFileExpression(filename):
    return '"' + filelist.fileExpressionToFileName(filename, '*') + '"';
  else:
    return '"' + filename + '"';
Beispiel #9
0
def fileNameToIlastikInput(filename):
    """Converts *ClearMap* file name to a string for use with Ilastik input
  
  Arguments:
    filename (str): image file name or expression
  
  Returns:
    str: Ilastik conform file name
    
  Note:
    file expressions in *ClearMap* are regular expressions but shell expressions in Ilastik.
  """

    if not isValidInputFileName(filename):
        raise RuntimeError('Ilastik: file format not compatibel with Ilastik')

    if io.isFileExpression(filename):
        return '"' + filelist.fileExpressionToFileName(filename, '*') + '"'
    else:
        return '"' + filename + '"'
Beispiel #10
0
def copyTeraStitcherStackToFileList(source, sink, verbose = True):
  """Copies image files from TeraSticher file structure to a list of files
  
  Arguments:
    source (str): base directory of the TeraStitcher files
    sink (str): regular expression of the files to copy to
    verbose (bool): show progress

  Returns:
    str: sink regular expression
  """
  #TODO: multiple tiles !
  fns = glob.glob(os.path.join(source, '*/*/*'));
  fns = natsort.natsorted(fns);
  #print fns
  
  io.createDirectory(sink);
  for i,f in enumerate(fns):
    fn = filelist.fileExpressionToFileName(sink, i);
    if verbose:
      print '%s -> %s' % (f,fn)
    shutil.copyfile(f, fn);
  
  return sink;  
def copyTeraStitcherStackToFileList(source, sink, verbose=True):
    """Copies image files from TeraSticher file structure to a list of files
  
  Arguments:
    source (str): base directory of the TeraStitcher files
    sink (str): regular expression of the files to copy to
    verbose (bool): show progress

  Returns:
    str: sink regular expression
  """
    # TODO: multiple tiles !
    fns = glob.glob(os.path.join(source, "*/*/*"))
    fns = natsort.natsorted(fns)
    # print fns

    io.createDirectory(sink)
    for i, f in enumerate(fns):
        fn = filelist.fileExpressionToFileName(sink, i)
        if verbose:
            print "%s -> %s" % (f, fn)
        shutil.copyfile(f, fn)

    return sink
Beispiel #12
0
def resampleDataInverse(sink,
                        source=None,
                        dataSizeSource=None,
                        orientation=None,
                        resolutionSource=(4.0625, 4.0625, 3),
                        resolutionSink=(25, 25, 25),
                        processingDirectory=None,
                        processes=1,
                        cleanup=True,
                        verbose=True,
                        interpolation='linear',
                        **args):
    """Resample data inversely to :func:`resampleData` routine
    
    Arguments:
        sink (str or None): image to be inversly resampled (=sink in :func:`resampleData`)
        source (str or array): destination for inversly resmapled image (=source in :func:`resampleData`)
        dataSizeSource (tuple or None): target size of the resampled image
        orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)
        resolutionSource (tuple): resolution of the source image (in length per pixel)
        resolutionSink (tuple): resolution of the resampled image (in length per pixel)
        processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created
        processes (int): number of processes to use for parallel resampling
        cleanup (bool): remove temporary files
        verbose (bool): display progress information
        interpolation (str): method to use for interpolating to the resmapled image
    
    Returns:
        (array or str): data or file name of resampled image

    Notes: 
        * resolutions are assumed to be given for the axes of the intrinsic 
          orientation of the data and reference as when viewed by matplotlib or ImageJ
        * orientation: permuation of 1,2,3 with potential sign, indicating which 
          axes map onto the reference axes, a negative sign indicates reversal 
          of that particular axes
        * only a minimal set of information to detremine the resampling parameter 
          has to be given, e.g. dataSizeSource and dataSizeSink
    """

    #orientation
    orientation = fixOrientation(orientation)

    #assume we can read data fully into memory
    resampledData = io.readData(sink)

    dataSizeSink = resampledData.shape

    if isinstance(dataSizeSource, basestring):
        dataSizeSource = io.dataSize(dataSizeSource)

    dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(
        dataSizeSource=dataSizeSource,
        dataSizeSink=dataSizeSink,
        resolutionSource=resolutionSource,
        resolutionSink=resolutionSink,
        orientation=orientation)

    #print (dataSizeSource, dataSizeSink, resolutionSource, resolutionSink )

    dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation)

    #flip axes back and permute inversely
    if not orientation is None:
        if orientation[0] < 0:
            resampledData = resampledData[::-1, :, :]
        if orientation[1] < 0:
            resampledData = resampledData[:, ::-1, :]
        if orientation[2] < 0:
            resampledData = resampledData[:, :, ::-1]

        #reorient
        peri = inverseOrientation(orientation)
        peri = orientationToPermuation(peri)
        resampledData = resampledData.transpose(peri)

    # upscale in z
    interpolation = fixInterpolation(interpolation)

    resampledDataXY = numpy.zeros(
        (dataSizeSinkI[0], dataSizeSinkI[1], dataSizeSource[2]),
        dtype=resampledData.dtype)

    for i in range(dataSizeSinkI[0]):
        if verbose and i % 25 == 0:
            print "resampleDataInverse: processing %d/%d" % (i,
                                                             dataSizeSinkI[0])

        #cv2.resize takes reverse order of sizes !
        resampledDataXY[i, :, :] = cv2.resize(
            resampledData[i, :, :], (dataSizeSource[2], dataSizeSinkI[1]),
            interpolation=interpolation)

    # upscale x, y in parallel

    if io.isFileExpression(source):
        files = source
    else:
        if processingDirectory == None:
            processingDirectory = tempfile.mkdtemp()
        files = os.path.join(sink[0], 'resample_\d{4}.tif')

    io.writeData(files, resampledDataXY)

    nZ = dataSizeSource[2]
    pool = multiprocessing.Pool(processes=processes)
    argdata = []
    for i in range(nZ):
        argdata.append((source, fl.fileExpressionToFileName(files, i),
                        dataSizeSource, interpolation, i, nZ))
    pool.map(_resampleXYParallel, argdata)

    if io.isFileExpression(source):
        return source
    else:
        data = io.convertData(files, source)

        if cleanup:
            shutil.rmtree(processingDirectory)

        return data
Beispiel #13
0
def resampleDataInverse(sink, source = None, dataSizeSource = None, orientation = None, resolutionSource = (4.0625, 4.0625, 3), resolutionSink = (25, 25, 25), 
                        processingDirectory = None, processes = 1, cleanup = True, verbose = True, interpolation = 'linear', **args):
    """Resample data inversely to :func:`resampleData` routine
    
    Arguments:
        sink (str or None): image to be inversly resampled (=sink in :func:`resampleData`)
        source (str or array): destination for inversly resmapled image (=source in :func:`resampleData`)
        dataSizeSource (tuple or None): target size of the resampled image
        orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)
        resolutionSource (tuple): resolution of the source image (in length per pixel)
        resolutionSink (tuple): resolution of the resampled image (in length per pixel)
        processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created
        processes (int): number of processes to use for parallel resampling
        cleanup (bool): remove temporary files
        verbose (bool): display progress information
        interpolation (str): method to use for interpolating to the resmapled image
    
    Returns:
        (array or str): data or file name of resampled image

    Notes: 
        * resolutions are assumed to be given for the axes of the intrinsic 
          orientation of the data and reference as when viewed by matplotlib or ImageJ
        * orientation: permuation of 1,2,3 with potential sign, indicating which 
          axes map onto the reference axes, a negative sign indicates reversal 
          of that particular axes
        * only a minimal set of information to detremine the resampling parameter 
          has to be given, e.g. dataSizeSource and dataSizeSink
    """    
    
    
    #orientation
    orientation = fixOrientation(orientation);
    
    #assume we can read data fully into memory
    resampledData = io.readData(sink);

    dataSizeSink = resampledData.shape;
    
    if isinstance(dataSizeSource, basestring):
        dataSizeSource = io.dataSize(dataSizeSource);

    dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(dataSizeSource = dataSizeSource, dataSizeSink = dataSizeSink, 
                                                                                      resolutionSource = resolutionSource, resolutionSink = resolutionSink, orientation = orientation);

    #print (dataSizeSource, dataSizeSink, resolutionSource, resolutionSink )
    
    dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation);
    
    
    #flip axes back and permute inversely
    if not orientation is None:
        if orientation[0] < 0:
            resampledData = resampledData[::-1, :, :];
        if orientation[1] < 0:
            resampledData = resampledData[:, ::-1, :]; 
        if orientation[2] < 0:
            resampledData = resampledData[:, :, ::-1];

        
        #reorient
        peri = inverseOrientation(orientation);
        peri = orientationToPermuation(peri);
        resampledData = resampledData.transpose(peri);
    
    # upscale in z
    interpolation = fixInterpolation(interpolation);
    
    resampledDataXY = numpy.zeros((dataSizeSinkI[0], dataSizeSinkI[1], dataSizeSource[2]), dtype = resampledData.dtype);    
    
    for i in range(dataSizeSinkI[0]):
        if verbose and i % 25 == 0:
            print "resampleDataInverse: processing %d/%d" % (i, dataSizeSinkI[0])

        #cv2.resize takes reverse order of sizes !
        resampledDataXY[i ,:, :] =  cv2.resize(resampledData[i,:,:], (dataSizeSource[2], dataSizeSinkI[1]), interpolation = interpolation);

    # upscale x, y in parallel
    
    if io.isFileExpression(source):
        files = source;
    else:
        if processingDirectory == None:
            processingDirectory = tempfile.mkdtemp();   
        files = os.path.join(sink[0], 'resample_\d{4}.tif');
    
    io.writeData(files, resampledDataXY);
    
    nZ = dataSizeSource[2];
    pool = multiprocessing.Pool(processes=processes);
    argdata = [];
    for i in range(nZ):
        argdata.append( (source, fl.fileExpressionToFileName(files, i), dataSizeSource, interpolation, i, nZ) );  
    pool.map(_resampleXYParallel, argdata);
    
    if io.isFileExpression(source):
        return source;
    else:
        data = io.convertData(files, source);
        
        if cleanup:
            shutil.rmtree(processingDirectory);
        
        return data;
Beispiel #14
0
def _test():
    from importlib import reload
    import ClearMap.Tests.Files as tf

    import ClearMap.IO.FileList as fl
    reload(fl)

    expression = tf.io.join(tf.tif_sequence, 'sequence<Z,I,4>.tif')

    f = fl.Source(expression=expression)
    print(f)
    print(f.expression.string({'Z': 10}))

    d = f.__getitem__((slice(None), slice(None), 1), processes='serial')
    d = f[:, :, 1]

    import numpy as np
    import ClearMap.IO.IO as io
    np.all(d == io.read(f.file_list[1]))

    # genreate some files
    data = np.asarray(20 * np.random.rand(4, 5, 2, 3), dtype='int32')
    data[5:15, 20:45, 2:9] = 0

    f = fl.Source('./test_file_list/test<I,3>_<I,2>.npy',
                  shape=(4, 5, 2, 3),
                  dtype='int32')

    f.__setitem__(slice(None, ), data, processes='serial')

    reload(fl)
    f2 = fl.Source('./test_file_list')
    print(f2)

    data2 = f2.__getitem__(slice(None), processes='serial')

    s = io.as_source(data)
    s2 = io.as_source(data2)
    print(s)
    print(s2)

    np.all(data2 == data)

    data3 = f2[:]
    np.all(data3 == data)

    np.all(f2[:, :, 1, :] == data[:, :, 1, :])

    fl.fu.delete_directory('./test_file_list')

    import ClearMap.Tests.Files as tf
    name = tf.tif_sequence

    fl1 = fl._file_list(name)

    name = fl.fu.join(tf.tif_sequence, 'sequence<I,4>.tif')
    fl2 = fl._file_list(name)

    print(fl1 == fl2)

    f = fl.Source(name)
    print(f)
    f.shape
    f.dtype
fn = os.path.join(datadir, r'160412_mosaic_15-20-19/15-20-19_mosaic_UltraII\[(?P<row>\d{2}) x (?P<col>\d{2})\]_C00_xyz-Table Z(?P<z>\d{4}).ome.tif')

_, gr = st.findFileList(fn , sort = True, groups = ['row','col'], absolute = True)
groups = [];
for i in range(gr.shape[1]):
    groups.append(np.unique(gr[:,i]));

print groups

for i in groups[0]:
    for j in groups[1]:
        fileExpression = os.path.join(datadir, r'160412_mosaic_15-20-19/15-20-19_mosaic_UltraII\[%s x %s]_C00_xyz-Table Z\d{4}.ome.tif' % (i,j))

        io.dataSize(fileExpression)

        io.readMetaData(fileExpression, info = ['size', 'overlap', 'resolution'])


        import ClearMap.IO.FileList as fl;
        reload(fl)

        fncrop = os.path.join(datadir, r'cropped/15-20-19_mosaic_UltraII_%s_x_%s_C00_xyz-Table Z\d{4}.ome.tif' % (i,j))


        fc = fl.cropData(fileExpression, fncrop, x = (400, -400), y = (550, -550),  adjustOverlap = True, processes = all)
        #fc1 = fl.firstFile(fc)
        #io.readMetaData(fc1, info = ['overlap', 'resolution', 'size']);