def moveTeraStitcherStackToFileList(source, sink, deleteDirectory = True, verbose = True):
"""Moves image files from TeraSticher file structure to a list of files
Arguments:
source (str): base directory of the TeraStitcher files
sink (str): regular expression of the files to copy to
verbose (bool): show progress
Returns:
str: sink regular expression
"""
fns = glob.glob(os.path.join(source, '*/*/*'));
fns = natsort.natsorted(fns);
io.createDirectory(sink);
for i,f in enumerate(fns):
fn = filelist.fileExpressionToFileName(sink, i);
if verbose:
print '%s -> %s' % (f,fn)
shutil.move(f, fn);
if deleteDirectory:
p,_ = os.path.split(fns[0]);
p = p.split(os.path.sep);
p = p[:-2];
p = os.path.sep.join(p);
shutil.rmtree(p);
return sink;
def fileNameToIlastikOuput(filename):
"""Converts *ClearMap* file name to an argument string for use with Ilastik headless mode
Arguments:
filename (str): image file name or expression
Returns:
str: Ilastik headless ouput specifications
Note:
The output is formated accroding to the Ilastik pixel calssification output specifications
"""
if not isValidOutputFileName(filename):
raise RuntimeError('Ilastik: file format not compatibel with Ilastik output');
if io.isFileExpression(filename):
o = '--output_format="' + io.fileExtension(filename) + ' sequence" '+ \
'--output_filename_format="' + filelist.fileExpressionToFileName(filename, '{slice_index}') + '"';
return o;
else: # single file
extensionToOuput = {'bmp' : 'bmp', 'gif' : 'gif', 'hdr' : 'hrd',
'jpg' : 'jpg', 'jpeg': 'jpeg','pbm' : 'pbm',
'pgm' : 'pgm', 'png' : 'png', 'pnm' : 'pnm', 'ppm' : 'ppm',
'ras' : 'ras', 'tif' : 'tif', 'tiff': 'tiff','xv' : 'xv',
'h5' : 'hdf5' , 'npy' : 'numpy'};
ext = extensionToOuput[io.fileExtension(filename)];
o = '--output_format="' + ext +'" ' + \
'--output_filename_format="' + filename + '"';
return o;
def moveTeraStitcherStackToFileList(source, sink, deleteDirectory=True, verbose=True):
"""Moves image files from TeraSticher file structure to a list of files
Arguments:
source (str): base directory of the TeraStitcher files
sink (str): regular expression of the files to copy to
verbose (bool): show progress
Returns:
str: sink regular expression
"""
fns = glob.glob(os.path.join(source, "*/*/*"))
fns = natsort.natsorted(fns)
io.createDirectory(sink)
for i, f in enumerate(fns):
fn = filelist.fileExpressionToFileName(sink, i)
if verbose:
print "%s -> %s" % (f, fn)
shutil.move(f, fn)
if deleteDirectory:
p, _ = os.path.split(fns[0])
p = p.split(os.path.sep)
p = p[:-2]
p = os.path.sep.join(p)
shutil.rmtree(p)
return sink
def fileNameToIlastikOuput(filename):
"""Converts *ClearMap* file name to an argument string for use with Ilastik headless mode
Arguments:
filename (str): image file name or expression
Returns:
str: Ilastik headless ouput specifications
Note:
The output is formated accroding to the Ilastik pixel calssification output specifications
"""
if not isValidOutputFileName(filename):
raise RuntimeError("Ilastik: file format not compatibel with Ilastik output")
if io.isFileExpression(filename):
o = (
'--output_format="'
+ io.fileExtension(filename)
+ ' sequence" '
+ '--output_filename_format="'
+ filelist.fileExpressionToFileName(filename, "{slice_index}")
+ '"'
)
return o
else: # single file
extensionToOuput = {
"bmp": "bmp",
"gif": "gif",
"hdr": "hrd",
"jpg": "jpg",
"jpeg": "jpeg",
"pbm": "pbm",
"pgm": "pgm",
"png": "png",
"pnm": "pnm",
"ppm": "ppm",
"ras": "ras",
"tif": "tif",
"tiff": "tiff",
"xv": "xv",
"h5": "hdf5",
"npy": "numpy",
}
ext = extensionToOuput[io.fileExtension(filename)]
o = '--output_format="' + ext + '" ' + '--output_filename_format="' + filename + '"'
return o
def location_to_module(location):
"""Returns the IO module associated with a location string.
Arguments
---------
location : object
Location of the source.
Returns
-------
module : module
The module that handles the IO of the source specified by its location.
"""
if fl.is_file_list(location):
return fl;
else:
return filename_to_module(location);
def fileNameToIlastikOuput(filename):
"""Converts *ClearMap* file name to an argument string for use with Ilastik headless mode
Arguments:
filename (str): image file name or expression
Returns:
str: Ilastik headless ouput specifications
Note:
The output is formated accroding to the Ilastik pixel calssification output specifications
"""
if not isValidOutputFileName(filename):
raise RuntimeError(
'Ilastik: file format not compatibel with Ilastik output')
if io.isFileExpression(filename):
o = '--output_format="' + io.fileExtension(filename) + ' sequence" '+ \
'--output_filename_format="' + filelist.fileExpressionToFileName(filename, '{slice_index}') + '"'
return o
else: # single file
extensionToOuput = {
'bmp': 'bmp',
'gif': 'gif',
'hdr': 'hrd',
'jpg': 'jpg',
'jpeg': 'jpeg',
'pbm': 'pbm',
'pgm': 'pgm',
'png': 'png',
'pnm': 'pnm',
'ppm': 'ppm',
'ras': 'ras',
'tif': 'tif',
'tiff': 'tiff',
'xv': 'xv',
'h5': 'hdf5',
'npy': 'numpy'
}
ext = extensionToOuput[io.fileExtension(filename)]
o = '--output_format="' + ext +'" ' + \
'--output_filename_format="' + filename + '"'
return o
def file_list(expression = None, file_list = None, sort = True, verbose = False):
"""Returns the list of files that match the tag expression.
Arguments
---------
expression :str
The regular expression the file names should match.
sort : bool
If True, sort files naturally.
verbose : bool
If True, print warning if no files exists.
Returns
-------
file_list : list of str
The list of files that matched the expression.
"""
return fl._file_list(expression=expression, file_list=file_list, sort=sort, verbose=verbose)
def fileNameToIlastikInput(filename):
"""Converts *ClearMap* file name to a string for use with Ilastik input
Arguments:
filename (str): image file name or expression
Returns:
str: Ilastik conform file name
Note:
file expressions in *ClearMap* are regular expressions but shell expressions in Ilastik.
"""
if not isValidInputFileName(filename):
raise RuntimeError('Ilastik: file format not compatibel with Ilastik');
if io.isFileExpression(filename):
return '"' + filelist.fileExpressionToFileName(filename, '*') + '"';
else:
return '"' + filename + '"';
def fileNameToIlastikInput(filename):
"""Converts *ClearMap* file name to a string for use with Ilastik input
Arguments:
filename (str): image file name or expression
Returns:
str: Ilastik conform file name
Note:
file expressions in *ClearMap* are regular expressions but shell expressions in Ilastik.
"""
if not isValidInputFileName(filename):
raise RuntimeError('Ilastik: file format not compatibel with Ilastik')
if io.isFileExpression(filename):
return '"' + filelist.fileExpressionToFileName(filename, '*') + '"'
else:
return '"' + filename + '"'
def copyTeraStitcherStackToFileList(source, sink, verbose = True):
"""Copies image files from TeraSticher file structure to a list of files
Arguments:
source (str): base directory of the TeraStitcher files
sink (str): regular expression of the files to copy to
verbose (bool): show progress
Returns:
str: sink regular expression
"""
#TODO: multiple tiles !
fns = glob.glob(os.path.join(source, '*/*/*'));
fns = natsort.natsorted(fns);
#print fns
io.createDirectory(sink);
for i,f in enumerate(fns):
fn = filelist.fileExpressionToFileName(sink, i);
if verbose:
print '%s -> %s' % (f,fn)
shutil.copyfile(f, fn);
return sink;
def copyTeraStitcherStackToFileList(source, sink, verbose=True):
"""Copies image files from TeraSticher file structure to a list of files
Arguments:
source (str): base directory of the TeraStitcher files
sink (str): regular expression of the files to copy to
verbose (bool): show progress
Returns:
str: sink regular expression
"""
# TODO: multiple tiles !
fns = glob.glob(os.path.join(source, "*/*/*"))
fns = natsort.natsorted(fns)
# print fns
io.createDirectory(sink)
for i, f in enumerate(fns):
fn = filelist.fileExpressionToFileName(sink, i)
if verbose:
print "%s -> %s" % (f, fn)
shutil.copyfile(f, fn)
return sink
def resampleDataInverse(sink,
source=None,
dataSizeSource=None,
orientation=None,
resolutionSource=(4.0625, 4.0625, 3),
resolutionSink=(25, 25, 25),
processingDirectory=None,
processes=1,
cleanup=True,
verbose=True,
interpolation='linear',
**args):
"""Resample data inversely to :func:`resampleData` routine
Arguments:
sink (str or None): image to be inversly resampled (=sink in :func:`resampleData`)
source (str or array): destination for inversly resmapled image (=source in :func:`resampleData`)
dataSizeSource (tuple or None): target size of the resampled image
orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)
resolutionSource (tuple): resolution of the source image (in length per pixel)
resolutionSink (tuple): resolution of the resampled image (in length per pixel)
processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created
processes (int): number of processes to use for parallel resampling
cleanup (bool): remove temporary files
verbose (bool): display progress information
interpolation (str): method to use for interpolating to the resmapled image
Returns:
(array or str): data or file name of resampled image
Notes:
* resolutions are assumed to be given for the axes of the intrinsic
orientation of the data and reference as when viewed by matplotlib or ImageJ
* orientation: permuation of 1,2,3 with potential sign, indicating which
axes map onto the reference axes, a negative sign indicates reversal
of that particular axes
* only a minimal set of information to detremine the resampling parameter
has to be given, e.g. dataSizeSource and dataSizeSink
"""
#orientation
orientation = fixOrientation(orientation)
#assume we can read data fully into memory
resampledData = io.readData(sink)
dataSizeSink = resampledData.shape
if isinstance(dataSizeSource, basestring):
dataSizeSource = io.dataSize(dataSizeSource)
dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(
dataSizeSource=dataSizeSource,
dataSizeSink=dataSizeSink,
resolutionSource=resolutionSource,
resolutionSink=resolutionSink,
orientation=orientation)
#print (dataSizeSource, dataSizeSink, resolutionSource, resolutionSink )
dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation)
#flip axes back and permute inversely
if not orientation is None:
if orientation[0] < 0:
resampledData = resampledData[::-1, :, :]
if orientation[1] < 0:
resampledData = resampledData[:, ::-1, :]
if orientation[2] < 0:
resampledData = resampledData[:, :, ::-1]
#reorient
peri = inverseOrientation(orientation)
peri = orientationToPermuation(peri)
resampledData = resampledData.transpose(peri)
# upscale in z
interpolation = fixInterpolation(interpolation)
resampledDataXY = numpy.zeros(
(dataSizeSinkI[0], dataSizeSinkI[1], dataSizeSource[2]),
dtype=resampledData.dtype)
for i in range(dataSizeSinkI[0]):
if verbose and i % 25 == 0:
print "resampleDataInverse: processing %d/%d" % (i,
dataSizeSinkI[0])
#cv2.resize takes reverse order of sizes !
resampledDataXY[i, :, :] = cv2.resize(
resampledData[i, :, :], (dataSizeSource[2], dataSizeSinkI[1]),
interpolation=interpolation)
# upscale x, y in parallel
if io.isFileExpression(source):
files = source
else:
if processingDirectory == None:
processingDirectory = tempfile.mkdtemp()
files = os.path.join(sink[0], 'resample_\d{4}.tif')
io.writeData(files, resampledDataXY)
nZ = dataSizeSource[2]
pool = multiprocessing.Pool(processes=processes)
argdata = []
for i in range(nZ):
argdata.append((source, fl.fileExpressionToFileName(files, i),
dataSizeSource, interpolation, i, nZ))
pool.map(_resampleXYParallel, argdata)
if io.isFileExpression(source):
return source
else:
data = io.convertData(files, source)
if cleanup:
shutil.rmtree(processingDirectory)
return data
def resampleDataInverse(sink, source = None, dataSizeSource = None, orientation = None, resolutionSource = (4.0625, 4.0625, 3), resolutionSink = (25, 25, 25),
processingDirectory = None, processes = 1, cleanup = True, verbose = True, interpolation = 'linear', **args):
"""Resample data inversely to :func:`resampleData` routine
Arguments:
sink (str or None): image to be inversly resampled (=sink in :func:`resampleData`)
source (str or array): destination for inversly resmapled image (=source in :func:`resampleData`)
dataSizeSource (tuple or None): target size of the resampled image
orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)
resolutionSource (tuple): resolution of the source image (in length per pixel)
resolutionSink (tuple): resolution of the resampled image (in length per pixel)
processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created
processes (int): number of processes to use for parallel resampling
cleanup (bool): remove temporary files
verbose (bool): display progress information
interpolation (str): method to use for interpolating to the resmapled image
Returns:
(array or str): data or file name of resampled image
Notes:
* resolutions are assumed to be given for the axes of the intrinsic
orientation of the data and reference as when viewed by matplotlib or ImageJ
* orientation: permuation of 1,2,3 with potential sign, indicating which
axes map onto the reference axes, a negative sign indicates reversal
of that particular axes
* only a minimal set of information to detremine the resampling parameter
has to be given, e.g. dataSizeSource and dataSizeSink
"""
#orientation
orientation = fixOrientation(orientation);
#assume we can read data fully into memory
resampledData = io.readData(sink);
dataSizeSink = resampledData.shape;
if isinstance(dataSizeSource, basestring):
dataSizeSource = io.dataSize(dataSizeSource);
dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(dataSizeSource = dataSizeSource, dataSizeSink = dataSizeSink,
resolutionSource = resolutionSource, resolutionSink = resolutionSink, orientation = orientation);
#print (dataSizeSource, dataSizeSink, resolutionSource, resolutionSink )
dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation);
#flip axes back and permute inversely
if not orientation is None:
if orientation[0] < 0:
resampledData = resampledData[::-1, :, :];
if orientation[1] < 0:
resampledData = resampledData[:, ::-1, :];
if orientation[2] < 0:
resampledData = resampledData[:, :, ::-1];
#reorient
peri = inverseOrientation(orientation);
peri = orientationToPermuation(peri);
resampledData = resampledData.transpose(peri);
# upscale in z
interpolation = fixInterpolation(interpolation);
resampledDataXY = numpy.zeros((dataSizeSinkI[0], dataSizeSinkI[1], dataSizeSource[2]), dtype = resampledData.dtype);
for i in range(dataSizeSinkI[0]):
if verbose and i % 25 == 0:
print "resampleDataInverse: processing %d/%d" % (i, dataSizeSinkI[0])
#cv2.resize takes reverse order of sizes !
resampledDataXY[i ,:, :] = cv2.resize(resampledData[i,:,:], (dataSizeSource[2], dataSizeSinkI[1]), interpolation = interpolation);
# upscale x, y in parallel
if io.isFileExpression(source):
files = source;
else:
if processingDirectory == None:
processingDirectory = tempfile.mkdtemp();
files = os.path.join(sink[0], 'resample_\d{4}.tif');
io.writeData(files, resampledDataXY);
nZ = dataSizeSource[2];
pool = multiprocessing.Pool(processes=processes);
argdata = [];
for i in range(nZ):
argdata.append( (source, fl.fileExpressionToFileName(files, i), dataSizeSource, interpolation, i, nZ) );
pool.map(_resampleXYParallel, argdata);
if io.isFileExpression(source):
return source;
else:
data = io.convertData(files, source);
if cleanup:
shutil.rmtree(processingDirectory);
return data;
def _test():
from importlib import reload
import ClearMap.Tests.Files as tf
import ClearMap.IO.FileList as fl
reload(fl)
expression = tf.io.join(tf.tif_sequence, 'sequence<Z,I,4>.tif')
f = fl.Source(expression=expression)
print(f)
print(f.expression.string({'Z': 10}))
d = f.__getitem__((slice(None), slice(None), 1), processes='serial')
d = f[:, :, 1]
import numpy as np
import ClearMap.IO.IO as io
np.all(d == io.read(f.file_list[1]))
# genreate some files
data = np.asarray(20 * np.random.rand(4, 5, 2, 3), dtype='int32')
data[5:15, 20:45, 2:9] = 0
f = fl.Source('./test_file_list/test<I,3>_<I,2>.npy',
shape=(4, 5, 2, 3),
dtype='int32')
f.__setitem__(slice(None, ), data, processes='serial')
reload(fl)
f2 = fl.Source('./test_file_list')
print(f2)
data2 = f2.__getitem__(slice(None), processes='serial')
s = io.as_source(data)
s2 = io.as_source(data2)
print(s)
print(s2)
np.all(data2 == data)
data3 = f2[:]
np.all(data3 == data)
np.all(f2[:, :, 1, :] == data[:, :, 1, :])
fl.fu.delete_directory('./test_file_list')
import ClearMap.Tests.Files as tf
name = tf.tif_sequence
fl1 = fl._file_list(name)
name = fl.fu.join(tf.tif_sequence, 'sequence<I,4>.tif')
fl2 = fl._file_list(name)
print(fl1 == fl2)
f = fl.Source(name)
print(f)
f.shape
f.dtype
fn = os.path.join(datadir, r'160412_mosaic_15-20-19/15-20-19_mosaic_UltraII\[(?P<row>\d{2}) x (?P<col>\d{2})\]_C00_xyz-Table Z(?P<z>\d{4}).ome.tif')
_, gr = st.findFileList(fn , sort = True, groups = ['row','col'], absolute = True)
groups = [];
for i in range(gr.shape[1]):
groups.append(np.unique(gr[:,i]));
print groups
for i in groups[0]:
for j in groups[1]:
fileExpression = os.path.join(datadir, r'160412_mosaic_15-20-19/15-20-19_mosaic_UltraII\[%s x %s]_C00_xyz-Table Z\d{4}.ome.tif' % (i,j))
io.dataSize(fileExpression)
io.readMetaData(fileExpression, info = ['size', 'overlap', 'resolution'])
import ClearMap.IO.FileList as fl;
reload(fl)
fncrop = os.path.join(datadir, r'cropped/15-20-19_mosaic_UltraII_%s_x_%s_C00_xyz-Table Z\d{4}.ome.tif' % (i,j))
fc = fl.cropData(fileExpression, fncrop, x = (400, -400), y = (550, -550), adjustOverlap = True, processes = all)
#fc1 = fl.firstFile(fc)
#io.readMetaData(fc1, info = ['overlap', 'resolution', 'size']);
