def __init__(self, filename='', metadata=None, taskQueue=None):
if filename is not '':
self.filename = getFullExistingFilename(
filename) #convert relative path to full path
#use metadata for glob
md = MetaDataHandler.SimpleMDHandler(self.filename)
else:
md = metadata
self.filename = ''
pattern = md.getEntry('SeriesPattern')
self.files = glob.glob(
os.path.join(os.path.split(self.filename)[0], pattern))
self.files.sort()
f0 = self.files[0]
if f0.endswith('.tif'):
self.im0 = tifffile.imread(f0).squeeze()
self._slice_shape = self.im0.shape[:2]
else:
self.im0 = Image.open(self.files[0])
self._slice_shape = self.im0.size[1], self.im0.size[0]
#self.im.seek(0)
#PIL's endedness support is subtly broken - try to fix it
#NB this is untested for floating point tiffs
self.endedness = 'LE'
if self.im0.ifd.prefix == 'MM':
self.endedness = 'BE'
print((self.im0.ifd.prefix))
print((self.endedness))
def setModel(modName, md):
global IntXVals, IntYVals, IntZVals, dx, dy, dz
#FIXME - use load_psf() instead
mf = open(getFullExistingFilename(modName), 'rb')
mod, voxelsize = pickle.load(mf)
mf.close()
mod = resizePSF(mod, interpModel().shape)
#if not voxelsize.x == md.voxelsize.x:
# raise RuntimeError("PSF and Image voxel sizes don't match")
IntXVals = 1e3 * voxelsize.x * mgrid[-(mod.shape[0] / 2.):(mod.shape[0] /
2.)]
IntYVals = 1e3 * voxelsize.y * mgrid[-(mod.shape[1] / 2.):(mod.shape[1] /
2.)]
IntZVals = 1e3 * voxelsize.z * mgrid[-(mod.shape[2] / 2.):(mod.shape[2] /
2.)]
dx = voxelsize.x * 1e3
dy = voxelsize.y * 1e3
dz = voxelsize.z * 1e3
#interpModel = mod
#interpModel = np.maximum(mod/mod.max(), 0) #normalise to 1
interpModel_by_chan[0] = np.maximum(
mod / mod[:, :, len(IntZVals) / 2].sum(), 0) #normalise to 1 and clip
def __init__(self, image_file, taskQueue=None, chanNum = 0, series=None):
self.chanNum = chanNum
if isinstance(image_file, BioformatsFile):
self.bff = image_file
else:
self.filename = getFullExistingFilename(image_file)#convert relative path to full path
print(self.filename)
self.bff = BioformatsFile(self.filename)
if series is not None:
self.bff.rdr.setSeries(series)
self.sizeX = self.bff.rdr.getSizeX()
self.sizeY = self.bff.rdr.getSizeY()
self.sizeZ = self.bff.rdr.getSizeZ()
self.sizeT = self.bff.rdr.getSizeT()
self.sizeC = self.bff.rdr.getSizeC()
#self.shape = [self.sizeX, self.sizeY, self.sizeZ*self.sizeT, self.sizeC]
#axisOrder = self.bff.rdr.getDimensionOrder()
#sh = {'X' : self.sizeX, 'Y':self.sizeY, 'Z':self.sizeZ, 'T':self.sizeT, 'C':self.sizeT}
self.additionalDims = 'TC'
def __init__(self, filename, taskQueue=None):
self.filename = getFullExistingFilename(
filename) #convert relative path to full path
#self.data = readTiff.read3DTiff(self.filename)
#use metadata for glob
md = MetaDataHandler.SimpleMDHandler(self.filename)
pattern = md.getEntry('SeriesPattern')
self.files = glob.glob(
os.path.join(os.path.split(self.filename)[0], pattern))
self.files.sort()
self.im0 = Image.open(self.files[0])
#self.im.seek(0)
#PIL's endedness support is subtly broken - try to fix it
#NB this is untested for floating point tiffs
self.endedness = 'LE'
if self.im0.ifd.prefix == 'MM':
self.endedness = 'BE'
print((self.im0.ifd.prefix))
print((self.endedness))
def read(filename):
'''
Read a file from disk or the cluster.
NOTE: filename is expected to be sanitized / trusted, this should not be called
with user data from a web endpoint.
Parameters
----------
filename
Returns
-------
'''
filename = nameUtils.getFullExistingFilename(filename)
if os.path.exists(filename):
with open(filename, 'rb') as f:
s = f.read()
return s
elif is_cluster_uri(filename):
from . import clusterIO
sequenceName, clusterfilter = split_cluster_url(filename)
s = clusterIO.get_file(sequenceName, clusterfilter)
return s
else:
raise IOError('File does not exist or URI not understood: %s' % filename)
def local_or_temp_filename(url):
"""
Gets a local filename for a given url.
The purpose is to let us load files from the cluster using IO libraries (e.g. pytables, tifffile) which need a
filename, not a file handle.
Does the following (in order).
* checks to see if the url refers to a local file, if so return the filename
* if the url points to a cluster file, check to see if it happens to be stored locally. If so, return the local path
* download file to a temporary file and return the filename of that temporary file.
NB: This should be used as a context manager (ie in a with statement) so that the temporary file gets cleaned up properly
Parameters
----------
url : basestring
Returns
-------
filename : basestring
Notes
-----
TODO - there is potential for optimization by using memmaps rather than files on disk.
"""
filename = nameUtils.getFullExistingFilename(url)
if os.path.exists(filename):
yield filename
elif filename.startswith('pyme-cluster') or filename.startswith(
'PYME-CLUSTER'):
from . import clusterIO
sequenceName, clusterfilter = split_cluster_url(filename)
localpath = clusterIO.get_local_path(sequenceName, clusterfilter)
if localpath:
yield localpath
else:
ext = os.path.splitext(sequenceName)[-1]
with tempfile.NamedTemporaryFile(mode='w+b', suffix=ext) as outf:
s = clusterIO.get_file(sequenceName, clusterfilter)
outf.write(s)
outf.flush()
yield outf.name
else:
raise IOError('Path "%s" could not be found' % url)
def _fetchMap(self, md, mapName):
"""retrive a map, with a given name. First try and get it from the Queue,
then try finding it locally"""
try:
map = md.taskQueue.getQueueData(md.dataSourceID, 'MAP', mapName)
except:
fn = getFullExistingFilename(mapName)
map = ImageStack(filename=fn, haveGUI=False).data[:,:,0].squeeze() #this should handle .tif, .h5, and a few others
return map
def __init__(self, filename, taskQueue=None, chanNum = 0):
self.filename = getFullExistingFilename(filename)#convert relative path to full path
self.chanNum = chanNum
self.RGB = False
#self.data = readTiff.read3DTiff(self.filename)
#self.im = Image.open(filename)
#self.im.seek(0)
#PIL's endedness support is subtly broken - try to fix it
#NB this is untested for floating point tiffs
#self.endedness = 'LE'
#if self.im.ifd.prefix =='MM':
# self.endedness = 'BE'
#to find the number of images we have to loop over them all
#this is obviously not ideal as PIL loads the image data into memory for each
#slice and this is going to represent a huge performance penalty for large stacks
#should still let them be opened without having all images in memory at once though
#self.numSlices = self.im.tell()
#try:
# while True:
# self.numSlices += 1
# self.im.seek(self.numSlices)
#except EOFError:
# pass
print((self.filename))
tf = tifffile.TIFFfile(self.filename)
print(tf.series[0].shape)
self.im = tf.series[0].pages
if tf.is_ome:
print('Detected OME TIFF')
sh = dict(zip(tf.series[0].axes, tf.series[0].shape))
print('sh = %s' % sh)
self.sizeC = sh['C']
axisOrder = tf.series[0].axes[::-1]
self.additionalDims = ''.join([a for a in axisOrder[2:] if sh[a] > 1])
elif tf.is_rgb:
print('Detected RGB TIFF')
self.sizeC = 3
self.RGB = True
self.additionalDims = 'C'
def getQueueData(self, fieldName, *args):
"""Get data, defined by fieldName and potntially additional arguments, ascociated with queue"""
if fieldName == 'FitResults':
startingAt, = args
#with self.fileResultsLock.rlock:
# if self.h5ResultsFile.__contains__('/FitResults'):
# res = self.h5ResultsFile.root.FitResults[startingAt:]
# else:
# res = []
with h5rFile.openH5R(self.resultsFilename, 'a') as h5f:
res = h5f.getTableData('FitResults', slice(startingAt, None))
return res
elif fieldName == 'PSF':
#from PYME.ParallelTasks.relativeFiles import getFullExistingFilename
from PYME.IO.load_psf import load_psf
res = None
modName = self.metaData.getEntry('PSFFile')
# mf = open(getFullExistingFilename(modName), 'rb')
#res = np.load(mf)
#mf.close()
res = load_psf(getFullExistingFilename(modName))
return res
elif fieldName == 'MAP':
mapName, = args
#from PYME.ParallelTasks.relativeFiles import getFullExistingFilename
from PYME.IO.image import ImageStack
print('Serving map: %s' % mapName)
fn = getFullExistingFilename(mapName)
varmap = ImageStack(
filename=fn, haveGUI=False).data[:, :, 0].squeeze(
) #this should handle .tif, .h5, and a few others
return varmap
else:
return None
def openFile(filename, mode='rb'):
filename = nameUtils.getFullExistingFilename(filename)
if os.path.exists(filename):
return open(filename, mode)
elif filename.startswith('pyme-cluster') or filename.startswith('PYME-CLUSTER'):
#TODO - add short-circuiting for local files
from . import clusterIO
sequenceName, clusterfilter = split_cluster_url(filename)
s = clusterIO.get_file(sequenceName, clusterfilter)
return BytesIO(s)
else:
raise IOError('File does not exist or URI not understood: %s' % filename)
def __init__(self, h5Filename, taskQueue=None):
self.h5Filename = getFullExistingFilename(
h5Filename) #convert relative path to full path
self.h5File = tables.open_file(self.h5Filename)
self._pzf_index = None
if getattr(self.h5File.root, 'PZFImageIndex', False):
self.usePZFFormat = True
try:
self.framesize = self.h5File.root.PZFImageData.attrs.framesize
except AttributeError:
self.framesize = PZFFormat.loads(
self.h5File.root.PZFImageData[0])[0].squeeze().shape
else:
self.usePZFFormat = False
def read(filename):
filename = nameUtils.getFullExistingFilename(filename)
if os.path.exists(filename):
with open(filename, 'rb') as f:
s = f.read()
return s
elif filename.startswith('pyme-cluster') or filename.startswith('PYME-CLUSTER'):
from . import clusterIO
sequenceName, clusterfilter = split_cluster_url(filename)
s = clusterIO.get_file(sequenceName, clusterfilter)
return s
else:
raise IOError('File does not exist or URI not understood: %s' % filename)
def __init__(self, h5Filename, taskQueue=None):
self.h5Filename = getFullExistingFilename(
h5Filename) #convert relative path to full path
self.h5File = tables.open_file(self.h5Filename)
self._pzf_index = None
if getattr(self.h5File.root, 'PZFImageIndex', False):
self.usePZFFormat = True
try:
self.framesize = self.h5File.root.PZFImageData.attrs.framesize
except AttributeError:
self.framesize = PZFFormat.loads(
self.h5File.root.PZFImageData[0])[0].squeeze().shape
else:
self.usePZFFormat = False
try:
dimorder = self._img_data.attrs.DimOrder
if isinstance(dimorder, bytes):
dimorder = dimorder.decode()
assert (dimorder[:2] == 'XY')
size_c = int(self._img_data.attrs.SizeC)
size_z = int(self._img_data.attrs.SizeZ)
size_t = int(self._img_data.attrs.SizeT)
except:
logger.exception(
'Error reading dim info (can be safely ignored for old files)')
dimorder = 'XYZTC'
size_z = self.getNumSlices()
size_t = 1
size_c = 1
XYZTCDataSource.__init__(self,
dimorder,
size_z=size_z,
size_t=size_t,
size_c=size_c)
self._shape = tuple(self.getSliceShape()) + (size_z, size_t, size_c)
self._dtype = self.getSlice(0).dtype
def local_or_named_temp_filename(url):
""" riff of PYME.IO.clusterIO.local_or_temp_filename, but one which returns
a filename with a matching file stub to the original rather than a random
temporary filename
Parameters
----------
url : str
local path or pyme-cluster url
Yields
-------
str
path to a (temporary) file so `url` can be loaded using modules which
expect a local filename
"""
from PYME.IO.FileUtils import nameUtils
from PYME.IO import unifiedIO
import tempfile
filename = nameUtils.getFullExistingFilename(url)
if os.path.exists(filename):
yield filename
elif unifiedIO.is_cluster_uri(url):
from PYME.IO import clusterIO
name, clusterfilter = unifiedIO.split_cluster_url(filename)
localpath = clusterIO.get_local_path(name, clusterfilter)
if localpath:
yield localpath
else:
ext = os.path.splitext(name)[-1]
with tempfile.TemporaryDirectory() as temp_dir:
with open(os.path.join(temp_dir,
os.path.split(name)[-1]), 'wb') as f:
s = clusterIO.get_file(name, clusterfilter)
f.write(s)
f.flush()
yield f.name
else:
raise IOError('Path "%s" could not be found' % url)
def __init__(self, h5Filename, taskQueue=None):
self.h5Filename = getFullExistingFilename(
h5Filename) #convert relative path to full path
self.h5File = tables.open_file(self.h5Filename)
self._pzf_index = None
if getattr(self.h5File.root, 'PZFImageIndex', False):
self.usePZFFormat = True
try:
self.framesize = self.h5File.root.PZFImageData.attrs.framesize
except AttributeError:
self.framesize = PZFFormat.loads(
self.h5File.root.PZFImageData[0])[0].squeeze().shape
else:
self.usePZFFormat = False
try:
self.dimorder = self._img_data.attrs.DimOrder
if isinstance(self.dimorder, bytes):
self.dimorder = self.dimorder.decode()
assert (self.dimorder[:2] == 'XY')
self.sizeC = int(self._img_data.attrs.SizeC)
self.sizeZ = int(self._img_data.attrs.SizeZ)
self.sizeT = int(self._img_data.attrs.SizeT)
# FIXME - we currently ignore SizeZ and SizeT and collapse to one dimension (to fit with the XY[Z/T]C data model)
# This should be changed once we fully move to an xyztc model. In the meantime, it's probably safest if C is
# always the last dimension.
if self.sizeC > 1:
if self.dimorder[-1] == 'C':
self.additionalDims = 'TC'
else:
self.additionalDims = 'CT'
else:
self.additionalDims = 'T'
except:
logger.exception(
'Error reading dim info (can be safely ignored for old files)')
pass
def __init__(self, filename, taskQueue=None, chanNum=0, series=0):
self.filename = getFullExistingFilename(
filename) #convert relative path to full path
self.chanNum = chanNum
self.RGB = False
#self.data = readTiff.read3DTiff(self.filename)
#self.im = Image.open(filename)
#self.im.seek(0)
#PIL's endedness support is subtly broken - try to fix it
#NB this is untested for floating point tiffs
#self.endedness = 'LE'
#if self.im.ifd.prefix =='MM':
# self.endedness = 'BE'
#to find the number of images we have to loop over them all
#this is obviously not ideal as PIL loads the image data into memory for each
#slice and this is going to represent a huge performance penalty for large stacks
#should still let them be opened without having all images in memory at once though
#self.numSlices = self.im.tell()
#try:
# while True:
# self.numSlices += 1
# self.im.seek(self.numSlices)
#except EOFError:
# pass
print((self.filename))
if local_tifffile:
logger.info(
'Using PYMEs built-in, old version of tifffile, better support for ImageJ tiffs can be had with the more recent pip version (`pip install tifffile`)'
)
tf = tifffile.TIFFfile(self.filename)
else:
tf = tifffile.TiffFile(self.filename)
self.tf = tf # keep a reference for debugging
print(tf.series[series].shape)
self.im = tf.series[series].pages
axisOrder = 'XYZTC'
size_z = len(self.im)
size_c, size_t = 1, 1
if tf.is_ome or ((not local_tifffile)):
#print('Detected OME TIFF')
sh = {'Z': 1, 'T': 1, 'C': 1}
_axes = tf.series[series].axes
if 'I' in _axes:
logger.info('Tiff file does not fully specify axes (axes=%s)' %
(_axes.replace('I', '?')[::-1]))
if 'Z' not in _axes:
logger.info('Assuming unknown axis is Z'
) # TODO - explain how to change axes later
_axes = _axes.replace('I', 'Z')
elif 'C' not in _axes:
logger.info('Assuming unknown axis is C')
_axes = _axes.replace('I', 'C')
elif 'T' not in _axes:
logger.info('Assuming unkown axis is T')
_axes = _axes.replace('I', 'T')
else:
logger.warning(
'Unknown axis with all standard axes defined - data might not read correctly'
)
sh.update(dict(zip(_axes, tf.series[0].shape)))
logger.debug('sh = %s' % sh)
size_c = sh['C']
size_z = sh['Z']
size_t = sh['T']
axisOrder = _axes[::-1]
axisOrder = axisOrder + ''.join(
[a for a in ['Z', 'T', 'C'] if not a in axisOrder])
logger.debug('raw TIFF axisOrder = %s' % axisOrder)
#self.additionalDims = ''.join([a for a in axisOrder[2:] if sh[a] > 1])
elif tf.is_rgb:
print(
'WARNING: Detected RGB TIFF - data not likely to be suitable for quantitative analysis'
)
size_c = 3
self.RGB = True
if len(self.im) > 1:
# we can have multi-page RGB TIFF - why?????
print(
'WARNING: Multi-page RGB TIFF detected - where did this come from???'
)
axisOrder = 'XYCZT'
size_z = len(self.im)
size_t = 1
XYZTCDataSource.__init__(self,
input_order=axisOrder,
size_z=size_z,
size_t=size_t,
size_c=size_c)
sl0 = self.getSlice(0)
self._dtype = sl0.dtype
self._shape = [sl0.shape[0], sl0.shape[1], size_z, size_t, size_c]
def __init__(self, filename, taskQueue=None):
self.filename = getFullExistingFilename(
filename) #convert relative path to full path
self.DCIFile = dcimg.DCIMGFile(self.filename)
def dirname(url):
try:
return os.path.dirname(split_cluster_url(url)[0])
except RuntimeError:
return os.path.dirname(nameUtils.getFullExistingFilename(url))
def __init__(self, filename, taskQueue=None, chanNum=0, series=0):
self.filename = getFullExistingFilename(
filename) #convert relative path to full path
self.chanNum = chanNum
self.RGB = False
#self.data = readTiff.read3DTiff(self.filename)
#self.im = Image.open(filename)
#self.im.seek(0)
#PIL's endedness support is subtly broken - try to fix it
#NB this is untested for floating point tiffs
#self.endedness = 'LE'
#if self.im.ifd.prefix =='MM':
# self.endedness = 'BE'
#to find the number of images we have to loop over them all
#this is obviously not ideal as PIL loads the image data into memory for each
#slice and this is going to represent a huge performance penalty for large stacks
#should still let them be opened without having all images in memory at once though
#self.numSlices = self.im.tell()
#try:
# while True:
# self.numSlices += 1
# self.im.seek(self.numSlices)
#except EOFError:
# pass
print((self.filename))
if local_tifffile:
tf = tifffile.TIFFfile(self.filename)
else:
tf = tifffile.TiffFile(self.filename)
self.tf = tf # keep a reference for debugging
print(tf.series[series].shape)
self.im = tf.series[series].pages
axisOrder = 'XYZTC'
size_z = len(self.im)
size_c, size_t = 1, 1
if tf.is_ome or (not local_tifffile):
print('Detected OME TIFF')
sh = {'Z': 1, 'T': 1, 'C': 1}
sh.update(dict(zip(tf.series[series].axes, tf.series[0].shape)))
print('sh = %s' % sh)
size_c = sh['C']
size_z = sh['Z']
size_t = sh['T']
axisOrder = tf.series[series].axes[::-1]
axisOrder = axisOrder + ''.join(
[a for a in ['Z', 'T', 'C'] if not a in axisOrder])
print('axisOrder = ', axisOrder)
#self.additionalDims = ''.join([a for a in axisOrder[2:] if sh[a] > 1])
elif tf.is_rgb:
print(
'WARNING: Detected RGB TIFF - data not likely to be suitable for quantitative analysis'
)
size_c = 3
self.RGB = True
if len(self.im) > 1:
# we can have multi-page RGB TIFF - why?????
print(
'WARNING: Multi-page RGB TIFF detected - where did this come from???'
)
axisOrder = 'XYCZT'
size_z = len(self.im)
size_t = 1
XYZTCDataSource.__init__(self,
input_order=axisOrder,
size_z=size_z,
size_t=size_t,
size_c=size_c)
sl0 = self.getSlice(0)
self._dtype = sl0.dtype
self._shape = [sl0.shape[0], sl0.shape[1], size_z, size_t, size_c]
