def __init__(self, filename='', metadata=None, taskQueue=None): if filename is not '': self.filename = getFullExistingFilename( filename) #convert relative path to full path #use metadata for glob md = MetaDataHandler.SimpleMDHandler(self.filename) else: md = metadata self.filename = '' pattern = md.getEntry('SeriesPattern') self.files = glob.glob( os.path.join(os.path.split(self.filename)[0], pattern)) self.files.sort() f0 = self.files[0] if f0.endswith('.tif'): self.im0 = tifffile.imread(f0).squeeze() self._slice_shape = self.im0.shape[:2] else: self.im0 = Image.open(self.files[0]) self._slice_shape = self.im0.size[1], self.im0.size[0] #self.im.seek(0) #PIL's endedness support is subtly broken - try to fix it #NB this is untested for floating point tiffs self.endedness = 'LE' if self.im0.ifd.prefix == 'MM': self.endedness = 'BE' print((self.im0.ifd.prefix)) print((self.endedness))
def setModel(modName, md): global IntXVals, IntYVals, IntZVals, dx, dy, dz #FIXME - use load_psf() instead mf = open(getFullExistingFilename(modName), 'rb') mod, voxelsize = pickle.load(mf) mf.close() mod = resizePSF(mod, interpModel().shape) #if not voxelsize.x == md.voxelsize.x: # raise RuntimeError("PSF and Image voxel sizes don't match") IntXVals = 1e3 * voxelsize.x * mgrid[-(mod.shape[0] / 2.):(mod.shape[0] / 2.)] IntYVals = 1e3 * voxelsize.y * mgrid[-(mod.shape[1] / 2.):(mod.shape[1] / 2.)] IntZVals = 1e3 * voxelsize.z * mgrid[-(mod.shape[2] / 2.):(mod.shape[2] / 2.)] dx = voxelsize.x * 1e3 dy = voxelsize.y * 1e3 dz = voxelsize.z * 1e3 #interpModel = mod #interpModel = np.maximum(mod/mod.max(), 0) #normalise to 1 interpModel_by_chan[0] = np.maximum( mod / mod[:, :, len(IntZVals) / 2].sum(), 0) #normalise to 1 and clip
def __init__(self, image_file, taskQueue=None, chanNum = 0, series=None): self.chanNum = chanNum if isinstance(image_file, BioformatsFile): self.bff = image_file else: self.filename = getFullExistingFilename(image_file)#convert relative path to full path print(self.filename) self.bff = BioformatsFile(self.filename) if series is not None: self.bff.rdr.setSeries(series) self.sizeX = self.bff.rdr.getSizeX() self.sizeY = self.bff.rdr.getSizeY() self.sizeZ = self.bff.rdr.getSizeZ() self.sizeT = self.bff.rdr.getSizeT() self.sizeC = self.bff.rdr.getSizeC() #self.shape = [self.sizeX, self.sizeY, self.sizeZ*self.sizeT, self.sizeC] #axisOrder = self.bff.rdr.getDimensionOrder() #sh = {'X' : self.sizeX, 'Y':self.sizeY, 'Z':self.sizeZ, 'T':self.sizeT, 'C':self.sizeT} self.additionalDims = 'TC'
def __init__(self, filename, taskQueue=None): self.filename = getFullExistingFilename( filename) #convert relative path to full path #self.data = readTiff.read3DTiff(self.filename) #use metadata for glob md = MetaDataHandler.SimpleMDHandler(self.filename) pattern = md.getEntry('SeriesPattern') self.files = glob.glob( os.path.join(os.path.split(self.filename)[0], pattern)) self.files.sort() self.im0 = Image.open(self.files[0]) #self.im.seek(0) #PIL's endedness support is subtly broken - try to fix it #NB this is untested for floating point tiffs self.endedness = 'LE' if self.im0.ifd.prefix == 'MM': self.endedness = 'BE' print((self.im0.ifd.prefix)) print((self.endedness))
def read(filename): ''' Read a file from disk or the cluster. NOTE: filename is expected to be sanitized / trusted, this should not be called with user data from a web endpoint. Parameters ---------- filename Returns ------- ''' filename = nameUtils.getFullExistingFilename(filename) if os.path.exists(filename): with open(filename, 'rb') as f: s = f.read() return s elif is_cluster_uri(filename): from . import clusterIO sequenceName, clusterfilter = split_cluster_url(filename) s = clusterIO.get_file(sequenceName, clusterfilter) return s else: raise IOError('File does not exist or URI not understood: %s' % filename)
def local_or_temp_filename(url): """ Gets a local filename for a given url. The purpose is to let us load files from the cluster using IO libraries (e.g. pytables, tifffile) which need a filename, not a file handle. Does the following (in order). * checks to see if the url refers to a local file, if so return the filename * if the url points to a cluster file, check to see if it happens to be stored locally. If so, return the local path * download file to a temporary file and return the filename of that temporary file. NB: This should be used as a context manager (ie in a with statement) so that the temporary file gets cleaned up properly Parameters ---------- url : basestring Returns ------- filename : basestring Notes ----- TODO - there is potential for optimization by using memmaps rather than files on disk. """ filename = nameUtils.getFullExistingFilename(url) if os.path.exists(filename): yield filename elif filename.startswith('pyme-cluster') or filename.startswith( 'PYME-CLUSTER'): from . import clusterIO sequenceName, clusterfilter = split_cluster_url(filename) localpath = clusterIO.get_local_path(sequenceName, clusterfilter) if localpath: yield localpath else: ext = os.path.splitext(sequenceName)[-1] with tempfile.NamedTemporaryFile(mode='w+b', suffix=ext) as outf: s = clusterIO.get_file(sequenceName, clusterfilter) outf.write(s) outf.flush() yield outf.name else: raise IOError('Path "%s" could not be found' % url)
def _fetchMap(self, md, mapName): """retrive a map, with a given name. First try and get it from the Queue, then try finding it locally""" try: map = md.taskQueue.getQueueData(md.dataSourceID, 'MAP', mapName) except: fn = getFullExistingFilename(mapName) map = ImageStack(filename=fn, haveGUI=False).data[:,:,0].squeeze() #this should handle .tif, .h5, and a few others return map
def __init__(self, filename, taskQueue=None, chanNum = 0): self.filename = getFullExistingFilename(filename)#convert relative path to full path self.chanNum = chanNum self.RGB = False #self.data = readTiff.read3DTiff(self.filename) #self.im = Image.open(filename) #self.im.seek(0) #PIL's endedness support is subtly broken - try to fix it #NB this is untested for floating point tiffs #self.endedness = 'LE' #if self.im.ifd.prefix =='MM': # self.endedness = 'BE' #to find the number of images we have to loop over them all #this is obviously not ideal as PIL loads the image data into memory for each #slice and this is going to represent a huge performance penalty for large stacks #should still let them be opened without having all images in memory at once though #self.numSlices = self.im.tell() #try: # while True: # self.numSlices += 1 # self.im.seek(self.numSlices) #except EOFError: # pass print((self.filename)) tf = tifffile.TIFFfile(self.filename) print(tf.series[0].shape) self.im = tf.series[0].pages if tf.is_ome: print('Detected OME TIFF') sh = dict(zip(tf.series[0].axes, tf.series[0].shape)) print('sh = %s' % sh) self.sizeC = sh['C'] axisOrder = tf.series[0].axes[::-1] self.additionalDims = ''.join([a for a in axisOrder[2:] if sh[a] > 1]) elif tf.is_rgb: print('Detected RGB TIFF') self.sizeC = 3 self.RGB = True self.additionalDims = 'C'
def getQueueData(self, fieldName, *args): """Get data, defined by fieldName and potntially additional arguments, ascociated with queue""" if fieldName == 'FitResults': startingAt, = args #with self.fileResultsLock.rlock: # if self.h5ResultsFile.__contains__('/FitResults'): # res = self.h5ResultsFile.root.FitResults[startingAt:] # else: # res = [] with h5rFile.openH5R(self.resultsFilename, 'a') as h5f: res = h5f.getTableData('FitResults', slice(startingAt, None)) return res elif fieldName == 'PSF': #from PYME.ParallelTasks.relativeFiles import getFullExistingFilename from PYME.IO.load_psf import load_psf res = None modName = self.metaData.getEntry('PSFFile') # mf = open(getFullExistingFilename(modName), 'rb') #res = np.load(mf) #mf.close() res = load_psf(getFullExistingFilename(modName)) return res elif fieldName == 'MAP': mapName, = args #from PYME.ParallelTasks.relativeFiles import getFullExistingFilename from PYME.IO.image import ImageStack print('Serving map: %s' % mapName) fn = getFullExistingFilename(mapName) varmap = ImageStack( filename=fn, haveGUI=False).data[:, :, 0].squeeze( ) #this should handle .tif, .h5, and a few others return varmap else: return None
def openFile(filename, mode='rb'): filename = nameUtils.getFullExistingFilename(filename) if os.path.exists(filename): return open(filename, mode) elif filename.startswith('pyme-cluster') or filename.startswith('PYME-CLUSTER'): #TODO - add short-circuiting for local files from . import clusterIO sequenceName, clusterfilter = split_cluster_url(filename) s = clusterIO.get_file(sequenceName, clusterfilter) return BytesIO(s) else: raise IOError('File does not exist or URI not understood: %s' % filename)
def __init__(self, h5Filename, taskQueue=None): self.h5Filename = getFullExistingFilename( h5Filename) #convert relative path to full path self.h5File = tables.open_file(self.h5Filename) self._pzf_index = None if getattr(self.h5File.root, 'PZFImageIndex', False): self.usePZFFormat = True try: self.framesize = self.h5File.root.PZFImageData.attrs.framesize except AttributeError: self.framesize = PZFFormat.loads( self.h5File.root.PZFImageData[0])[0].squeeze().shape else: self.usePZFFormat = False
def read(filename): filename = nameUtils.getFullExistingFilename(filename) if os.path.exists(filename): with open(filename, 'rb') as f: s = f.read() return s elif filename.startswith('pyme-cluster') or filename.startswith('PYME-CLUSTER'): from . import clusterIO sequenceName, clusterfilter = split_cluster_url(filename) s = clusterIO.get_file(sequenceName, clusterfilter) return s else: raise IOError('File does not exist or URI not understood: %s' % filename)
def __init__(self, h5Filename, taskQueue=None): self.h5Filename = getFullExistingFilename( h5Filename) #convert relative path to full path self.h5File = tables.open_file(self.h5Filename) self._pzf_index = None if getattr(self.h5File.root, 'PZFImageIndex', False): self.usePZFFormat = True try: self.framesize = self.h5File.root.PZFImageData.attrs.framesize except AttributeError: self.framesize = PZFFormat.loads( self.h5File.root.PZFImageData[0])[0].squeeze().shape else: self.usePZFFormat = False try: dimorder = self._img_data.attrs.DimOrder if isinstance(dimorder, bytes): dimorder = dimorder.decode() assert (dimorder[:2] == 'XY') size_c = int(self._img_data.attrs.SizeC) size_z = int(self._img_data.attrs.SizeZ) size_t = int(self._img_data.attrs.SizeT) except: logger.exception( 'Error reading dim info (can be safely ignored for old files)') dimorder = 'XYZTC' size_z = self.getNumSlices() size_t = 1 size_c = 1 XYZTCDataSource.__init__(self, dimorder, size_z=size_z, size_t=size_t, size_c=size_c) self._shape = tuple(self.getSliceShape()) + (size_z, size_t, size_c) self._dtype = self.getSlice(0).dtype
def local_or_named_temp_filename(url): """ riff of PYME.IO.clusterIO.local_or_temp_filename, but one which returns a filename with a matching file stub to the original rather than a random temporary filename Parameters ---------- url : str local path or pyme-cluster url Yields ------- str path to a (temporary) file so `url` can be loaded using modules which expect a local filename """ from PYME.IO.FileUtils import nameUtils from PYME.IO import unifiedIO import tempfile filename = nameUtils.getFullExistingFilename(url) if os.path.exists(filename): yield filename elif unifiedIO.is_cluster_uri(url): from PYME.IO import clusterIO name, clusterfilter = unifiedIO.split_cluster_url(filename) localpath = clusterIO.get_local_path(name, clusterfilter) if localpath: yield localpath else: ext = os.path.splitext(name)[-1] with tempfile.TemporaryDirectory() as temp_dir: with open(os.path.join(temp_dir, os.path.split(name)[-1]), 'wb') as f: s = clusterIO.get_file(name, clusterfilter) f.write(s) f.flush() yield f.name else: raise IOError('Path "%s" could not be found' % url)
def __init__(self, h5Filename, taskQueue=None): self.h5Filename = getFullExistingFilename( h5Filename) #convert relative path to full path self.h5File = tables.open_file(self.h5Filename) self._pzf_index = None if getattr(self.h5File.root, 'PZFImageIndex', False): self.usePZFFormat = True try: self.framesize = self.h5File.root.PZFImageData.attrs.framesize except AttributeError: self.framesize = PZFFormat.loads( self.h5File.root.PZFImageData[0])[0].squeeze().shape else: self.usePZFFormat = False try: self.dimorder = self._img_data.attrs.DimOrder if isinstance(self.dimorder, bytes): self.dimorder = self.dimorder.decode() assert (self.dimorder[:2] == 'XY') self.sizeC = int(self._img_data.attrs.SizeC) self.sizeZ = int(self._img_data.attrs.SizeZ) self.sizeT = int(self._img_data.attrs.SizeT) # FIXME - we currently ignore SizeZ and SizeT and collapse to one dimension (to fit with the XY[Z/T]C data model) # This should be changed once we fully move to an xyztc model. In the meantime, it's probably safest if C is # always the last dimension. if self.sizeC > 1: if self.dimorder[-1] == 'C': self.additionalDims = 'TC' else: self.additionalDims = 'CT' else: self.additionalDims = 'T' except: logger.exception( 'Error reading dim info (can be safely ignored for old files)') pass
def __init__(self, filename, taskQueue=None, chanNum=0, series=0): self.filename = getFullExistingFilename( filename) #convert relative path to full path self.chanNum = chanNum self.RGB = False #self.data = readTiff.read3DTiff(self.filename) #self.im = Image.open(filename) #self.im.seek(0) #PIL's endedness support is subtly broken - try to fix it #NB this is untested for floating point tiffs #self.endedness = 'LE' #if self.im.ifd.prefix =='MM': # self.endedness = 'BE' #to find the number of images we have to loop over them all #this is obviously not ideal as PIL loads the image data into memory for each #slice and this is going to represent a huge performance penalty for large stacks #should still let them be opened without having all images in memory at once though #self.numSlices = self.im.tell() #try: # while True: # self.numSlices += 1 # self.im.seek(self.numSlices) #except EOFError: # pass print((self.filename)) if local_tifffile: logger.info( 'Using PYMEs built-in, old version of tifffile, better support for ImageJ tiffs can be had with the more recent pip version (`pip install tifffile`)' ) tf = tifffile.TIFFfile(self.filename) else: tf = tifffile.TiffFile(self.filename) self.tf = tf # keep a reference for debugging print(tf.series[series].shape) self.im = tf.series[series].pages axisOrder = 'XYZTC' size_z = len(self.im) size_c, size_t = 1, 1 if tf.is_ome or ((not local_tifffile)): #print('Detected OME TIFF') sh = {'Z': 1, 'T': 1, 'C': 1} _axes = tf.series[series].axes if 'I' in _axes: logger.info('Tiff file does not fully specify axes (axes=%s)' % (_axes.replace('I', '?')[::-1])) if 'Z' not in _axes: logger.info('Assuming unknown axis is Z' ) # TODO - explain how to change axes later _axes = _axes.replace('I', 'Z') elif 'C' not in _axes: logger.info('Assuming unknown axis is C') _axes = _axes.replace('I', 'C') elif 'T' not in _axes: logger.info('Assuming unkown axis is T') _axes = _axes.replace('I', 'T') else: logger.warning( 'Unknown axis with all standard axes defined - data might not read correctly' ) sh.update(dict(zip(_axes, tf.series[0].shape))) logger.debug('sh = %s' % sh) size_c = sh['C'] size_z = sh['Z'] size_t = sh['T'] axisOrder = _axes[::-1] axisOrder = axisOrder + ''.join( [a for a in ['Z', 'T', 'C'] if not a in axisOrder]) logger.debug('raw TIFF axisOrder = %s' % axisOrder) #self.additionalDims = ''.join([a for a in axisOrder[2:] if sh[a] > 1]) elif tf.is_rgb: print( 'WARNING: Detected RGB TIFF - data not likely to be suitable for quantitative analysis' ) size_c = 3 self.RGB = True if len(self.im) > 1: # we can have multi-page RGB TIFF - why????? print( 'WARNING: Multi-page RGB TIFF detected - where did this come from???' ) axisOrder = 'XYCZT' size_z = len(self.im) size_t = 1 XYZTCDataSource.__init__(self, input_order=axisOrder, size_z=size_z, size_t=size_t, size_c=size_c) sl0 = self.getSlice(0) self._dtype = sl0.dtype self._shape = [sl0.shape[0], sl0.shape[1], size_z, size_t, size_c]
def __init__(self, filename, taskQueue=None): self.filename = getFullExistingFilename( filename) #convert relative path to full path self.DCIFile = dcimg.DCIMGFile(self.filename)
def dirname(url): try: return os.path.dirname(split_cluster_url(url)[0]) except RuntimeError: return os.path.dirname(nameUtils.getFullExistingFilename(url))
def __init__(self, filename, taskQueue=None, chanNum=0, series=0): self.filename = getFullExistingFilename( filename) #convert relative path to full path self.chanNum = chanNum self.RGB = False #self.data = readTiff.read3DTiff(self.filename) #self.im = Image.open(filename) #self.im.seek(0) #PIL's endedness support is subtly broken - try to fix it #NB this is untested for floating point tiffs #self.endedness = 'LE' #if self.im.ifd.prefix =='MM': # self.endedness = 'BE' #to find the number of images we have to loop over them all #this is obviously not ideal as PIL loads the image data into memory for each #slice and this is going to represent a huge performance penalty for large stacks #should still let them be opened without having all images in memory at once though #self.numSlices = self.im.tell() #try: # while True: # self.numSlices += 1 # self.im.seek(self.numSlices) #except EOFError: # pass print((self.filename)) if local_tifffile: tf = tifffile.TIFFfile(self.filename) else: tf = tifffile.TiffFile(self.filename) self.tf = tf # keep a reference for debugging print(tf.series[series].shape) self.im = tf.series[series].pages axisOrder = 'XYZTC' size_z = len(self.im) size_c, size_t = 1, 1 if tf.is_ome or (not local_tifffile): print('Detected OME TIFF') sh = {'Z': 1, 'T': 1, 'C': 1} sh.update(dict(zip(tf.series[series].axes, tf.series[0].shape))) print('sh = %s' % sh) size_c = sh['C'] size_z = sh['Z'] size_t = sh['T'] axisOrder = tf.series[series].axes[::-1] axisOrder = axisOrder + ''.join( [a for a in ['Z', 'T', 'C'] if not a in axisOrder]) print('axisOrder = ', axisOrder) #self.additionalDims = ''.join([a for a in axisOrder[2:] if sh[a] > 1]) elif tf.is_rgb: print( 'WARNING: Detected RGB TIFF - data not likely to be suitable for quantitative analysis' ) size_c = 3 self.RGB = True if len(self.im) > 1: # we can have multi-page RGB TIFF - why????? print( 'WARNING: Multi-page RGB TIFF detected - where did this come from???' ) axisOrder = 'XYCZT' size_z = len(self.im) size_t = 1 XYZTCDataSource.__init__(self, input_order=axisOrder, size_z=size_z, size_t=size_t, size_c=size_c) sl0 = self.getSlice(0) self._dtype = sl0.dtype self._shape = [sl0.shape[0], sl0.shape[1], size_z, size_t, size_c]