def _loadBioformats(self, filename): #from PYME.IO.FileUtils import readTiff from PYME.IO.DataSources import BioformatsDataSource try: import bioformats except ImportError: logger.exception( 'Error importing bioformats - is the python-bioformats module installed?' ) raise #mdfn = self.FindAndParseMetadata(filename) print("Bioformats:loading data") self.dataSource = BioformatsDataSource.DataSource(filename, None) self.mdh = MetaDataHandler.NestedClassMDHandler(MetaData.BareBones) print("Bioformats:loading metadata") OMEXML = bioformats.get_omexml_metadata(filename).encode('utf8') print("Bioformats:parsing metadata") OMEmd = MetaDataHandler.OMEXMLMDHandler(OMEXML) self.mdh.copyEntriesFrom(OMEmd) print("Bioformats:done") print(self.dataSource.shape) self.dataSource = BufferedDataSource.DataSource( self.dataSource, min(self.dataSource.getNumSlices(), 50)) self.data = self.dataSource #this will get replaced with a wrapped version print(self.data.shape) #from PYME.ParallelTasks.relativeFiles import getRelFilename self.seriesName = getRelFilename(filename) self.mode = 'default'
def _loadh5(self, filename): """Load PYMEs semi-custom HDF5 image data format. Offloads all the hard work to the HDFDataSource class""" import tables from PYME.IO.DataSources import HDFDataSource, BGSDataSource from PYME.IO import tabular self.dataSource = HDFDataSource.DataSource(filename, None) #chain on a background subtraction data source, so we can easily do #background subtraction in the GUI the same way as in the analysis self.data = BGSDataSource.DataSource( self.dataSource) #this will get replaced with a wrapped version if 'MetaData' in self.dataSource.h5File.root: #should be true the whole time self.mdh = MetaData.TIRFDefault self.mdh.copyEntriesFrom( MetaDataHandler.HDFMDHandler(self.dataSource.h5File)) else: self.mdh = MetaData.TIRFDefault import wx wx.MessageBox( "Carrying on with defaults - no gaurantees it'll work well", 'ERROR: No metadata found in file ...', wx.OK) print( "ERROR: No metadata fond in file ... Carrying on with defaults - no gaurantees it'll work well" ) #attempt to estimate any missing parameters from the data itself try: MetaData.fillInBlanks(self.mdh, self.dataSource) except: logger.exception('Error attempting to populate missing metadata') #calculate the name to use when we do batch analysis on this #from PYME.IO.FileUtils.nameUtils import getRelFilename self.seriesName = getRelFilename(filename) #try and find a previously performed analysis fns = filename.split(os.path.sep) cand = os.path.sep.join(fns[:-2] + [ 'analysis', ] + fns[-2:]) + 'r' print(cand) if False: #os.path.exists(cand): h5Results = tables.open_file(cand) if 'FitResults' in dir(h5Results.root): self.fitResults = h5Results.root.FitResults[:] self.resultsSource = tabular.H5RSource(h5Results) self.resultsMdh = MetaData.TIRFDefault self.resultsMdh.copyEntriesFrom( MetaDataHandler.HDFMDHandler(h5Results)) self.events = self.dataSource.getEvents() self.mode = 'LM'
def _loadNPY(self, filename): """Load numpy .npy data. """ from PYME.IO import unifiedIO mdfn = self._findAndParseMetadata(filename) with unifiedIO.local_or_temp_filename(filename) as fn: self.data = numpy.load(fn) #from PYME.ParallelTasks.relativeFiles import getRelFilename self.seriesName = getRelFilename(filename) self.mode = 'default'
def _loadImageSeries(self, filename): #from PYME.IO.FileUtils import readTiff from PYME.IO.DataSources import ImageSeriesDataSource self.dataSource = ImageSeriesDataSource.DataSource(filename, None) self.dataSource = BufferedDataSource.DataSource( self.dataSource, min(self.dataSource.getNumSlices(), 50)) self.data = self.dataSource #this will get replaced with a wrapped version #self.data = readTiff.read3DTiff(filename) self._findAndParseMetadata(filename) #from PYME.ParallelTasks.relativeFiles import getRelFilename self.seriesName = getRelFilename(filename) self.mode = 'default'
def _loadDCIMG(self, filename): from PYME.IO.DataSources import DcimgDataSource, MultiviewDataSource self._findAndParseMetadata(filename) self.dataSource = DcimgDataSource.DataSource(filename) if 'Multiview.NumROIs' in self.mdh.keys(): self.dataSource = MultiviewDataSource.DataSource( self.dataSource, self.mdh) self.data = self.dataSource #this will get replaced with a wrapped version self.seriesName = getRelFilename(filename) self.mode = 'default'
def _loadPZF(self, filename): """Load .pzf data. """ from PYME.IO import unifiedIO from PYME.IO import PZFFormat mdfn = self._findAndParseMetadata(filename) with unifiedIO.openFile(filename) as f: self.data = PZFFormat.loads(f.read())[0] #from PYME.ParallelTasks.relativeFiles import getRelFilename self.seriesName = getRelFilename(filename) self.mode = 'default'
def _loadPSF(self, filename): """Load PYME .psf data. .psf files consist of a tuple containing the data and the voxelsize. """ from PYME.IO import unifiedIO with unifiedIO.local_or_temp_filename(filename) as fn: self.data, vox = numpy.load(fn) self.mdh = MetaDataHandler.NestedClassMDHandler(MetaData.ConfocDefault) self.mdh.setEntry('voxelsize.x', vox.x) self.mdh.setEntry('voxelsize.y', vox.y) self.mdh.setEntry('voxelsize.z', vox.z) #from PYME.ParallelTasks.relativeFiles import getRelFilename self.seriesName = getRelFilename(filename) self.mode = 'psf'
def _loadDBL(self, filename): """Load Bewersdorf custom STED data. """ mdfn = self._findAndParseMetadata(filename) self.data = numpy.memmap(filename, dtype='<f4', mode='r', offset=128, shape=(self.mdh['Camera.ROIWidth'], self.mdh['Camera.ROIHeight'], self.mdh['NumImages']), order='F') #from PYME.ParallelTasks.relativeFiles import getRelFilename self.seriesName = getRelFilename(filename) self.mode = 'default'
def StartSpooling(self, fn=None, stack=False, compLevel=2, zDwellTime=None, doPreflightCheck=True, maxFrames=sys.maxsize, pzf_compression_settings=None, cluster_h5=False): """Start spooling """ # these settings were managed by the GUI, but are now managed by the controller, still allow them to be passed in, # but default to using our internal values compLevel = self.hdf_compression_level if compLevel is None else compLevel pzf_compression_settings = self.pzf_compression_settings if pzf_compression_settings is None else pzf_compression_settings stack = self.z_stepped if stack is None else stack cluster_h5 = self.cluster_h5 if cluster_h5 is None else cluster_h5 fn = self.seriesName if fn in ['', None] else fn zDwellTime = self.z_dwell if zDwellTime is None else zDwellTime #make directories as needed if not (self.spoolType == 'Cluster'): dirname = os.path.split(self._get_queue_name(fn))[0] if not os.path.exists(dirname): os.makedirs(dirname) if self._checkOutputExists( fn): #check to see if data with the same name exists self.seriesCounter += 1 self.seriesName = self._GenSeriesName() raise IOError('Output file already exists') if stack: protocol = self.protocolZ if not zDwellTime is None: protocol.dwellTime = zDwellTime print(protocol) else: protocol = self.protocol if doPreflightCheck and not preflight.ShowPreflightResults( None, self.protocol.PreflightCheck()): return #bail if we failed the pre flight check, and the user didn't choose to continue #fix timing when using fake camera if self.scope.cam.__class__.__name__ == 'FakeCamera': fakeCycleTime = self.scope.cam.GetIntegTime() else: fakeCycleTime = None frameShape = (self.scope.cam.GetPicWidth(), self.scope.cam.GetPicHeight()) if self.spoolType == 'Queue': from PYME.Acquire import QueueSpooler self.queueName = getRelFilename(self._get_queue_name(fn)) self.spooler = QueueSpooler.Spooler( self.queueName, self.scope.frameWrangler.onFrame, frameShape=frameShape, protocol=protocol, guiUpdateCallback=self._ProgressUpate, complevel=compLevel, fakeCamCycleTime=fakeCycleTime, maxFrames=maxFrames) elif self.spoolType == 'Cluster': from PYME.Acquire import HTTPSpooler self.queueName = self._get_queue_name(fn, pcs=(not cluster_h5)) self.spooler = HTTPSpooler.Spooler( self.queueName, self.scope.frameWrangler.onFrame, frameShape=frameShape, protocol=protocol, guiUpdateCallback=self._ProgressUpate, complevel=compLevel, fakeCamCycleTime=fakeCycleTime, maxFrames=maxFrames, compressionSettings=pzf_compression_settings, aggregate_h5=cluster_h5) else: from PYME.Acquire import HDFSpooler self.spooler = HDFSpooler.Spooler( self._get_queue_name(fn), self.scope.frameWrangler.onFrame, frameShape=frameShape, protocol=protocol, guiUpdateCallback=self._ProgressUpate, complevel=compLevel, fakeCamCycleTime=fakeCycleTime, maxFrames=maxFrames) #TODO - sample info is probably better handled with a metadata hook #if sampInf: # try: # sampleInformation.getSampleData(self, self.spooler.md) # except: # #the connection to the database will timeout if not present # #FIXME: catch the right exception (or delegate handling to sampleInformation module) # pass self.spooler.onSpoolStop.connect(self.SpoolStopped) self.spooler.StartSpool() self.onSpoolStart.send(self) #return a function which can be called to indicate if we are done return lambda: not self.spooler.spoolOn
def start_spooling(self, fn=None, settings={}, preflight_mode='interactive'): """ Parameters ---------- fn : str, optional fn can be hardcoded here, otherwise differs to the seriesName property which will create one if need-be. settings : dict keys should be `SpoolController` attributes or properties with setters. Not all keys must be present, and example keys include: method : str One of 'File', 'Cluster', or 'Queue'(py2 only) hdf_compression_level: int zlib compression level that pytables should use (spool to file and queue) z_stepped : bool toggle z-stepping during acquisition z_dwell : int number of frames to acquire at each z level (predicated on `SpoolController.z_stepped` being True) cluster_h5 : bool Toggle spooling to single h5 file on cluster rather than pzf file per frame. Only applicable to 'Cluster' `method` and preferred for PYMEClusterOfOne. pzf_compression_settings : dict Compression settings relevant for 'Cluster' `method` if `cluster_h5` is False. See HTTPSpooler.defaultCompSettings. protocol_name : str Note that passing the protocol name will force a (re)load of the protocol file (even if it is already selected). max_frames : int, optional point at which to end the series automatically, by default sys.maxsize subdirectory : str, optional Directory within current set directory to spool this series. The directory will be created if it doesn't already exist. extra_metadata : dict, optional metadata to supplement this series for entries known prior to acquisition which do not have handlers to hook start metadata preflight_mode : str (default='interactive') What to do when the preflight check fails. Options are 'interactive', 'warn', 'abort' and 'skip' which will display a dialog and prompt the user, log a warning and continue, and log an error and abort, or skip completely. The former is suitable for interactive acquisition, whereas one of the latter modes is likely better for automated spooling via the action manager. """ # these settings were managed by the GUI, but are now managed by the # controller, still allow them to be passed in, but default to internals fn = self.seriesName if fn in ['', None] else fn stack = settings.get('z_stepped', self.z_stepped) compLevel = settings.get('hdf_compression_level', self.hdf_compression_level) pzf_compression_settings = settings.get('pzf_compression_settings', self.pzf_compression_settings) cluster_h5 = settings.get('cluster_h5', self.cluster_h5) maxFrames = settings.get('max_frames', sys.maxsize) stack_settings = settings.get('stack_settings', None) # try stack settings for z_dwell, then aq settings. # precedence is settings > stack_settings > self.z_dwell # The reasoning for allowing the dwell time to be set in either the spooling or stack settings is to allow # API users to choose which is most coherent for their use case (it would seem logical to put dwell time with # the other stack settings, but this becomes problematic when sharing stack settings across modalities - e.g. # PALM/STORM and widefield stacks which are likely to share most of the stack settings but have greatly different # z dwell times). PYMEAcquire specifies it in the spooling/series settings by default to allow shared usage # between modalities. if stack_settings: if isinstance(stack_settings, dict): z_dwell = stack_settings.get('DwellFrames', self.z_dwell) else: # have a StackSettings object # TODO - fix this to be a bit more sane and not use private attributes etc ... z_dwell = stack_settings._dwell_frames # z_dwell defaults to -1 (with a meaning of ignore) in StackSettings objects if not value is not # explicitly provided. In this case, use our internal value instead. The reason for the 'ignore' # special value is to allow the same StackSettings object to be used for widefield stacks and # localization series (where sharing everything except dwell time makes sense). if z_dwell < 1: z_dwell = self.z_dwell else: z_dwell = self.z_dwell z_dwell = settings.get('z_dwell', z_dwell) if (stack_settings is not None) and (not isinstance(stack_settings, stackSettings.StackSettings)): # let us pass stack settings as a dict, constructing a StackSettings instance as needed stack_settings = stackSettings.StackSettings(**dict(stack_settings)) protocol_name = settings.get('protocol_name') if protocol_name is None: protocol, protocol_z = self.protocol, self.protocolZ else: pmod = prot.get_protocol(protocol_name) protocol, protocol_z = pmod.PROTOCOL, pmod.PROTOCOL_STACK subdirectory = settings.get('subdirectory', None) # make directories as needed, makedirs(dir, exist_ok=True) once py2 support is dropped if (self.spoolType != 'Cluster') and (not os.path.exists(self.get_dirname(subdirectory))): os.makedirs(self.get_dirname(subdirectory)) if self._checkOutputExists(fn): #check to see if data with the same name exists self.seriesCounter +=1 self.seriesName = self._GenSeriesName() raise IOError('A series with the same name already exists') if stack: protocol = protocol_z protocol.dwellTime = z_dwell #print(protocol) else: protocol = protocol if (preflight_mode != 'skip') and not preflight.ShowPreflightResults(protocol.PreflightCheck(), preflight_mode): return #bail if we failed the pre flight check, and the user didn't choose to continue #fix timing when using fake camera if self.scope.cam.__class__.__name__ == 'FakeCamera': fakeCycleTime = self.scope.cam.GetIntegTime() else: fakeCycleTime = None frameShape = (self.scope.cam.GetPicWidth(), self.scope.cam.GetPicHeight()) if self.spoolType == 'Queue': from PYME.Acquire import QueueSpooler self.queueName = getRelFilename(self._get_queue_name(fn, subdirectory=subdirectory)) self.spooler = QueueSpooler.Spooler(self.queueName, self.scope.frameWrangler.onFrame, frameShape = frameShape, protocol=protocol, guiUpdateCallback=self._ProgressUpate, complevel=compLevel, fakeCamCycleTime=fakeCycleTime, maxFrames=maxFrames, stack_settings=stack_settings) elif self.spoolType == 'Cluster': from PYME.Acquire import HTTPSpooler self.queueName = self._get_queue_name(fn, pcs=(not cluster_h5), subdirectory=subdirectory) self.spooler = HTTPSpooler.Spooler(self.queueName, self.scope.frameWrangler.onFrame, frameShape = frameShape, protocol=protocol, guiUpdateCallback=self._ProgressUpate, fakeCamCycleTime=fakeCycleTime, maxFrames=maxFrames, compressionSettings=pzf_compression_settings, aggregate_h5=cluster_h5, stack_settings=stack_settings) else: from PYME.Acquire import HDFSpooler self.spooler = HDFSpooler.Spooler(self._get_queue_name(fn, subdirectory=subdirectory), self.scope.frameWrangler.onFrame, frameShape = frameShape, protocol=protocol, guiUpdateCallback=self._ProgressUpate, complevel=compLevel, fakeCamCycleTime=fakeCycleTime, maxFrames=maxFrames, stack_settings=stack_settings) #TODO - sample info is probably better handled with a metadata hook #if sampInf: # try: # sampleInformation.getSampleData(self, self.spooler.md) # except: # #the connection to the database will timeout if not present # #FIXME: catch the right exception (or delegate handling to sampleInformation module) # pass extra_metadata = settings.get('extra_metadata') if extra_metadata is not None: self.spooler.md.mergeEntriesFrom(MetaDataHandler.DictMDHandler(extra_metadata)) # stop the frameWrangler before we start spooling # this serves to ensure that a) we don't accidentally spool frames which were in the camera buffer when we hit start # and b) we get a nice clean timestamp for when the actual frames start (after any protocol init tasks) # it might also slightly improve performance. self.scope.frameWrangler.stop() try: self.spooler.onSpoolStop.connect(self.SpoolStopped) self.spooler.StartSpool() except: self.spooler.abort() raise # restart frame wrangler self.scope.frameWrangler.Prepare() self.scope.frameWrangler.start() self.onSpoolStart.send(self) #return a function which can be called to indicate if we are done return lambda : self.spooler.spool_complete
def _loadTiff(self, filename): #from PYME.IO.FileUtils import readTiff from PYME.IO.DataSources import TiffDataSource, BGSDataSource mdfn = self._findAndParseMetadata(filename) self.dataSource = TiffDataSource.DataSource(filename, None) print(self.dataSource.shape) self.dataSource = BufferedDataSource.DataSource( self.dataSource, min(self.dataSource.getNumSlices(), 50)) self.data = self.dataSource #this will get replaced with a wrapped version if self.dataSource.getNumSlices( ) > 500: #this is likely to be a localization data set #background subtraction in the GUI the same way as in the analysis self.data = BGSDataSource.DataSource( self.dataSource ) #this will get replaced with a wrapped version print(self.data.shape) #if we have a multi channel data set, try and pull in all the channels if 'ChannelFiles' in self.mdh.getEntryNames() and not len( self.mdh['ChannelFiles']) == self.data.shape[3]: try: from PYME.IO.dataWrap import ListWrap #pull in all channels chans = [] for cf in self.mdh.getEntry('ChannelFiles'): cfn = os.path.join(os.path.split(filename)[0], cf) ds = TiffDataSource.DataSource(cfn, None) ds = BufferedDataSource.DataSource( ds, min(ds.getNumSlices(), 50)) chans.append(ds) self.data = ListWrap( chans) #this will get replaced with a wrapped version self.filename = mdfn except: pass elif 'ChannelNames' in self.mdh.getEntryNames() and len( self.mdh['ChannelNames']) == self.data.getNumSlices(): from PYME.IO.dataWrap import ListWrap chans = [ numpy.atleast_3d(self.data.getSlice(i)) for i in range(len(self.mdh['ChannelNames'])) ] self.data = ListWrap(chans) elif filename.endswith( '.lsm') and 'LSM.images_number_channels' in self.mdh.keys( ) and self.mdh['LSM.images_number_channels'] > 1: from PYME.IO.dataWrap import ListWrap nChans = self.mdh['LSM.images_number_channels'] chans = [] for n in range(nChans): ds = TiffDataSource.DataSource(filename, None, n) ds = BufferedDataSource.DataSource(ds, min(ds.getNumSlices(), 50)) chans.append(ds) self.data = ListWrap(chans) #from PYME.ParallelTasks.relativeFiles import getRelFilename self.seriesName = getRelFilename(filename) self.mode = 'default' if self.mdh.getOrDefault('ImageType', '') == 'PSF': self.mode = 'psf' elif self.dataSource.getNumSlices() > 5000: #likely to want to localize this self.mode = 'LM'
def _loadBioformats(self, filename): try: import bioformats except ImportError: logger.exception( 'Error importing bioformats - is the python-bioformats module installed?' ) raise from PYME.IO.DataSources import BioformatsDataSource series_num = None if '?' in filename: #we have a query string to pick the series from six.moves import urllib filename, query = filename.split('?') try: series_num = int(urllib.parse.parse_qs(query)['series'][0]) except KeyError: pass #mdfn = self.FindAndParseMetadata(filename) print("Bioformats:loading data") bioformats_file = BioformatsDataSource.BioformatsFile(filename) if series_num is None and bioformats_file.series_count > 1: print('File has multiple series, need to pick one.') if self.haveGUI: import wx dlg = wx.SingleChoiceDialog(None, 'Series', 'Select a series', bioformats_file.series_names) if dlg.ShowModal() == wx.ID_OK: series_num = dlg.GetSelection() else: logger.warning('No GUI, using 0th series.') self.dataSource = BioformatsDataSource.DataSource(bioformats_file, series=series_num) self.mdh = MetaDataHandler.NestedClassMDHandler(MetaData.BareBones) # NOTE: We are triple-loading metadata. BioformatsDataSource.BioformatsFile.rdr has metadata (hard to access), # BioformatsDataSource.BioformatsFile has metadata, and now we are loading the same metadata again here. print("Bioformats:loading metadata") OMEXML = bioformats.get_omexml_metadata(filename).encode('utf8') print("Bioformats:parsing metadata") OMEmd = MetaDataHandler.OMEXMLMDHandler(OMEXML) self.mdh.copyEntriesFrom(OMEmd) print("Bioformats:done") #fix voxelsizes if not specified in OME metadata if self.haveGUI and ((self.mdh['voxelsize.x'] < 0) or (self.mdh['voxelsize.y'] < 0)): from PYME.DSView.voxSizeDialog import VoxSizeDialog dlg = VoxSizeDialog(None) dlg.ShowModal() self.mdh.setEntry('voxelsize.x', dlg.GetVoxX()) self.mdh.setEntry('voxelsize.y', dlg.GetVoxY()) self.mdh.setEntry('voxelsize.z', dlg.GetVoxZ()) dlg.Destroy() print(self.dataSource.shape) self.dataSource = BufferedDataSource.DataSource( self.dataSource, min(self.dataSource.getNumSlices(), 50)) self.data = self.dataSource #this will get replaced with a wrapped version print(self.data.shape) #from PYME.ParallelTasks.relativeFiles import getRelFilename self.seriesName = getRelFilename(filename) self.mode = 'default'