def pushImagesHDF(self, startingAt=0, detThresh = .9, fitFcn = 'LatGaussFitFR'): #global seriesName dataFilename = self.image.seriesName resultsFilename = genResultFileName(self.image.seriesName) if os.path.exists(resultsFilename): di, fn = os.path.split(resultsFilename) i = 1 stub = os.path.splitext(fn)[0] while os.path.exists(os.path.join(di, stub + '_%d.h5r' % i)): i += 1 fdialog = wx.FileDialog(None, 'Analysis file already exists, please select a new filename', wildcard='H5R files|*.h5r', defaultDir=di, defaultFile=stub + '_%d.h5r' % i, style=wx.SAVE) succ = fdialog.ShowModal() if (succ == wx.ID_OK): resultsFilename = fdialog.GetPath().encode() else: raise RuntimeError('Invalid results file - not running') self.image.seriesName = resultsFilename self.tq.createQueue('HDFTaskQueue', self.image.seriesName, dataFilename = dataFilename, resultsFilename=resultsFilename, startAt = 'notYet') mdhQ = MetaDataHandler.QueueMDHandler(self.tq, self.image.seriesName, self.image.mdh) mdhQ.setEntry('Analysis.DetectionThreshold', detThresh) mdhQ.setEntry('Analysis.FitModule', fitFcn) mdhQ.setEntry('Analysis.DataFileID', fileID.genDataSourceID(self.image.dataSource)) evts = self.image.dataSource.getEvents() if len(evts) > 0: self.tq.addQueueEvents(self.image.seriesName, evts) self.tq.releaseTasks(self.image.seriesName, startingAt)
def pushImagesDS(self, startingAt=0, detThresh = .9, fitFcn = 'LatGaussFitFR'): #global seriesName #dataFilename = self.image.seriesName resultsFilename = genResultFileName(self.image.seriesName) while os.path.exists(resultsFilename): di, fn = os.path.split(resultsFilename) fdialog = wx.FileDialog(None, 'Analysis file already exists, please select a new filename', wildcard='H5R files|*.h5r', defaultDir=di, defaultFile=os.path.splitext(fn)[0] + '_1.h5r', style=wx.SAVE) succ = fdialog.ShowModal() if (succ == wx.ID_OK): resultsFilename = fdialog.GetPath().encode() else: raise RuntimeError('Invalid results file - not running') #self.image.seriesName = resultsFilename debugPrint('Results file = %s' % resultsFilename) self.tq.createQueue('HDFResultsTaskQueue', resultsFilename, None) debugPrint('Queue created') mdhQ = MetaDataHandler.QueueMDHandler(self.tq, resultsFilename, self.image.mdh) mdhQ.setEntry('Analysis.DetectionThreshold', detThresh) mdhQ.setEntry('Analysis.FitModule', fitFcn) mdhQ.setEntry('Analysis.DataFileID', fileID.genDataSourceID(self.image.dataSource)) debugPrint('Metadata transferred to queue') evts = self.image.dataSource.getEvents() if len(evts) > 0: self.tq.addQueueEvents(resultsFilename, evts) md = MetaDataHandler.NestedClassMDHandler(mdhQ) mn = self.image.dataSource.moduleName #if it's a buffered source, go back to underlying source if mn == 'BufferedDataSource': mn = self.image.dataSource.dataSource.moduleName for i in range(startingAt, self.image.dataSource.getNumSlices()): debugPrint('Posting task %d' %i) if 'Analysis.BGRange' in md.getEntryNames(): bgi = range(max(i + md.Analysis.BGRange[0],md.EstimatedLaserOnFrameNo), max(i + md.Analysis.BGRange[1],md.EstimatedLaserOnFrameNo)) elif 'Analysis.NumBGFrames' in md.getEntryNames(): bgi = range(max(i - md.Analysis.NumBGFrames, md.EstimatedLaserOnFrameNo), i) else: bgi = range(max(i - 10, md.EstimatedLaserOnFrameNo), i) #task = fitTask(self.queueID, taskNum, self.metaData.Analysis.DetectionThreshold, self.metaData, self.metaData.Analysis.FitModule, 'TQDataSource', bgindices =bgi, SNThreshold = True) self.tq.postTask(remFitBuf.fitTask(self.image.seriesName,i, detThresh, md, fitFcn, bgindices=bgi, SNThreshold=True, dataSourceModule=mn), queueName=resultsFilename) self.image.seriesName = resultsFilename
def __init__(self, name, resultsFilename, initialTasks=[], onEmpty=doNix, fTaskToPop=popZero): if resultsFilename is None: resultsFilename = genResultFileName(name) if os.path.exists( resultsFilename): #bail if output file already exists raise RuntimeError('Output file already exists: ' + resultsFilename) TaskQueue.__init__(self, name, initialTasks, onEmpty, fTaskToPop) self.resultsFilename = resultsFilename self.numClosedTasks = 0 logging.info('Creating results file') self.h5ResultsFile = tables.openFile(self.resultsFilename, 'w') self.prepResultsFile() # pass #self.fileResultsLock = threading.Lock() self.fileResultsLock = tablesLock logging.info('Creating results metadata') self.resultsMDH = MetaDataHandler.HDFMDHandler(self.h5ResultsFile) self.metaData = MetaDataHandler.NestedClassMDHandler() #self.metaData = None #MetaDataHandler.NestedClassMDHandler(self.resultsMDH) self.metaDataStale = True self.MDHCache = [] logging.info('Creating results events table') with self.fileResultsLock.wlock: self.resultsEvents = self.h5ResultsFile.createTable( self.h5ResultsFile.root, 'Events', SpoolEvent, filters=tables.Filters(complevel=5, shuffle=True)) logging.info('Events table created') self.haveResultsTable = False self.resultsQueue = [] #Queue.Queue() self.resultsQueueLock = threading.Lock() self.lastResultsQueuePurge = time.time() logging.info('Results file initialised')
def pushImagesHDF(self, startingAt=0, detThresh=.9, fitFcn='LatGaussFitFR'): #global seriesName dataFilename = self.image.seriesName resultsFilename = genResultFileName(self.image.seriesName) if os.path.exists(resultsFilename): di, fn = os.path.split(resultsFilename) i = 1 stub = os.path.splitext(fn)[0] while os.path.exists(os.path.join(di, stub + '_%d.h5r' % i)): i += 1 fdialog = wx.FileDialog( None, 'Analysis file already exists, please select a new filename', wildcard='H5R files|*.h5r', defaultDir=di, defaultFile=stub + '_%d.h5r' % i, style=wx.SAVE) succ = fdialog.ShowModal() if (succ == wx.ID_OK): resultsFilename = fdialog.GetPath().encode() else: raise RuntimeError('Invalid results file - not running') self.image.seriesName = resultsFilename self.tq.createQueue('HDFTaskQueue', self.image.seriesName, dataFilename=dataFilename, resultsFilename=resultsFilename, startAt='notYet') mdhQ = MetaDataHandler.QueueMDHandler(self.tq, self.image.seriesName, self.image.mdh) mdhQ.setEntry('Analysis.DetectionThreshold', detThresh) mdhQ.setEntry('Analysis.FitModule', fitFcn) mdhQ.setEntry('Analysis.DataFileID', fileID.genDataSourceID(self.image.dataSource)) evts = self.image.dataSource.getEvents() if len(evts) > 0: self.tq.addQueueEvents(self.image.seriesName, evts) self.tq.releaseTasks(self.image.seriesName, startingAt)
def __init__(self, name, resultsFilename, initialTasks=[], onEmpty = doNix, fTaskToPop = popZero): if resultsFilename is None: resultsFilename = genResultFileName(name) if os.path.exists(resultsFilename): #bail if output file already exists raise RuntimeError('Output file already exists: ' + resultsFilename) TaskQueue.__init__(self, name, initialTasks, onEmpty, fTaskToPop) self.resultsFilename = resultsFilename self.numClosedTasks = 0 logging.info('Creating results file') self.h5ResultsFile = tables.openFile(self.resultsFilename, 'w') self.prepResultsFile() # pass #self.fileResultsLock = threading.Lock() self.fileResultsLock = tablesLock logging.info('Creating results metadata') self.resultsMDH = MetaDataHandler.HDFMDHandler(self.h5ResultsFile) self.metaData = MetaDataHandler.NestedClassMDHandler() #self.metaData = None #MetaDataHandler.NestedClassMDHandler(self.resultsMDH) self.metaDataStale = True self.MDHCache = [] logging.info('Creating results events table') with self.fileResultsLock.wlock: self.resultsEvents = self.h5ResultsFile.createTable(self.h5ResultsFile.root, 'Events', SpoolEvent,filters=tables.Filters(complevel=5, shuffle=True)) logging.info('Events table created') self.haveResultsTable = False self.resultsQueue = []#Queue.Queue() self.resultsQueueLock = threading.Lock() self.lastResultsQueuePurge = time.time() logging.info('Results file initialised')
def __init__(self, name, dataFilename = None, resultsFilename=None, onEmpty = doNix, fTaskToPop = popZero, startAt = 'guestimate', frameSize=(-1,-1), complevel=6, complib='zlib'): if dataFilename is None: self.dataFilename = genDataFilename(name) else: self.dataFilename = dataFilename if resultsFilename is None: resultsFilename = genResultFileName(self.dataFilename) else: resultsFilename = resultsFilename ffn = getFullFilename(self.dataFilename) self.acceptNewTasks = False self.releaseNewTasks = False self.postTaskBuffer = [] initialTasks = [] if os.path.exists(ffn): #file already exists - read from it self.h5DataFile = tables.openFile(ffn, 'r') #self.metaData = MetaData.genMetaDataFromHDF(self.h5DataFile) self.dataMDH = MetaDataHandler.NestedClassMDHandler(MetaDataHandler.HDFMDHandler(self.h5DataFile)) #self.dataMDH.mergeEntriesFrom(MetaData.TIRFDefault) self.imageData = self.h5DataFile.root.ImageData if startAt == 'guestimate': #calculate a suitable starting value tLon = self.dataMDH.EstimatedLaserOnFrameNo if tLon == 0: startAt = 0 else: startAt = tLon + 10 if startAt == 'notYet': initialTasks = [] else: initialTasks = list(range(startAt, self.h5DataFile.root.ImageData.shape[0])) self.imNum = len(self.imageData) self.dataRW = False else: #make ourselves a new file self.h5DataFile = tables.openFile(ffn, 'w') filt = tables.Filters(complevel, complib, shuffle=True) self.imageData = self.h5DataFile.createEArray(self.h5DataFile.root, 'ImageData', tables.UInt16Atom(), (0,)+tuple(frameSize), filters=filt, chunkshape=(1,)+tuple(frameSize)) self.events = self.h5DataFile.createTable(self.h5DataFile.root, 'Events', SpoolEvent,filters=filt) self.imNum=0 self.acceptNewTasks = True self.dataMDH = MetaDataHandler.HDFMDHandler(self.h5DataFile) self.dataMDH.mergeEntriesFrom(MetaData.TIRFDefault) self.dataRW = True HDFResultsTaskQueue.__init__(self, name, resultsFilename, initialTasks, onEmpty, fTaskToPop) self.resultsMDH.copyEntriesFrom(self.dataMDH) self.metaData.copyEntriesFrom(self.resultsMDH) #copy events to results file if len (self.h5DataFile.root.Events) > 0: self.resultsEvents.append(self.h5DataFile.root.Events[:]) self.queueID = name self.numSlices = self.imageData.shape[0] #self.dataFileLock = threading.Lock() self.dataFileLock = tablesLock #self.getTaskLock = threading.Lock() self.lastTaskTime = 0
def __init__(self, name, dataFilename=None, resultsFilename=None, onEmpty=doNix, fTaskToPop=popZero, startAt='guestimate', frameSize=(-1, -1), complevel=6, complib='zlib'): if dataFilename is None: self.dataFilename = genDataFilename(name) else: self.dataFilename = dataFilename if resultsFilename is None: resultsFilename = genResultFileName(self.dataFilename) else: resultsFilename = resultsFilename ffn = getFullFilename(self.dataFilename) self.acceptNewTasks = False self.releaseNewTasks = False self.postTaskBuffer = [] initialTasks = [] if os.path.exists(ffn): #file already exists - read from it self.h5DataFile = tables.openFile(ffn, 'r') #self.metaData = MetaData.genMetaDataFromHDF(self.h5DataFile) self.dataMDH = MetaDataHandler.NestedClassMDHandler( MetaDataHandler.HDFMDHandler(self.h5DataFile)) #self.dataMDH.mergeEntriesFrom(MetaData.TIRFDefault) self.imageData = self.h5DataFile.root.ImageData if startAt == 'guestimate': #calculate a suitable starting value tLon = self.dataMDH.EstimatedLaserOnFrameNo if tLon == 0: startAt = 0 else: startAt = tLon + 10 if startAt == 'notYet': initialTasks = [] else: initialTasks = list( range(startAt, self.h5DataFile.root.ImageData.shape[0])) self.imNum = len(self.imageData) self.dataRW = False else: #make ourselves a new file self.h5DataFile = tables.openFile(ffn, 'w') filt = tables.Filters(complevel, complib, shuffle=True) self.imageData = self.h5DataFile.createEArray( self.h5DataFile.root, 'ImageData', tables.UInt16Atom(), (0, ) + tuple(frameSize), filters=filt, chunkshape=(1, ) + tuple(frameSize)) self.events = self.h5DataFile.createTable(self.h5DataFile.root, 'Events', SpoolEvent, filters=filt) self.imNum = 0 self.acceptNewTasks = True self.dataMDH = MetaDataHandler.HDFMDHandler(self.h5DataFile) self.dataMDH.mergeEntriesFrom(MetaData.TIRFDefault) self.dataRW = True HDFResultsTaskQueue.__init__(self, name, resultsFilename, initialTasks, onEmpty, fTaskToPop) self.resultsMDH.copyEntriesFrom(self.dataMDH) self.metaData.copyEntriesFrom(self.resultsMDH) #copy events to results file if len(self.h5DataFile.root.Events) > 0: self.resultsEvents.append(self.h5DataFile.root.Events[:]) self.queueID = name self.numSlices = self.imageData.shape[0] #self.dataFileLock = threading.Lock() self.dataFileLock = tablesLock #self.getTaskLock = threading.Lock() self.lastTaskTime = 0
def pushImagesDS(self, startingAt=0, detThresh=.9, fitFcn='LatGaussFitFR'): #global seriesName #dataFilename = self.image.seriesName resultsFilename = genResultFileName(self.image.seriesName) while os.path.exists(resultsFilename): di, fn = os.path.split(resultsFilename) fdialog = wx.FileDialog( None, 'Analysis file already exists, please select a new filename', wildcard='H5R files|*.h5r', defaultDir=di, defaultFile=os.path.splitext(fn)[0] + '_1.h5r', style=wx.SAVE) succ = fdialog.ShowModal() if (succ == wx.ID_OK): resultsFilename = fdialog.GetPath().encode() else: raise RuntimeError('Invalid results file - not running') #self.image.seriesName = resultsFilename debugPrint('Results file = %s' % resultsFilename) self.tq.createQueue('HDFResultsTaskQueue', resultsFilename, None) debugPrint('Queue created') mdhQ = MetaDataHandler.QueueMDHandler(self.tq, resultsFilename, self.image.mdh) mdhQ.setEntry('Analysis.DetectionThreshold', detThresh) mdhQ.setEntry('Analysis.FitModule', fitFcn) mdhQ.setEntry('Analysis.DataFileID', fileID.genDataSourceID(self.image.dataSource)) debugPrint('Metadata transferred to queue') evts = self.image.dataSource.getEvents() if len(evts) > 0: self.tq.addQueueEvents(resultsFilename, evts) md = MetaDataHandler.NestedClassMDHandler(mdhQ) mn = self.image.dataSource.moduleName #if it's a buffered source, go back to underlying source if mn == 'BufferedDataSource': mn = self.image.dataSource.dataSource.moduleName for i in range(startingAt, self.image.dataSource.getNumSlices()): debugPrint('Posting task %d' % i) if 'Analysis.BGRange' in md.getEntryNames(): bgi = range( max(i + md.Analysis.BGRange[0], md.EstimatedLaserOnFrameNo), max(i + md.Analysis.BGRange[1], md.EstimatedLaserOnFrameNo)) elif 'Analysis.NumBGFrames' in md.getEntryNames(): bgi = range( max(i - md.Analysis.NumBGFrames, md.EstimatedLaserOnFrameNo), i) else: bgi = range(max(i - 10, md.EstimatedLaserOnFrameNo), i) #task = fitTask(self.queueID, taskNum, self.metaData.Analysis.DetectionThreshold, self.metaData, self.metaData.Analysis.FitModule, 'TQDataSource', bgindices =bgi, SNThreshold = True) self.tq.postTask(remFitBuf.fitTask(self.image.seriesName, i, detThresh, md, fitFcn, bgindices=bgi, SNThreshold=True, dataSourceModule=mn), queueName=resultsFilename) self.image.seriesName = resultsFilename