def pushImagesHDF(self, startingAt=0, detThresh = .9, fitFcn = 'LatGaussFitFR'):
#global seriesName
dataFilename = self.image.seriesName
resultsFilename = genResultFileName(self.image.seriesName)
if os.path.exists(resultsFilename):
di, fn = os.path.split(resultsFilename)
i = 1
stub = os.path.splitext(fn)[0]
while os.path.exists(os.path.join(di, stub + '_%d.h5r' % i)):
i += 1
fdialog = wx.FileDialog(None, 'Analysis file already exists, please select a new filename',
wildcard='H5R files|*.h5r', defaultDir=di, defaultFile=stub + '_%d.h5r' % i, style=wx.SAVE)
succ = fdialog.ShowModal()
if (succ == wx.ID_OK):
resultsFilename = fdialog.GetPath().encode()
else:
raise RuntimeError('Invalid results file - not running')
self.image.seriesName = resultsFilename
self.tq.createQueue('HDFTaskQueue', self.image.seriesName, dataFilename = dataFilename, resultsFilename=resultsFilename, startAt = 'notYet')
mdhQ = MetaDataHandler.QueueMDHandler(self.tq, self.image.seriesName, self.image.mdh)
mdhQ.setEntry('Analysis.DetectionThreshold', detThresh)
mdhQ.setEntry('Analysis.FitModule', fitFcn)
mdhQ.setEntry('Analysis.DataFileID', fileID.genDataSourceID(self.image.dataSource))
evts = self.image.dataSource.getEvents()
if len(evts) > 0:
self.tq.addQueueEvents(self.image.seriesName, evts)
self.tq.releaseTasks(self.image.seriesName, startingAt)
def pushImagesDS(self, startingAt=0, detThresh = .9, fitFcn = 'LatGaussFitFR'):
#global seriesName
#dataFilename = self.image.seriesName
resultsFilename = genResultFileName(self.image.seriesName)
while os.path.exists(resultsFilename):
di, fn = os.path.split(resultsFilename)
fdialog = wx.FileDialog(None, 'Analysis file already exists, please select a new filename',
wildcard='H5R files|*.h5r', defaultDir=di, defaultFile=os.path.splitext(fn)[0] + '_1.h5r', style=wx.SAVE)
succ = fdialog.ShowModal()
if (succ == wx.ID_OK):
resultsFilename = fdialog.GetPath().encode()
else:
raise RuntimeError('Invalid results file - not running')
#self.image.seriesName = resultsFilename
debugPrint('Results file = %s' % resultsFilename)
self.tq.createQueue('HDFResultsTaskQueue', resultsFilename, None)
debugPrint('Queue created')
mdhQ = MetaDataHandler.QueueMDHandler(self.tq, resultsFilename, self.image.mdh)
mdhQ.setEntry('Analysis.DetectionThreshold', detThresh)
mdhQ.setEntry('Analysis.FitModule', fitFcn)
mdhQ.setEntry('Analysis.DataFileID', fileID.genDataSourceID(self.image.dataSource))
debugPrint('Metadata transferred to queue')
evts = self.image.dataSource.getEvents()
if len(evts) > 0:
self.tq.addQueueEvents(resultsFilename, evts)
md = MetaDataHandler.NestedClassMDHandler(mdhQ)
mn = self.image.dataSource.moduleName
#if it's a buffered source, go back to underlying source
if mn == 'BufferedDataSource':
mn = self.image.dataSource.dataSource.moduleName
for i in range(startingAt, self.image.dataSource.getNumSlices()):
debugPrint('Posting task %d' %i)
if 'Analysis.BGRange' in md.getEntryNames():
bgi = range(max(i + md.Analysis.BGRange[0],md.EstimatedLaserOnFrameNo), max(i + md.Analysis.BGRange[1],md.EstimatedLaserOnFrameNo))
elif 'Analysis.NumBGFrames' in md.getEntryNames():
bgi = range(max(i - md.Analysis.NumBGFrames, md.EstimatedLaserOnFrameNo), i)
else:
bgi = range(max(i - 10, md.EstimatedLaserOnFrameNo), i)
#task = fitTask(self.queueID, taskNum, self.metaData.Analysis.DetectionThreshold, self.metaData, self.metaData.Analysis.FitModule, 'TQDataSource', bgindices =bgi, SNThreshold = True)
self.tq.postTask(remFitBuf.fitTask(self.image.seriesName,i, detThresh, md, fitFcn, bgindices=bgi, SNThreshold=True, dataSourceModule=mn), queueName=resultsFilename)
self.image.seriesName = resultsFilename
def __init__(self,
name,
resultsFilename,
initialTasks=[],
onEmpty=doNix,
fTaskToPop=popZero):
if resultsFilename is None:
resultsFilename = genResultFileName(name)
if os.path.exists(
resultsFilename): #bail if output file already exists
raise RuntimeError('Output file already exists: ' +
resultsFilename)
TaskQueue.__init__(self, name, initialTasks, onEmpty, fTaskToPop)
self.resultsFilename = resultsFilename
self.numClosedTasks = 0
logging.info('Creating results file')
self.h5ResultsFile = tables.openFile(self.resultsFilename, 'w')
self.prepResultsFile() # pass
#self.fileResultsLock = threading.Lock()
self.fileResultsLock = tablesLock
logging.info('Creating results metadata')
self.resultsMDH = MetaDataHandler.HDFMDHandler(self.h5ResultsFile)
self.metaData = MetaDataHandler.NestedClassMDHandler()
#self.metaData = None #MetaDataHandler.NestedClassMDHandler(self.resultsMDH)
self.metaDataStale = True
self.MDHCache = []
logging.info('Creating results events table')
with self.fileResultsLock.wlock:
self.resultsEvents = self.h5ResultsFile.createTable(
self.h5ResultsFile.root,
'Events',
SpoolEvent,
filters=tables.Filters(complevel=5, shuffle=True))
logging.info('Events table created')
self.haveResultsTable = False
self.resultsQueue = [] #Queue.Queue()
self.resultsQueueLock = threading.Lock()
self.lastResultsQueuePurge = time.time()
logging.info('Results file initialised')
def pushImagesHDF(self,
startingAt=0,
detThresh=.9,
fitFcn='LatGaussFitFR'):
#global seriesName
dataFilename = self.image.seriesName
resultsFilename = genResultFileName(self.image.seriesName)
if os.path.exists(resultsFilename):
di, fn = os.path.split(resultsFilename)
i = 1
stub = os.path.splitext(fn)[0]
while os.path.exists(os.path.join(di, stub + '_%d.h5r' % i)):
i += 1
fdialog = wx.FileDialog(
None,
'Analysis file already exists, please select a new filename',
wildcard='H5R files|*.h5r',
defaultDir=di,
defaultFile=stub + '_%d.h5r' % i,
style=wx.SAVE)
succ = fdialog.ShowModal()
if (succ == wx.ID_OK):
resultsFilename = fdialog.GetPath().encode()
else:
raise RuntimeError('Invalid results file - not running')
self.image.seriesName = resultsFilename
self.tq.createQueue('HDFTaskQueue',
self.image.seriesName,
dataFilename=dataFilename,
resultsFilename=resultsFilename,
startAt='notYet')
mdhQ = MetaDataHandler.QueueMDHandler(self.tq, self.image.seriesName,
self.image.mdh)
mdhQ.setEntry('Analysis.DetectionThreshold', detThresh)
mdhQ.setEntry('Analysis.FitModule', fitFcn)
mdhQ.setEntry('Analysis.DataFileID',
fileID.genDataSourceID(self.image.dataSource))
evts = self.image.dataSource.getEvents()
if len(evts) > 0:
self.tq.addQueueEvents(self.image.seriesName, evts)
self.tq.releaseTasks(self.image.seriesName, startingAt)
def __init__(self, name, resultsFilename, initialTasks=[], onEmpty = doNix, fTaskToPop = popZero):
if resultsFilename is None:
resultsFilename = genResultFileName(name)
if os.path.exists(resultsFilename): #bail if output file already exists
raise RuntimeError('Output file already exists: ' + resultsFilename)
TaskQueue.__init__(self, name, initialTasks, onEmpty, fTaskToPop)
self.resultsFilename = resultsFilename
self.numClosedTasks = 0
logging.info('Creating results file')
self.h5ResultsFile = tables.openFile(self.resultsFilename, 'w')
self.prepResultsFile() # pass
#self.fileResultsLock = threading.Lock()
self.fileResultsLock = tablesLock
logging.info('Creating results metadata')
self.resultsMDH = MetaDataHandler.HDFMDHandler(self.h5ResultsFile)
self.metaData = MetaDataHandler.NestedClassMDHandler()
#self.metaData = None #MetaDataHandler.NestedClassMDHandler(self.resultsMDH)
self.metaDataStale = True
self.MDHCache = []
logging.info('Creating results events table')
with self.fileResultsLock.wlock:
self.resultsEvents = self.h5ResultsFile.createTable(self.h5ResultsFile.root, 'Events', SpoolEvent,filters=tables.Filters(complevel=5, shuffle=True))
logging.info('Events table created')
self.haveResultsTable = False
self.resultsQueue = []#Queue.Queue()
self.resultsQueueLock = threading.Lock()
self.lastResultsQueuePurge = time.time()
logging.info('Results file initialised')
def __init__(self, name, dataFilename = None, resultsFilename=None, onEmpty = doNix, fTaskToPop = popZero, startAt = 'guestimate', frameSize=(-1,-1), complevel=6, complib='zlib'):
if dataFilename is None:
self.dataFilename = genDataFilename(name)
else:
self.dataFilename = dataFilename
if resultsFilename is None:
resultsFilename = genResultFileName(self.dataFilename)
else:
resultsFilename = resultsFilename
ffn = getFullFilename(self.dataFilename)
self.acceptNewTasks = False
self.releaseNewTasks = False
self.postTaskBuffer = []
initialTasks = []
if os.path.exists(ffn): #file already exists - read from it
self.h5DataFile = tables.openFile(ffn, 'r')
#self.metaData = MetaData.genMetaDataFromHDF(self.h5DataFile)
self.dataMDH = MetaDataHandler.NestedClassMDHandler(MetaDataHandler.HDFMDHandler(self.h5DataFile))
#self.dataMDH.mergeEntriesFrom(MetaData.TIRFDefault)
self.imageData = self.h5DataFile.root.ImageData
if startAt == 'guestimate': #calculate a suitable starting value
tLon = self.dataMDH.EstimatedLaserOnFrameNo
if tLon == 0:
startAt = 0
else:
startAt = tLon + 10
if startAt == 'notYet':
initialTasks = []
else:
initialTasks = list(range(startAt, self.h5DataFile.root.ImageData.shape[0]))
self.imNum = len(self.imageData)
self.dataRW = False
else: #make ourselves a new file
self.h5DataFile = tables.openFile(ffn, 'w')
filt = tables.Filters(complevel, complib, shuffle=True)
self.imageData = self.h5DataFile.createEArray(self.h5DataFile.root,
'ImageData',
tables.UInt16Atom(),
(0,)+tuple(frameSize),
filters=filt,
chunkshape=(1,)+tuple(frameSize))
self.events = self.h5DataFile.createTable(self.h5DataFile.root, 'Events', SpoolEvent,filters=filt)
self.imNum=0
self.acceptNewTasks = True
self.dataMDH = MetaDataHandler.HDFMDHandler(self.h5DataFile)
self.dataMDH.mergeEntriesFrom(MetaData.TIRFDefault)
self.dataRW = True
HDFResultsTaskQueue.__init__(self,
name,
resultsFilename,
initialTasks, onEmpty, fTaskToPop)
self.resultsMDH.copyEntriesFrom(self.dataMDH)
self.metaData.copyEntriesFrom(self.resultsMDH)
#copy events to results file
if len (self.h5DataFile.root.Events) > 0:
self.resultsEvents.append(self.h5DataFile.root.Events[:])
self.queueID = name
self.numSlices = self.imageData.shape[0]
#self.dataFileLock = threading.Lock()
self.dataFileLock = tablesLock
#self.getTaskLock = threading.Lock()
self.lastTaskTime = 0
def __init__(self,
name,
dataFilename=None,
resultsFilename=None,
onEmpty=doNix,
fTaskToPop=popZero,
startAt='guestimate',
frameSize=(-1, -1),
complevel=6,
complib='zlib'):
if dataFilename is None:
self.dataFilename = genDataFilename(name)
else:
self.dataFilename = dataFilename
if resultsFilename is None:
resultsFilename = genResultFileName(self.dataFilename)
else:
resultsFilename = resultsFilename
ffn = getFullFilename(self.dataFilename)
self.acceptNewTasks = False
self.releaseNewTasks = False
self.postTaskBuffer = []
initialTasks = []
if os.path.exists(ffn): #file already exists - read from it
self.h5DataFile = tables.openFile(ffn, 'r')
#self.metaData = MetaData.genMetaDataFromHDF(self.h5DataFile)
self.dataMDH = MetaDataHandler.NestedClassMDHandler(
MetaDataHandler.HDFMDHandler(self.h5DataFile))
#self.dataMDH.mergeEntriesFrom(MetaData.TIRFDefault)
self.imageData = self.h5DataFile.root.ImageData
if startAt == 'guestimate': #calculate a suitable starting value
tLon = self.dataMDH.EstimatedLaserOnFrameNo
if tLon == 0:
startAt = 0
else:
startAt = tLon + 10
if startAt == 'notYet':
initialTasks = []
else:
initialTasks = list(
range(startAt, self.h5DataFile.root.ImageData.shape[0]))
self.imNum = len(self.imageData)
self.dataRW = False
else: #make ourselves a new file
self.h5DataFile = tables.openFile(ffn, 'w')
filt = tables.Filters(complevel, complib, shuffle=True)
self.imageData = self.h5DataFile.createEArray(
self.h5DataFile.root,
'ImageData',
tables.UInt16Atom(), (0, ) + tuple(frameSize),
filters=filt,
chunkshape=(1, ) + tuple(frameSize))
self.events = self.h5DataFile.createTable(self.h5DataFile.root,
'Events',
SpoolEvent,
filters=filt)
self.imNum = 0
self.acceptNewTasks = True
self.dataMDH = MetaDataHandler.HDFMDHandler(self.h5DataFile)
self.dataMDH.mergeEntriesFrom(MetaData.TIRFDefault)
self.dataRW = True
HDFResultsTaskQueue.__init__(self, name, resultsFilename, initialTasks,
onEmpty, fTaskToPop)
self.resultsMDH.copyEntriesFrom(self.dataMDH)
self.metaData.copyEntriesFrom(self.resultsMDH)
#copy events to results file
if len(self.h5DataFile.root.Events) > 0:
self.resultsEvents.append(self.h5DataFile.root.Events[:])
self.queueID = name
self.numSlices = self.imageData.shape[0]
#self.dataFileLock = threading.Lock()
self.dataFileLock = tablesLock
#self.getTaskLock = threading.Lock()
self.lastTaskTime = 0
def pushImagesDS(self, startingAt=0, detThresh=.9, fitFcn='LatGaussFitFR'):
#global seriesName
#dataFilename = self.image.seriesName
resultsFilename = genResultFileName(self.image.seriesName)
while os.path.exists(resultsFilename):
di, fn = os.path.split(resultsFilename)
fdialog = wx.FileDialog(
None,
'Analysis file already exists, please select a new filename',
wildcard='H5R files|*.h5r',
defaultDir=di,
defaultFile=os.path.splitext(fn)[0] + '_1.h5r',
style=wx.SAVE)
succ = fdialog.ShowModal()
if (succ == wx.ID_OK):
resultsFilename = fdialog.GetPath().encode()
else:
raise RuntimeError('Invalid results file - not running')
#self.image.seriesName = resultsFilename
debugPrint('Results file = %s' % resultsFilename)
self.tq.createQueue('HDFResultsTaskQueue', resultsFilename, None)
debugPrint('Queue created')
mdhQ = MetaDataHandler.QueueMDHandler(self.tq, resultsFilename,
self.image.mdh)
mdhQ.setEntry('Analysis.DetectionThreshold', detThresh)
mdhQ.setEntry('Analysis.FitModule', fitFcn)
mdhQ.setEntry('Analysis.DataFileID',
fileID.genDataSourceID(self.image.dataSource))
debugPrint('Metadata transferred to queue')
evts = self.image.dataSource.getEvents()
if len(evts) > 0:
self.tq.addQueueEvents(resultsFilename, evts)
md = MetaDataHandler.NestedClassMDHandler(mdhQ)
mn = self.image.dataSource.moduleName
#if it's a buffered source, go back to underlying source
if mn == 'BufferedDataSource':
mn = self.image.dataSource.dataSource.moduleName
for i in range(startingAt, self.image.dataSource.getNumSlices()):
debugPrint('Posting task %d' % i)
if 'Analysis.BGRange' in md.getEntryNames():
bgi = range(
max(i + md.Analysis.BGRange[0],
md.EstimatedLaserOnFrameNo),
max(i + md.Analysis.BGRange[1],
md.EstimatedLaserOnFrameNo))
elif 'Analysis.NumBGFrames' in md.getEntryNames():
bgi = range(
max(i - md.Analysis.NumBGFrames,
md.EstimatedLaserOnFrameNo), i)
else:
bgi = range(max(i - 10, md.EstimatedLaserOnFrameNo), i)
#task = fitTask(self.queueID, taskNum, self.metaData.Analysis.DetectionThreshold, self.metaData, self.metaData.Analysis.FitModule, 'TQDataSource', bgindices =bgi, SNThreshold = True)
self.tq.postTask(remFitBuf.fitTask(self.image.seriesName,
i,
detThresh,
md,
fitFcn,
bgindices=bgi,
SNThreshold=True,
dataSourceModule=mn),
queueName=resultsFilename)
self.image.seriesName = resultsFilename