def _checkNewInput(self):
""" Check if there are new particles to be processed and add
the necessary steps."""
particlesFile = self.inputParticles.get().getFileName()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
mTime = datetime.fromtimestamp(getmtime(particlesFile))
self.debug('Last check: %s, modification: %s' %
(prettyTime(self.lastCheck), prettyTime(mTime)))
# If the input have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'listOfParticles'):
return None
self.lastCheck = now
outputStep = self._getFirstJoinStep()
# Open input and close it as soon as possible
self._loadInputList()
fDeps = []
fDeps = self._insertStepsForParticles(fDeps)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def _checkNewInput(self):
# Check if there are new micrographs to process from the input set
localFile = self.getInputMicrographs().getFileName()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
mTime = datetime.fromtimestamp(os.path.getmtime(localFile))
self.debug('Last check: %s, modification: %s'
% (pwutils.prettyTime(self.lastCheck),
pwutils.prettyTime(mTime)))
# If the input micrographs.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'listOfMics'):
return None
self.lastCheck = now
# Open input micrographs.sqlite and close it as soon as possible
micDict, self.streamClosed = self._loadInputList()
newMics = micDict.values()
outputStep = self._getFirstJoinStep()
if newMics:
fDeps = self._insertNewMicsSteps(newMics)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def _checkNewInput(self):
""" Check if there are new ctf to be processed and add the necessary
steps."""
ctfFile = self.inputCTF.get().getFileName()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
mTime = datetime.fromtimestamp(getmtime(ctfFile))
self.debug(
'Last check: %s, modification: %s' %
(pwutils.prettyTime(self.lastCheck), pwutils.prettyTime(mTime)))
# Open input ctfs.sqlite and close it as soon as possible
self._loadInputList()
# If the input ctfs.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'listOfCtf'):
return None
self.lastCheck = now
newCtf = any(ctf.getObjId() not in self.insertedDict
for ctf in self.listOfCtf)
outputStep = self._getFirstJoinStep()
if newCtf:
fDeps = self._insertNewSelectionSteps(self.insertedDict,
self.listOfCtf)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def _checkNewInput(self):
# Check if there are new micrographs to process from the input set
localFile = self.getInputMicrographs().getFileName()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
mTime = datetime.fromtimestamp(getmtime(localFile))
self.debug(
'Last check: %s, modification: %s' %
(pwutils.prettyTime(self.lastCheck), pwutils.prettyTime(mTime)))
# If the input micrographs.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'listOfMics'):
return None
self.lastCheck = now
# Open input micrographs.sqlite and close it as soon as possible
micDict, self.streamClosed = self._loadInputList()
newMics = micDict.values()
outputStep = self._getFirstJoinStep()
if newMics:
fDeps = self._insertNewMicsSteps(newMics)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def _checkNewInput(self):
# Check if there are new movies to process from the input set
localFile = self.inputMovies.get().getFileName()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
mTime = datetime.fromtimestamp(os.path.getmtime(localFile))
self.debug(
'Last check: %s, modification: %s' %
(pwutils.prettyTime(self.lastCheck), pwutils.prettyTime(mTime)))
# If the input movies.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'listOfMovies'):
return None
self.lastCheck = now
# Open input movies.sqlite and close it as soon as possible
self._loadInputList()
newMovies = any(m.getObjId() not in self.insertedDict
for m in self.listOfMovies)
outputStep = self._getFirstJoinStep()
if newMovies:
fDeps = self._insertNewMoviesSteps(self.insertedDict,
self.listOfMovies)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def _checkNewInput(self):
""" Check if there are new ctf to be processed and add the necessary
steps."""
ctfFile = self.inputCTF.get().getFileName()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
mTime = datetime.fromtimestamp(getmtime(ctfFile))
self.debug('Last check: %s, modification: %s'
% (pwutils.prettyTime(self.lastCheck),
pwutils.prettyTime(mTime)))
# Open input ctfs.sqlite and close it as soon as possible
self._loadInputList()
# If the input ctfs.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'listOfCtf'):
return None
self.lastCheck = now
newCtf = any(ctf.getObjId() not in
self.insertedDict for ctf in self.listOfCtf)
outputStep = self._getFirstJoinStep()
if newCtf:
fDeps = self._insertNewSelectionSteps(self.insertedDict,
self.listOfCtf)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def _checkNewInput(self):
self.debug(">>> _checkNewInput ")
def _modificationTime():
""" Check the last modification time of any of the three possible
input files. """
items = [self.inputCoordinates.get()]
if self._micsOther():
items.append(self.inputMicrographs.get())
else:
items.append(self.inputCoordinates.get().getMicrographs())
if self._useCTF():
items.append(self.ctfRelations.get())
def _mTime(fn):
return datetime.fromtimestamp(os.path.getmtime(fn))
return max([_mTime(i.getFileName()) for i in items])
mTime = _modificationTime()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
self.debug('Last check: %s, modification: %s'
% (pwutils.prettyTime(self.lastCheck),
pwutils.prettyTime(mTime)))
# If the input micrographs.sqlite have not changed since our last check,
# it does not make sense to check for new input data, but we must
# check if sets are closed.
self.debug("self.lastCheck > mTime %s , hasattr(self, 'micDict') %s"
% (self.lastCheck > mTime, hasattr(self, 'micDict')))
if self.lastCheck > mTime and hasattr(self, 'micDict'):
return None
# Open input micrographs.sqlite and close it as soon as possible
newMics = self._loadInputList()
self.lastCheck = now
outputStep = self._getFirstJoinStep()
if newMics:
fDeps = self._insertNewMicsSteps(newMics.values())
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def _checkNewInput(self):
self.debug(">>> _checkNewInput ")
def _modificationTime():
""" Check the last modification time of any of the three possible
input files. """
items = [self.inputCoordinates.get()]
if self._micsOther():
items.append(self.inputMicrographs.get())
else:
items.append(self.inputCoordinates.get().getMicrographs())
if self._useCTF():
items.append(self.ctfRelations.get())
def _mTime(fn):
return datetime.fromtimestamp(os.path.getmtime(fn))
return max([_mTime(i.getFileName()) for i in items])
mTime = _modificationTime()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
self.debug('Last check: %s, modification: %s'
% (pwutils.prettyTime(self.lastCheck),
pwutils.prettyTime(mTime)))
# If the input micrographs.sqlite have not changed since our last check,
# it does not make sense to check for new input data, but we must
# check if sets are closed.
self.debug("self.lastCheck > mTime %s , hasattr(self, 'micDict') %s"
% (self.lastCheck > mTime, hasattr(self, 'micDict')))
if self.lastCheck > mTime and hasattr(self, 'micDict'):
return None
# Open input micrographs.sqlite and close it as soon as possible
newMics = self._loadInputList()
self.lastCheck = now
outputStep = self._getFirstJoinStep()
if newMics:
fDeps = self._insertNewMicsSteps(newMics.values())
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def fileModified(self, fileName, fileTimeout):
""" Check if the fileName modification time is less
than a given timeout.
Params:
fileName: input filename that will be checked.
fileTimeout: timeout """
self.debug('Checking file: %s' % fileName)
mTime = datetime.fromtimestamp(os.path.getmtime(fileName))
delta = datetime.now() - mTime
self.debug(' Modification time: %s' % pwutils.prettyTime(mTime))
self.debug(' Delta: %s' % pwutils.prettyDelta(delta))
return delta < fileTimeout
def fileModified(self, fileName, fileTimeout):
""" Check if the fileName modification time is less
than a given timeout.
Params:
fileName: input filename that will be checked.
fileTimeout: timeout """
self.debug('Checking file: %s' % fileName)
mTime = datetime.fromtimestamp(os.path.getmtime(fileName))
delta = datetime.now() - mTime
self.debug(' Modification time: %s' % pwutils.prettyTime(mTime))
self.debug(' Delta: %s' % pwutils.prettyDelta(delta))
return delta < fileTimeout
def _checkNewInput(self):
""" Check if there are new particles to be processed and add
the necessary steps."""
particlesFile = self.inputParticles.get().getFileName()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
mTime = datetime.fromtimestamp(getmtime(particlesFile))
self.debug('Last check: %s, modification: %s' %
(prettyTime(self.lastCheck), prettyTime(mTime)))
# If the input have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'listOfParticles'):
return None
self.lastCheck = now
outputStep = self._getFirstJoinStep()
# Open input and close it as soon as possible
self._loadInputList()
if len(self.listOfParticles) == 0:
return None
deps = []
numBlk, rem = divmod(float(len(self.listOfParticles)),
float(self.blockSize.get()))
numBlk = int(numBlk)
rem = int(rem)
if rem > 0:
numBlk += 1
for i in range(numBlk):
deps += self._insertStepsForParticles(False, True)
if outputStep is not None:
outputStep.addPrerequisites(*deps)
self.updateSteps()
def runTests(self, tests):
self.testCount = 0
if self.log:
self.testsDir = join(os.environ['SCIPION_USER_DATA'], 'Tests',
self.log)
pwutils.cleanPath(self.testsDir)
pwutils.makePath(self.testsDir)
self.testLog = join(self.testsDir, 'tests.html')
self.testTimer = pwutils.Timer()
self.testTimer.tic()
self.headerPrefix = '<h3>Test results (%s) Duration: ' % pwutils.prettyTime(
)
f = open(self.testLog, 'w')
f.write("""<!DOCTYPE html>
<html>
<body>
""")
f.write(self.headerPrefix + '</h3>')
f.write("""
<table style="width:100%" border="1">
<tr>
<th>#</th>
<th>Command</th>
<th>Time</th>
<th>Result</th>
<th>Log file</th>
</tr>
<!-- LAST_ROW -->
</table>
</body>
</html>""")
f.close()
self._visitTests(tests, self._runNewItem)
if self.log:
print "\n\nOpen results in your browser: \nfile:///%s" % self.testLog
def runTests(self, tests):
self.testCount = 0
if self.log:
self.testsDir = join(os.environ['SCIPION_USER_DATA'], 'Tests', self.log)
pwutils.cleanPath(self.testsDir)
pwutils.makePath(self.testsDir)
self.testLog = join(self.testsDir, 'tests.html')
self.testTimer = pwutils.Timer()
self.testTimer.tic()
self.headerPrefix = '<h3>Test results (%s) Duration: ' % pwutils.prettyTime()
f = open(self.testLog, 'w')
f.write("""<!DOCTYPE html>
<html>
<body>
""")
f.write(self.headerPrefix + '</h3>')
f.write("""
<table style="width:100%" border="1">
<tr>
<th>#</th>
<th>Command</th>
<th>Time</th>
<th>Result</th>
<th>Log file</th>
</tr>
<!-- LAST_ROW -->
</table>
</body>
</html>""")
f.close()
self._visitTests(tests, self._runNewItem)
if self.log:
print "\n\nOpen results in your browser: \nfile:///%s" % self.testLog
def generate(self, finished):
reportTemplate = self.getHTMLReportText()
if not reportTemplate:
raise Exception("HTML template file '%s' not found. "
% self.template)
project = self.protocol.getProject()
projName = project.getShortName()
acquisitionLines = ''
self.provider.refreshObjects()
for item in self.provider.acquisition:
if not acquisitionLines == '':
acquisitionLines += ','
acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item
runLines = ''
wasProtocol = None
for obj in self.provider.getObjects():
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
if runLines != '': runLines += ']},'
runLines += '{protocolName: "%s", output:[' % obj.name
else:
if not wasProtocol: runLines += ','
runLines += '{name: "%s", size:"%s"}' % (obj.output,
obj.outSize)
wasProtocol = isProtocol
# End the runLines JSON object
runLines += ']}'
# Ctf monitor chart data
data = {} if self.ctfMonitor is None else self.ctfMonitor.getData()
if data:
numMicsDone = len(self.thumbPaths[PSD_THUMBS])
numMics = len(data[PSD_PATH])
numMicsToDo = numMics - numMicsDone
self.getThumbPaths(ctfData=data, thumbsDone=numMicsDone, micIdSet=data['idValues'])
if len(data['defocusU']) < 100:
data['defocusCoverage'] = self.processDefocusValues(data['defocusU'])
else:
data['defocusCoverage'] = self.processDefocusValues(data['defocusU'][:-50])
data['defocusCoverageLast50'] = self.processDefocusValues(data['defocusU'][-50:])
data['resolutionHistogram'] = self.getResolutionHistogram(data['resolution'])
else:
# Thumbnails for Micrograph Table
numMicsDone = len(self.thumbPaths[MIC_THUMBS])
self.getThumbPaths(thumbsDone=numMicsDone)
numMics = len(self.thumbPaths[MIC_PATH])
numMicsToDo = numMics - numMicsDone
if numMicsToDo <= 10:
# we have few new images, eg streaming mode, generate thumbnails now
self.generateReportImages(firstThumbIndex=numMicsDone)
else:
# we have many images, generate thumbs in a separate process
process = multiprocessing.Process(target=self.generateReportImages,
args=(numMicsDone,))
process.start()
# send over only thumbnails of the mics that have been fully processed
self.thumbsReady = self.checkNewThumbsReady()
thumbsLoading = numMics - self.thumbsReady
for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]:
if k in self.thumbPaths:
data[k] = self.thumbPaths[k][:self.thumbsReady] + ['']*thumbsLoading
data[MIC_ID] = self.thumbPaths[MIC_ID]
reportFinished = self.thumbsReady == numMics
ctfData = json.dumps(data)
# Movie gain monitor chart data
data = [] if self.movieGainMonitor is None else self.movieGainMonitor.getData()
movieGainData = json.dumps(data)
# system monitor chart data
data = self.sysMonitor.getData()
systemData = json.dumps(data)
tnow = datetime.now()
args = {'projectName': projName,
'startTime': pwutils.dateStr(project.getCreationTime(), secs=True),
'dateStr': pwutils.prettyTime(dt=tnow, secs=True),
'projectDuration': pwutils.prettyDelta(tnow-project.getCreationTime()),
'projectStatus': "FINISHED" if finished else "RUNNING",
'scipionVersion': os.environ['SCIPION_VERSION'],
'acquisitionLines': acquisitionLines,
'runLines': runLines,
'ctfData': ctfData,
'movieGainData': movieGainData,
'systemData': systemData,
'refresh': self.refreshSecs
}
self.info("Writing report html to: %s" % abspath(self.reportPath))
pwutils.cleanPath(self.reportPath)
reportFile = open(self.reportPath, 'w')
reportTemplate = reportTemplate % args
reportFile.write(reportTemplate.encode('utf-8'))
reportFile.close()
if self.publishCmd:
self.info("Publishing the report:")
cmd = self.publishCmd % {'REPORT_FOLDER': self.reportDir}
self.info(cmd)
os.system(cmd)
return reportFinished
def _checkNewInput(self):
if self.calculateConsensus:
# Check if there are new ctf to process from the input set
self.lastCheck = getattr(self, 'lastCheck', datetime.now())
mTime = max(datetime.fromtimestamp(os.path.getmtime(self.ctfFn1)),
datetime.fromtimestamp(os.path.getmtime(self.ctfFn2)))
self.debug('Last check: %s, modification: %s'
% (pwutils.prettyTime(self.lastCheck),
pwutils.prettyTime(mTime)))
# If the input movies.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and (hasattr(self, 'outputCTF') or hasattr(self, "outputCTFDiscarded")):
return None
ctfsSet1 = self._loadInputCtfSet(self.ctfFn1)
ctfsSet2 = self._loadInputCtfSet(self.ctfFn2)
ctfDict1 = {ctf.getObjId(): ctf.clone() for ctf
in ctfsSet1.iterItems()}
ctfDict2 = {ctf.getObjId(): ctf.clone() for ctf
in ctfsSet2.iterItems()}
newIds1 = [idCTF1 for idCTF1 in ctfDict1.keys() if idCTF1 not in self.insertedDict]
self.allCtf1.update(ctfDict1)
newIds2 = [idCTF2 for idCTF2 in ctfDict2.keys() if idCTF2 not in self.insertedDict]
self.allCtf2.update(ctfDict2)
self.lastCheck = datetime.now()
self.isStreamClosed = ctfsSet1.isStreamClosed() and \
ctfsSet2.isStreamClosed()
ctfsSet1.close()
ctfsSet2.close()
outputStep = self._getFirstJoinStep()
if len(set(self.allCtf1)) > len(set(self.insertedDict)) and \
len(set(self.allCtf2)) > len(set(self.insertedDict)):
fDeps = self._insertNewCtfsSteps(newIds1, newIds2,
self.insertedDict)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
else:
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
mTime = datetime.fromtimestamp(os.path.getmtime(self.ctfFn1))
self.debug('Last check: %s, modification: %s'
% (pwutils.prettyTime(self.lastCheck),
pwutils.prettyTime(mTime)))
# If the input ctfs.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and (hasattr(self, 'outputCTF') or hasattr(self, "outputCTFDiscarded")):
return None
# Open input ctfs.sqlite and close it as soon as possible
ctfSet = self._loadInputCtfSet(self.ctfFn1)
ctfDict = {ctf.getObjId(): ctf.clone() for ctf
in ctfSet.iterItems()}
newIds = [idCTF for idCTF in ctfDict.keys() if idCTF not in self.insertedDict]
self.allCtf1.update(ctfDict)
self.isStreamClosed = ctfSet.isStreamClosed()
ctfSet.close()
self.lastCheck = now
newCtf = any(ctf.getObjId() not in
self.insertedDict for ctf in self.allCtf1.values())
outputStep = self._getFirstJoinStep()
if newCtf:
fDeps = self._insertNewSelectionSteps(self.insertedDict,
newIds)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def generate(self, finished):
reportTemplate = self.getHTMLReportText()
if not reportTemplate:
raise Exception("HTML template file '%s' not found. " %
self.template)
project = self.protocol.getProject()
projName = project.getShortName()
acquisitionLines = ''
self.provider.refreshObjects()
for item in self.provider.acquisition:
if not acquisitionLines == '':
acquisitionLines += ','
acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item
runLines = ''
wasProtocol = None
for obj in self.provider.getObjects():
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
if runLines != '': runLines += ']},'
runLines += '{protocolName: "%s", output:[' % obj.name
else:
if not wasProtocol: runLines += ','
runLines += '{name: "%s", size:"%s"}' % (obj.output,
obj.outSize)
wasProtocol = isProtocol
# End the runLines JSON object
runLines += ']}'
# Ctf monitor chart data
data = {} if self.ctfMonitor is None else self.ctfMonitor.getData()
if data:
numMicsDone = len(self.thumbPaths[PSD_THUMBS])
numMics = len(data[PSD_PATH])
numMicsToDo = numMics - numMicsDone
self.getThumbPaths(ctfData=data,
thumbsDone=numMicsDone,
micIdSet=data['idValues'])
if len(data['defocusU']) < 100:
data['defocusCoverage'] = self.processDefocusValues(
data['defocusU'])
else:
data['defocusCoverage'] = self.processDefocusValues(
data['defocusU'][:-50])
data['defocusCoverageLast50'] = self.processDefocusValues(
data['defocusU'][-50:])
data['resolutionHistogram'] = self.getResolutionHistogram(
data['resolution'])
else:
# Thumbnails for Micrograph Table
numMicsDone = len(self.thumbPaths[MIC_THUMBS])
self.getThumbPaths(thumbsDone=numMicsDone)
numMics = len(self.thumbPaths[MIC_PATH])
numMicsToDo = numMics - numMicsDone
if numMicsToDo <= 10:
# we have few new images, eg streaming mode, generate thumbnails now
self.generateReportImages(firstThumbIndex=numMicsDone)
else:
# we have many images, generate thumbs in a separate process
process = multiprocessing.Process(target=self.generateReportImages,
args=(numMicsDone, ))
process.start()
# send over only thumbnails of the mics that have been fully processed
self.thumbsReady = self.checkNewThumbsReady()
thumbsLoading = numMics - self.thumbsReady
for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]:
if k in self.thumbPaths:
data[k] = self.thumbPaths[k][:self.thumbsReady] + [
''
] * thumbsLoading
data[MIC_ID] = self.thumbPaths[MIC_ID]
reportFinished = self.thumbsReady == numMics
ctfData = json.dumps(data)
# Movie gain monitor chart data
data = [] if self.movieGainMonitor is None else self.movieGainMonitor.getData(
)
movieGainData = json.dumps(data)
# system monitor chart data
data = self.sysMonitor.getData()
systemData = json.dumps(data)
tnow = datetime.now()
args = {
'projectName':
projName,
'startTime':
pwutils.dateStr(project.getCreationTime(), secs=True),
'dateStr':
pwutils.prettyTime(dt=tnow, secs=True),
'projectDuration':
pwutils.prettyDelta(tnow - project.getCreationTime()),
'projectStatus':
"FINISHED" if finished else "RUNNING",
'scipionVersion':
os.environ['SCIPION_VERSION'],
'acquisitionLines':
acquisitionLines,
'runLines':
runLines,
'ctfData':
ctfData,
'movieGainData':
movieGainData,
'systemData':
systemData,
'refresh':
self.refreshSecs
}
self.info("Writing report html to: %s" % abspath(self.reportPath))
pwutils.cleanPath(self.reportPath)
reportFile = open(self.reportPath, 'w')
reportTemplate = reportTemplate % args
reportFile.write(reportTemplate.encode('utf-8'))
reportFile.close()
if self.publishCmd:
self.info("Publishing the report:")
cmd = self.publishCmd % {'REPORT_FOLDER': self.reportDir}
self.info(cmd)
os.system(cmd)
return reportFinished
def __init__(self, parent, windows, **kwargs):
tk.Frame.__init__(self, parent, bg='white', **kwargs)
self.windows = windows
self.manager = windows.manager
self.root = windows.root
self.vars = {}
self.checkvars = []
self.microscope = None
self.configDict = self.windows.config
# Regular expression to validate username and sample name
self.re = re.compile('\A[a-zA-Z0-9][a-zA-Z0-9_-]+[a-zA-Z0-9]\Z')
# tkFont.Font(size=12, family='verdana', weight='bold')
bigSize = pwgui.cfgFontSize + 2
smallSize = pwgui.cfgFontSize - 2
fontName = pwgui.cfgFontName
self.bigFont = tkFont.Font(size=bigSize, family=fontName)
self.bigFontBold = tkFont.Font(size=bigSize,
family=fontName,
weight='bold')
self.projDateFont = tkFont.Font(size=smallSize, family=fontName)
self.projDelFont = tkFont.Font(size=smallSize,
family=fontName,
weight='bold')
self.manager = Manager()
# Header section
headerFrame = tk.Frame(self, bg='white')
headerFrame.grid(row=0, column=0, sticky='new')
headerText = "Create New Session"
headerText += " %s" % pwutils.prettyTime(dateFormat='%Y-%m-%d')
label = tk.Label(headerFrame,
text=headerText,
font=self.bigFontBold,
borderwidth=0,
anchor='nw',
bg='white',
fg=pwgui.Color.DARK_GREY_COLOR)
label.grid(row=0, column=0, sticky='nw', padx=(20, 5), pady=10)
# Body section
bodyFrame = tk.Frame(self, bg='white')
bodyFrame.grid(row=1, column=0, sticky='news')
self._fillContent(bodyFrame)
# Add the create project button
btnFrame = tk.Frame(self, bg='white')
btn = HotButton(btnFrame,
text="Create New Session",
activebackground="dark grey",
activeforeground='black',
font=self.bigFontBold,
command=self._onAction)
btn.grid(row=0, column=1, sticky='ne', padx=10, pady=10)
# Add the Cancel project button
btn = Button(btnFrame,
Message.LABEL_BUTTON_CANCEL,
Icon.ACTION_CLOSE,
font=self.bigFontBold,
command=self.windows.close)
btn.grid(row=0, column=0, sticky='ne', padx=10, pady=10)
btnFrame.grid(row=2, column=0, sticky='sew')
btnFrame.columnconfigure(0, weight=1)
self.columnconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
def _getProjectName(self):
usr = self._getUserName()
sam = self._getSampleName()
return '%s_%s_%s' % (pwutils.prettyTime(dateFormat='%Y%m%d'), usr, sam)
def _checkNewInput(self):
if self.calculateConsensus:
# Check if there are new ctf to process from the input set
ctfsFile1 = self.inputCTF.get().getFileName()
ctfsFile2 = self.inputCTF2.get().getFileName()
self.lastCheck = getattr(self, 'lastCheck', datetime.now())
mTime = max(datetime.fromtimestamp(os.path.getmtime(ctfsFile1)),
datetime.fromtimestamp(os.path.getmtime(ctfsFile2)))
# If the input movies.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'SetOfCtf1'):
return None
ctfsSet1 = SetOfCTF(filename=ctfsFile1)
ctfsSet2 = SetOfCTF(filename=ctfsFile2)
ctfsSet1.loadAllProperties()
ctfsSet2.loadAllProperties()
if len(self.allCtf1) > 0:
newCtf1 = [ctf.clone() for ctf in
ctfsSet1.iterItems(orderBy='creation',
where='creation>"' + str(
self.checkCtf1) + '"')]
else:
newCtf1 = [ctf.clone() for ctf in ctfsSet1]
self.allCtf1 = self.allCtf1 + newCtf1
if len(newCtf1) > 0:
for ctf in ctfsSet1.iterItems(orderBy='creation',
direction='DESC'):
self.checkCtf1 = ctf.getObjCreation()
break
if len(self.allCtf2) > 0:
newCtf2 = [ctf.clone() for ctf in
ctfsSet2.iterItems(orderBy='creation',
where='creation>"' + str(
self.checkCtf2) + '"')]
else:
newCtf2 = [ctf.clone() for ctf in ctfsSet2]
self.allCtf2 = self.allCtf2 + newCtf2
if len(newCtf2) > 0:
for ctf in ctfsSet2.iterItems(orderBy='creation',
direction='DESC'):
self.checkCtf2 = ctf.getObjCreation()
break
self.lastCheck = datetime.now()
self.isStreamClosed = ctfsSet1.isStreamClosed() and \
ctfsSet2.isStreamClosed()
ctfsSet1.close()
ctfsSet2.close()
outputStep = self._getFirstJoinStep()
if len(set(self.allCtf1)) > len(set(self.processedDict)) and \
len(set(self.allCtf2)) > len(set(self.processedDict)):
fDeps = self._insertNewCtfsSteps(ctfsSet1, ctfsSet2,
self.insertedDict)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
else:
ctfFile = self.inputCTF.get().getFileName()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
mTime = datetime.fromtimestamp(os.path.getmtime(ctfFile))
self.debug('Last check: %s, modification: %s'
% (pwutils.prettyTime(self.lastCheck),
pwutils.prettyTime(mTime)))
# Open input ctfs.sqlite and close it as soon as possible
ctfSet = self._loadInputCtfSet()
self.isStreamClosed = ctfSet.isStreamClosed()
self.allCtf1 = [m.clone() for m in ctfSet]
ctfSet.close()
# If the input ctfs.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'allCtf1'):
return None
self.lastCheck = now
newCtf = any(ctf.getObjId() not in
self.insertedDict for ctf in self.allCtf1)
outputStep = self._getFirstJoinStep()
if newCtf:
fDeps = self._insertNewSelectionSteps(self.insertedDict,
self.allCtf1)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def generate(self, finished):
project = self.protocol.getProject()
# Project Properties Section
# Do not delete this variables. We are using them
# in an eval command
self.projectName = project.getShortName()
startTime = pwutils.dateStr(project.getCreationTime(), secs=True),
tnow = datetime.now()
_now = project.getCreationTime()
dateStr = pwutils.prettyTime(dt=tnow, secs=True),
projectDuration = pwutils.prettyDelta(tnow -
project.getCreationTime()),
projectStatus = "FINISHED" if finished else "RUNNING",
scipionVersion = os.environ['SCIPION_VERSION'],
# create structure with information related with the properties section
# the primmary key is time, projectionName, section
# if we insert two points with these three values idential
# the second updates the first
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'properties'
pointsDict['tags'] = tags
fields = {}
firstTime = self.confParser.getint("project", "properties")
fieldKeys = {'dateStr': 4, 'projectDuration': 3, 'projectStatus': 2}
fieldNames = {
'dateStr': "<b>Last Update</b>",
'projectDuration': '<b>Duration</b>',
'projectStatus': '<b>Status</b>'
}
if firstTime:
fieldKeys['startTime'] = 5
fieldKeys['scipionVersion'] = 1
fieldNames['startTime'] = '<b>Start Time</b>'
fieldNames['scipionVersion'] = '<b>Scipion Version</b>'
for metric, delta in fieldKeys.items():
localNow = _now + timedelta(seconds=delta)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
fields['metric'] = fieldNames[metric]
# str is need because all values must have the same type
fields['valueStr'] = str(eval(metric)[0])
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
self.confParser.set("project", "properties", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# acquisition section
self.provider.refreshObjects()
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'acquisition'
pointsDict['tags'] = tags
fields = {}
firstTime = self.confParser.getint("project", "acquisition")
if firstTime:
delta = 0
for metricName, value in self.provider.acquisition:
localNow = _now + timedelta(seconds=delta)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
fields['metric'] = "<b>" + metricName + "</b>"
fields['valueNum'] = float(value)
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
delta = delta - 1
# update first time only if some date has been read.
# do not place this upside the loop
self.confParser.set("project", "acquisition", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# send summary section
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'summary'
pointsDict['tags'] = tags
localNow = _now
for obj in self.provider.getObjects():
fields = {}
localNow = localNow + timedelta(seconds=-1)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
fields['protocolName'] = '<b>' + obj.name + '</b>'
fields['output'] = ""
fields['size'] = ""
else:
fields['protocolName'] = ""
fields['output'] = obj.output
fields['size'] = str(obj.outSize)
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
# Ctf monitor chart data
last_id = self.confParser.getint("ctf", "lastId")
listDictionaryCTF = {} if self.ctfMonitor is None else \
self.ctfMonitor.getData(lastId=last_id)
if listDictionaryCTF:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'ctf'
for ctf in listDictionaryCTF:
fields = {}
for key in ctf:
if key == 'timestamp':
pointsDict['time'] = ctf['timestamp']
# id must be a tag since tw CTF may have the same timestamp
# and time + tags must be unique
elif key == 'id':
tags['id'] = ctf['id']
# elif key == 'ctfID':
# fields['ctfID'] = "<b>" + str(ctf[key]) + "</b>"
elif key == 'shiftPlotPath':
temp = os.path.join(self.apacheImgDir,
self.projectName,
basename(ctf[key]))
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, basename(ctf[key]))
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
elif key == 'psdPath':
# convert to pǹg
baseName = basename(ctf[key])
baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
temp = os.path.join(self.apacheImgDir,
self.projectName, baseName)
temp = pwutils.replaceExt(temp, "png")
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, baseNamePng)
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
fields[key + 'LocalPng'] = pwutils.replaceExt(
ctf[key], "png")
elif key == 'micPath':
# convert to pǹg
baseName = basename(ctf[key])
baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
temp = os.path.join(self.apacheImgDir,
self.projectName, baseName)
temp = pwutils.replaceExt(temp, "png")
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, baseNamePng)
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
fields[key + 'LocalPng'] = pwutils.replaceExt(
ctf[key], "png")
else:
fields[key] = ctf[key]
# while be use to control image creation
fields["transferImage"] = False
pointsDict['fields'] = fields
pointsDict['tags'] = tags
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("ctf", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# GAIN Section
last_id = self.confParser.getint("gain", "lastId")
listDictionaryGain = {} if self.movieGainMonitor is None else \
self.movieGainMonitor.getData(lastId=last_id)
if listDictionaryGain:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'gain'
pointsDict['tags'] = tags
counter = 1
tnow = datetime.now()
for gain in listDictionaryGain:
fields = {}
for key in gain:
fields[key] = gain[key]
pointsDict['fields'] = fields
# gain has no time information so I just
# put current time
localNow = tnow + timedelta(seconds=counter)
pointsDict[
'time'] = localNow # .strftime('%Y-%m-%dT%H:%M:%SZ')
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("gain", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# SYSTEM data
last_id = self.confParser.getint("system", "lastId")
listDictionarySystem = {} if self.sysMonitor is None else \
self.sysMonitor.getData(lastId=last_id)
if listDictionarySystem:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'system'
pointsDict['tags'] = tags
for system in listDictionarySystem:
fields = {}
for key in system:
if key == 'timestamp':
pointsDict['time'] = system['timestamp']
elif key == 'id':
continue
else:
fields[key] = system[key]
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("system", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
self.transferFiles()
return last_id # reportFinished
def _updateLabel(self):
self.updateVar.set('Updated: %s' % pwutils.prettyTime(secs=True))
# Schedule a refresh in some seconds
self.tree.after(self.refresh * 1000, self._updateData)
def _updateLabel(self):
self.updateVar.set('Updated: %s' % pwutils.prettyTime(secs=True))
# Schedule a refresh in some seconds
self.tree.after(self.refresh * 1000, self._updateData)
def __init__(self, parent, windows, **kwargs):
tk.Frame.__init__(self, parent, bg='white', **kwargs)
self.windows = windows
self.manager = windows.manager
self.root = windows.root
self.vars = {}
self.checkvars = []
self.microscope = None
# Regular expression to validate username and sample name
self.re = re.compile('\A[a-zA-Z][a-zA-Z0-9_-]+\Z')
# tkFont.Font(size=12, family='verdana', weight='bold')
bigSize = pwgui.cfgFontSize + 2
smallSize = pwgui.cfgFontSize - 2
fontName = pwgui.cfgFontName
self.bigFont = tkFont.Font(size=bigSize, family=fontName)
self.bigFontBold = tkFont.Font(size=bigSize, family=fontName,
weight='bold')
self.projDateFont = tkFont.Font(size=smallSize, family=fontName)
self.projDelFont = tkFont.Font(size=smallSize, family=fontName,
weight='bold')
self.manager = Manager()
# Header section
headerFrame = tk.Frame(self, bg='white')
headerFrame.grid(row=0, column=0, sticky='new')
headerText = "Create New Session"
headerText += " %s" % pwutils.prettyTime(dateFormat='%Y-%m-%d')
label = tk.Label(headerFrame, text=headerText,
font=self.bigFontBold,
borderwidth=0, anchor='nw', bg='white',
fg=pwgui.Color.DARK_GREY_COLOR)
label.grid(row=0, column=0, sticky='nw', padx=(20, 5), pady=10)
# Body section
bodyFrame = tk.Frame(self, bg='white')
bodyFrame.grid(row=1, column=0, sticky='news')
self._fillContent(bodyFrame)
# Add the create project button
btnFrame = tk.Frame(self, bg='white')
btn = HotButton(btnFrame, text="Create New Session",
font=self.bigFontBold,
command=self._onAction)
btn.grid(row=0, column=1, sticky='ne', padx=10, pady=10)
# Add the Import project button
btn = Button(btnFrame, Message.LABEL_BUTTON_CANCEL,
Icon.ACTION_CLOSE,
font=self.bigFontBold,
command=self.windows.close)
btn.grid(row=0, column=0, sticky='ne', padx=10, pady=10)
btnFrame.grid(row=2, column=0, sticky='sew')
btnFrame.columnconfigure(0, weight=1)
self.columnconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
def _getProjectName(self):
return '%s_%s_%s' % (pwutils.prettyTime(dateFormat='%Y%m%d'),
self._getValue(USER_NAME),
self._getValue(SAMPLE_NAME))
def generate(self, finished):
reportTemplate = self.getHTMLReportText()
if not reportTemplate:
raise Exception("HTML template file '%s' not found. "
% self.template)
project = self.protocol.getProject()
reportName = 'index.html'
projName = project.getShortName()
reportDir = abspath(self.protocol._getExtraPath(projName))
self.info("Creating report directory: %s" % reportDir)
pwutils.cleanPath(reportDir)
pwutils.makePath(reportDir)
reportPath = join(reportDir, reportName)
acquisitionLines = ''
self.provider.refreshObjects()
for item in self.provider.acquisition:
if not acquisitionLines == '':
acquisitionLines += ','
acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item
runLines = ''
wasProtocol = None
for obj in self.provider.getObjects():
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
if runLines != '': runLines += ']},'
runLines += '{protocolName: "%s", output:[' % obj.name
else:
if not wasProtocol: runLines += ','
runLines += '{name: "%s", size:"%s"}' % (obj.output,
obj.outSize)
wasProtocol = isProtocol
# End the runLines JSON object
runLines += ']}'
# Ctf monitor chart data
data = [] if self.ctfMonitor is None else self.ctfMonitor.getData()
ctfData = json.dumps(data)
# system monitor chart data
data = self.sysMonitor.getData()
systemData = json.dumps(data)
args = {'projectName': projName,
'startTime': pwutils.dateStr(project.getCreationTime(), secs=True),
'dateStr': pwutils.prettyTime(secs=True),
'projectDuration': pwutils.prettyDelta(project.getElapsedTime()),
'projectStatus': "FINISHED" if finished else "RUNNING",
'scipionVersion': os.environ['SCIPION_VERSION'],
'acquisitionLines': acquisitionLines,
'runLines': runLines,
'ctfData': ctfData,
'systemData': systemData,
'refresh': '<META http-equiv="refresh" content="%s" >' % self.refreshSecs if not finished else ''
}
self.info("Writing report html to: %s" % abspath(reportPath))
reportFile = open(reportPath, 'w')
reportTemplate = reportTemplate % args
reportFile.write(reportTemplate.encode('utf-8'))
reportFile.close()
if self.publishCmd:
self.info("Publishing the report:")
cmd = self.publishCmd % {'REPORT_FOLDER': reportDir}
self.info(cmd)
os.system(cmd)
return reportPath