def sleepStep(self, t, s):
log = self._getPath("step_%02d.txt" % t)
import time, datetime
f = open(log, 'w+')
f.write("Going to sleep at %s\n" % dateStr(datetime.datetime.now(), True))
time.sleep(t)
f.write(" Slept: %d seconds\n" % t)
f.write("Awaked at %s\n" % dateStr(datetime.datetime.now(), True))
f.close()
return [log]
def sleepStep(self, t):
log = self._getPath("step_%02d.txt" % t)
f = open(log, 'w+')
f.write("Going to sleep at %s\n" %
pwutils.dateStr(datetime.datetime.now(), True))
time.sleep(t)
f.write(" Slept: %d seconds\n" % t)
f.write("Awakened at %s\n" %
pwutils.dateStr(datetime.datetime.now(), True))
f.close()
return [log]
def generate(self, finished):
reportTemplate = self.getHTMLReportText()
if not reportTemplate:
raise Exception("HTML template file '%s' not found. " %
self.template)
project = self.protocol.getProject()
projName = project.getShortName()
acquisitionLines = ''
self.provider.refreshObjects()
for item in self.provider.acquisition:
if not acquisitionLines == '':
acquisitionLines += ','
acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item
runLines = ''
wasProtocol = None
for obj in self.provider.getObjects():
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
if runLines != '': runLines += ']},'
runLines += '{protocolName: "%s", output:[' % obj.name
else:
if not wasProtocol: runLines += ','
runLines += '{name: "%s", size:"%s"}' % (obj.output,
obj.outSize)
wasProtocol = isProtocol
# End the runLines JSON object
runLines += ']}'
# Ctf monitor chart data
data = {} if self.ctfMonitor is None else self.ctfMonitor.getData()
if data:
numMicsDone = len(self.thumbPaths[PSD_THUMBS])
numMics = len(data[PSD_PATH])
numMicsToDo = numMics - numMicsDone
self.getThumbPaths(ctfData=data,
thumbsDone=numMicsDone,
micIdSet=data['idValues'])
if len(data['defocusU']) < 100:
data['defocusCoverage'] = self.processDefocusValues(
data['defocusU'])
else:
data['defocusCoverage'] = self.processDefocusValues(
data['defocusU'][:-50])
data['defocusCoverageLast50'] = self.processDefocusValues(
data['defocusU'][-50:])
data['resolutionHistogram'] = self.getResolutionHistogram(
data['resolution'])
else:
# Thumbnails for Micrograph Table
numMicsDone = len(self.thumbPaths[MIC_THUMBS])
self.getThumbPaths(thumbsDone=numMicsDone)
numMics = len(self.thumbPaths[MIC_PATH])
numMicsToDo = numMics - numMicsDone
if numMicsToDo <= 10:
# we have few new images, eg streaming mode, generate thumbnails now
self.generateReportImages(firstThumbIndex=numMicsDone)
else:
# we have many images, generate thumbs in a separate process
process = multiprocessing.Process(target=self.generateReportImages,
args=(numMicsDone, ))
process.start()
# send over only thumbnails of the mics that have been fully processed
self.thumbsReady = self.checkNewThumbsReady()
thumbsLoading = numMics - self.thumbsReady
for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]:
if k in self.thumbPaths:
data[k] = self.thumbPaths[k][:self.thumbsReady] + [
''
] * thumbsLoading
data[MIC_ID] = self.thumbPaths[MIC_ID]
reportFinished = self.thumbsReady == numMics
ctfData = json.dumps(data)
# Movie gain monitor chart data
data = [] if self.movieGainMonitor is None else self.movieGainMonitor.getData(
)
movieGainData = json.dumps(data)
# system monitor chart data
data = self.sysMonitor.getData()
systemData = json.dumps(data)
tnow = datetime.now()
args = {
'projectName':
projName,
'startTime':
pwutils.dateStr(project.getCreationTime(), secs=True),
'dateStr':
pwutils.prettyTime(dt=tnow, secs=True),
'projectDuration':
pwutils.prettyDelta(tnow - project.getCreationTime()),
'projectStatus':
"FINISHED" if finished else "RUNNING",
'scipionVersion':
os.environ['SCIPION_VERSION'],
'acquisitionLines':
acquisitionLines,
'runLines':
runLines,
'ctfData':
ctfData,
'movieGainData':
movieGainData,
'systemData':
systemData,
'refresh':
self.refreshSecs
}
self.info("Writing report html to: %s" % abspath(self.reportPath))
pwutils.cleanPath(self.reportPath)
reportFile = open(self.reportPath, 'w')
reportTemplate = reportTemplate % args
reportFile.write(reportTemplate.encode('utf-8'))
reportFile.close()
if self.publishCmd:
self.info("Publishing the report:")
cmd = self.publishCmd % {'REPORT_FOLDER': self.reportDir}
self.info(cmd)
os.system(cmd)
return reportFinished
def getDateStr(self):
return dateStr(self.getDate()) if self._stat else '0'
def getDateStr(self):
return pwutils.dateStr(self.getDate()) if self._stat else '0'
def generate(self, finished):
reportTemplate = self.getHTMLReportText()
if not reportTemplate:
raise Exception("HTML template file '%s' not found. "
% self.template)
project = self.protocol.getProject()
projName = project.getShortName()
acquisitionLines = ''
self.provider.refreshObjects()
for item in self.provider.acquisition:
if not acquisitionLines == '':
acquisitionLines += ','
acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item
runLines = ''
wasProtocol = None
for obj in self.provider.getObjects():
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
if runLines != '': runLines += ']},'
runLines += '{protocolName: "%s", output:[' % obj.name
else:
if not wasProtocol: runLines += ','
runLines += '{name: "%s", size:"%s"}' % (obj.output,
obj.outSize)
wasProtocol = isProtocol
# End the runLines JSON object
runLines += ']}'
# Ctf monitor chart data
data = {} if self.ctfMonitor is None else self.ctfMonitor.getData()
if data:
numMicsDone = len(self.thumbPaths[PSD_THUMBS])
numMics = len(data[PSD_PATH])
numMicsToDo = numMics - numMicsDone
self.getThumbPaths(ctfData=data, thumbsDone=numMicsDone, micIdSet=data['idValues'])
if len(data['defocusU']) < 100:
data['defocusCoverage'] = self.processDefocusValues(data['defocusU'])
else:
data['defocusCoverage'] = self.processDefocusValues(data['defocusU'][:-50])
data['defocusCoverageLast50'] = self.processDefocusValues(data['defocusU'][-50:])
data['resolutionHistogram'] = self.getResolutionHistogram(data['resolution'])
else:
# Thumbnails for Micrograph Table
numMicsDone = len(self.thumbPaths[MIC_THUMBS])
self.getThumbPaths(thumbsDone=numMicsDone)
numMics = len(self.thumbPaths[MIC_PATH])
numMicsToDo = numMics - numMicsDone
if numMicsToDo <= 10:
# we have few new images, eg streaming mode, generate thumbnails now
self.generateReportImages(firstThumbIndex=numMicsDone)
else:
# we have many images, generate thumbs in a separate process
process = multiprocessing.Process(target=self.generateReportImages,
args=(numMicsDone,))
process.start()
# send over only thumbnails of the mics that have been fully processed
self.thumbsReady = self.checkNewThumbsReady()
thumbsLoading = numMics - self.thumbsReady
for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]:
if k in self.thumbPaths:
data[k] = self.thumbPaths[k][:self.thumbsReady] + ['']*thumbsLoading
data[MIC_ID] = self.thumbPaths[MIC_ID]
reportFinished = self.thumbsReady == numMics
ctfData = json.dumps(data)
# Movie gain monitor chart data
data = [] if self.movieGainMonitor is None else self.movieGainMonitor.getData()
movieGainData = json.dumps(data)
# system monitor chart data
data = self.sysMonitor.getData()
systemData = json.dumps(data)
tnow = datetime.now()
args = {'projectName': projName,
'startTime': pwutils.dateStr(project.getCreationTime(), secs=True),
'dateStr': pwutils.prettyTime(dt=tnow, secs=True),
'projectDuration': pwutils.prettyDelta(tnow-project.getCreationTime()),
'projectStatus': "FINISHED" if finished else "RUNNING",
'scipionVersion': os.environ['SCIPION_VERSION'],
'acquisitionLines': acquisitionLines,
'runLines': runLines,
'ctfData': ctfData,
'movieGainData': movieGainData,
'systemData': systemData,
'refresh': self.refreshSecs
}
self.info("Writing report html to: %s" % abspath(self.reportPath))
pwutils.cleanPath(self.reportPath)
reportFile = open(self.reportPath, 'w')
reportTemplate = reportTemplate % args
reportFile.write(reportTemplate.encode('utf-8'))
reportFile.close()
if self.publishCmd:
self.info("Publishing the report:")
cmd = self.publishCmd % {'REPORT_FOLDER': self.reportDir}
self.info(cmd)
os.system(cmd)
return reportFinished
def generate(self, finished):
reportTemplate = self.getHTMLReportText()
if not reportTemplate:
raise Exception("HTML template file '%s' not found. "
% self.template)
project = self.protocol.getProject()
reportName = 'index.html'
projName = project.getShortName()
reportDir = abspath(self.protocol._getExtraPath(projName))
self.info("Creating report directory: %s" % reportDir)
pwutils.cleanPath(reportDir)
pwutils.makePath(reportDir)
reportPath = join(reportDir, reportName)
acquisitionLines = ''
self.provider.refreshObjects()
for item in self.provider.acquisition:
if not acquisitionLines == '':
acquisitionLines += ','
acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item
runLines = ''
wasProtocol = None
for obj in self.provider.getObjects():
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
if runLines != '': runLines += ']},'
runLines += '{protocolName: "%s", output:[' % obj.name
else:
if not wasProtocol: runLines += ','
runLines += '{name: "%s", size:"%s"}' % (obj.output,
obj.outSize)
wasProtocol = isProtocol
# End the runLines JSON object
runLines += ']}'
# Ctf monitor chart data
data = [] if self.ctfMonitor is None else self.ctfMonitor.getData()
ctfData = json.dumps(data)
# system monitor chart data
data = self.sysMonitor.getData()
systemData = json.dumps(data)
args = {'projectName': projName,
'startTime': pwutils.dateStr(project.getCreationTime(), secs=True),
'dateStr': pwutils.prettyTime(secs=True),
'projectDuration': pwutils.prettyDelta(project.getElapsedTime()),
'projectStatus': "FINISHED" if finished else "RUNNING",
'scipionVersion': os.environ['SCIPION_VERSION'],
'acquisitionLines': acquisitionLines,
'runLines': runLines,
'ctfData': ctfData,
'systemData': systemData,
'refresh': '<META http-equiv="refresh" content="%s" >' % self.refreshSecs if not finished else ''
}
self.info("Writing report html to: %s" % abspath(reportPath))
reportFile = open(reportPath, 'w')
reportTemplate = reportTemplate % args
reportFile.write(reportTemplate.encode('utf-8'))
reportFile.close()
if self.publishCmd:
self.info("Publishing the report:")
cmd = self.publishCmd % {'REPORT_FOLDER': reportDir}
self.info(cmd)
os.system(cmd)
return reportPath
def getDate(self):
return dateStr(self._stat.st_mtime) if self._stat else 0
def generate(self, finished):
project = self.protocol.getProject()
# Project Properties Section
# Do not delete this variables. We are using them
# in an eval command
self.projectName = project.getShortName()
startTime = pwutils.dateStr(project.getCreationTime(), secs=True),
tnow = datetime.now()
_now = project.getCreationTime()
dateStr = pwutils.prettyTime(dt=tnow, secs=True),
projectDuration = pwutils.prettyDelta(tnow -
project.getCreationTime()),
projectStatus = "FINISHED" if finished else "RUNNING",
scipionVersion = os.environ['SCIPION_VERSION'],
# create structure with information related with the properties section
# the primmary key is time, projectionName, section
# if we insert two points with these three values idential
# the second updates the first
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'properties'
pointsDict['tags'] = tags
fields = {}
firstTime = self.confParser.getint("project", "properties")
fieldKeys = {'dateStr': 4, 'projectDuration': 3, 'projectStatus': 2}
fieldNames = {
'dateStr': "<b>Last Update</b>",
'projectDuration': '<b>Duration</b>',
'projectStatus': '<b>Status</b>'
}
if firstTime:
fieldKeys['startTime'] = 5
fieldKeys['scipionVersion'] = 1
fieldNames['startTime'] = '<b>Start Time</b>'
fieldNames['scipionVersion'] = '<b>Scipion Version</b>'
for metric, delta in fieldKeys.items():
localNow = _now + timedelta(seconds=delta)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
fields['metric'] = fieldNames[metric]
# str is need because all values must have the same type
fields['valueStr'] = str(eval(metric)[0])
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
self.confParser.set("project", "properties", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# acquisition section
self.provider.refreshObjects()
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'acquisition'
pointsDict['tags'] = tags
fields = {}
firstTime = self.confParser.getint("project", "acquisition")
if firstTime:
delta = 0
for metricName, value in self.provider.acquisition:
localNow = _now + timedelta(seconds=delta)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
fields['metric'] = "<b>" + metricName + "</b>"
fields['valueNum'] = float(value)
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
delta = delta - 1
# update first time only if some date has been read.
# do not place this upside the loop
self.confParser.set("project", "acquisition", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# send summary section
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'summary'
pointsDict['tags'] = tags
localNow = _now
for obj in self.provider.getObjects():
fields = {}
localNow = localNow + timedelta(seconds=-1)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
fields['protocolName'] = '<b>' + obj.name + '</b>'
fields['output'] = ""
fields['size'] = ""
else:
fields['protocolName'] = ""
fields['output'] = obj.output
fields['size'] = str(obj.outSize)
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
# Ctf monitor chart data
last_id = self.confParser.getint("ctf", "lastId")
listDictionaryCTF = {} if self.ctfMonitor is None else \
self.ctfMonitor.getData(lastId=last_id)
if listDictionaryCTF:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'ctf'
for ctf in listDictionaryCTF:
fields = {}
for key in ctf:
if key == 'timestamp':
pointsDict['time'] = ctf['timestamp']
# id must be a tag since tw CTF may have the same timestamp
# and time + tags must be unique
elif key == 'id':
tags['id'] = ctf['id']
# elif key == 'ctfID':
# fields['ctfID'] = "<b>" + str(ctf[key]) + "</b>"
elif key == 'shiftPlotPath':
temp = os.path.join(self.apacheImgDir,
self.projectName,
basename(ctf[key]))
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, basename(ctf[key]))
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
elif key == 'psdPath':
# convert to pǹg
baseName = basename(ctf[key])
baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
temp = os.path.join(self.apacheImgDir,
self.projectName, baseName)
temp = pwutils.replaceExt(temp, "png")
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, baseNamePng)
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
fields[key + 'LocalPng'] = pwutils.replaceExt(
ctf[key], "png")
elif key == 'micPath':
# convert to pǹg
baseName = basename(ctf[key])
baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
temp = os.path.join(self.apacheImgDir,
self.projectName, baseName)
temp = pwutils.replaceExt(temp, "png")
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, baseNamePng)
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
fields[key + 'LocalPng'] = pwutils.replaceExt(
ctf[key], "png")
else:
fields[key] = ctf[key]
# while be use to control image creation
fields["transferImage"] = False
pointsDict['fields'] = fields
pointsDict['tags'] = tags
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("ctf", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# GAIN Section
last_id = self.confParser.getint("gain", "lastId")
listDictionaryGain = {} if self.movieGainMonitor is None else \
self.movieGainMonitor.getData(lastId=last_id)
if listDictionaryGain:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'gain'
pointsDict['tags'] = tags
counter = 1
tnow = datetime.now()
for gain in listDictionaryGain:
fields = {}
for key in gain:
fields[key] = gain[key]
pointsDict['fields'] = fields
# gain has no time information so I just
# put current time
localNow = tnow + timedelta(seconds=counter)
pointsDict[
'time'] = localNow # .strftime('%Y-%m-%dT%H:%M:%SZ')
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("gain", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# SYSTEM data
last_id = self.confParser.getint("system", "lastId")
listDictionarySystem = {} if self.sysMonitor is None else \
self.sysMonitor.getData(lastId=last_id)
if listDictionarySystem:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'system'
pointsDict['tags'] = tags
for system in listDictionarySystem:
fields = {}
for key in system:
if key == 'timestamp':
pointsDict['time'] = system['timestamp']
elif key == 'id':
continue
else:
fields[key] = system[key]
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("system", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
self.transferFiles()
return last_id # reportFinished
def getDate(self):
return dateStr(self._stat.st_mtime) if self._stat else 0
def getDateStr(self):
return dateStr(self.getDate()) if self._stat else '0'