def getObjectInfo(self, obj):
objId = obj.getObjId()
self._objDict[objId] = obj
info = {'key': obj.getObjId(), 'text': obj.getRunName(),
'values': (obj.getStatusMessage(), prettyDelta(obj.getElapsedTime()))}
objPid = obj.getObjParentId()
if objPid in self._objDict:
info['parent'] = self._objDict[objPid]
return info
def fileModified(self, fileName, fileTimeout):
""" Check if the fileName modification time is less
than a given timeout.
Params:
fileName: input filename that will be checked.
fileTimeout: timeout """
self.debug('Checking file: %s' % fileName)
mTime = datetime.fromtimestamp(os.path.getmtime(fileName))
delta = datetime.now() - mTime
self.debug(' Modification time: %s' % pwutils.prettyTime(mTime))
self.debug(' Delta: %s' % pwutils.prettyDelta(delta))
return delta < fileTimeout
def fileModified(self, fileName, fileTimeout):
""" Check if the fileName modification time is less
than a given timeout.
Params:
fileName: input filename that will be checked.
fileTimeout: timeout """
self.debug('Checking file: %s' % fileName)
mTime = datetime.fromtimestamp(os.path.getmtime(fileName))
delta = datetime.now() - mTime
self.debug(' Modification time: %s' % pwutils.prettyTime(mTime))
self.debug(' Delta: %s' % pwutils.prettyDelta(delta))
return delta < fileTimeout
def getObjectInfo(self, obj):
objId = obj.getObjId()
self._objDict[objId] = obj
info = {
'key': obj.getObjId(),
'text': obj.getRunName(),
'values':
(obj.getStatusMessage(), prettyDelta(obj.getElapsedTime()))
}
objPid = obj.getObjParentId()
if objPid in self._objDict:
info['parent'] = self._objDict[objPid]
return info
def importImagesStreamStep(self, pattern, voltage, sphericalAberration,
amplitudeContrast, magnification):
""" Copy images matching the filename pattern
Register other parameters.
"""
self.info("Using pattern: '%s'" % pattern)
imgSet = self._getOutputSet() if self.isContinued() else None
self.importedFiles = set()
if imgSet is None:
createSetFunc = getattr(self, '_create' + self._outputClassName)
imgSet = createSetFunc()
elif imgSet.getSize() > 0: # in case of continue
imgSet.loadAllProperties()
self._fillImportedFiles(imgSet)
imgSet.enableAppend()
pointerExcludedMovs = getattr(self, 'moviesToExclude', None)
if pointerExcludedMovs is not None:
excludedMovs = pointerExcludedMovs.get()
self._fillImportedFiles(excludedMovs)
imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
acquisition = imgSet.getAcquisition()
self.fillAcquisition(acquisition)
# Call a function that should be implemented by each subclass
self.setSamplingRate(imgSet)
outFiles = [imgSet.getFileName()]
imgh = ImageHandler()
img = imgSet.ITEM_TYPE()
img.setAcquisition(acquisition)
n = 1
copyOrLink = self.getCopyOrLink()
outputName = self._getOutputName()
finished = False
# this is only used when creating stacks from frame files
self.createdStacks = set()
i = 0
lastDetectedChange = datetime.now()
# Ignore the timeout variables if we are not really in streaming mode
if self.dataStreaming:
timeout = timedelta(seconds=self.timeout.get())
fileTimeout = timedelta(seconds=self.fileTimeout.get())
else:
timeout = timedelta(seconds=5)
fileTimeout = timedelta(seconds=5)
while not finished:
time.sleep(3) # wait 3 seconds before check for new files
someNew = False
someAdded = False
for fileName, uniqueFn, fileId in self.iterNewInputFiles():
someNew = True
if self.fileModified(fileName, fileTimeout):
continue
dst = self._getExtraPath(uniqueFn)
self.importedFiles.add(uniqueFn)
copyOrLink(fileName, dst)
self.debug('Importing file: %s' % fileName)
self.debug("uniqueFn: %s" % uniqueFn)
self.debug("dst Fn: %s" % dst)
if self._checkStacks:
_, _, _, n = imgh.getDimensions(dst)
someAdded = True
self.debug('Appending file to DB...')
if self.importedFiles: # enable append after first append
imgSet.enableAppend()
if n > 1:
for index in range(1, n + 1):
img.cleanObjId()
img.setMicId(fileId)
img.setFileName(dst)
img.setIndex(index)
self._addImageToSet(img, imgSet)
else:
img.setObjId(fileId)
img.setFileName(dst)
# Fill the micName if img is either a Micrograph or a Movie
self.debug("FILENAME TO fillMicName: %s" % uniqueFn)
self._fillMicName(img, uniqueFn)
self._addImageToSet(img, imgSet)
outFiles.append(dst)
self.debug('After append. Files: %d' % len(outFiles))
if someAdded:
self.debug('Updating output...')
self._updateOutputSet(outputName,
imgSet,
state=imgSet.STREAM_OPEN)
self.debug('Update Done.')
self.debug('Checking if finished...someNew: %s' % someNew)
now = datetime.now()
if not someNew:
# If there are no new detected files, we should check the
# inactivity time elapsed (from last event to now) and
# if it is greater than the defined timeout, we conclude
# the import and close the output set
# Another option is to check if the protocol have some
# special stop condition, this can be used to manually stop
# some protocols such as import movies
finished = (now - lastDetectedChange > timeout
or self.streamingHasFinished())
self.debug("Checking if finished:")
self.debug(" Now - Last Change: %s" %
pwutils.prettyDelta(now - lastDetectedChange))
self.debug("Finished: %s" % finished)
else:
# If we have detected some files, we should update
# the timestamp of the last event
lastDetectedChange = now
self._updateOutputSet(outputName, imgSet, state=imgSet.STREAM_CLOSED)
self._cleanUp()
return outFiles
def generate(self, finished):
reportTemplate = self.getHTMLReportText()
if not reportTemplate:
raise Exception("HTML template file '%s' not found. " %
self.template)
project = self.protocol.getProject()
projName = project.getShortName()
acquisitionLines = ''
self.provider.refreshObjects()
for item in self.provider.acquisition:
if not acquisitionLines == '':
acquisitionLines += ','
acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item
runLines = ''
wasProtocol = None
for obj in self.provider.getObjects():
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
if runLines != '': runLines += ']},'
runLines += '{protocolName: "%s", output:[' % obj.name
else:
if not wasProtocol: runLines += ','
runLines += '{name: "%s", size:"%s"}' % (obj.output,
obj.outSize)
wasProtocol = isProtocol
# End the runLines JSON object
runLines += ']}'
# Ctf monitor chart data
data = {} if self.ctfMonitor is None else self.ctfMonitor.getData()
if data:
numMicsDone = len(self.thumbPaths[PSD_THUMBS])
numMics = len(data[PSD_PATH])
numMicsToDo = numMics - numMicsDone
self.getThumbPaths(ctfData=data,
thumbsDone=numMicsDone,
micIdSet=data['idValues'])
if len(data['defocusU']) < 100:
data['defocusCoverage'] = self.processDefocusValues(
data['defocusU'])
else:
data['defocusCoverage'] = self.processDefocusValues(
data['defocusU'][:-50])
data['defocusCoverageLast50'] = self.processDefocusValues(
data['defocusU'][-50:])
data['resolutionHistogram'] = self.getResolutionHistogram(
data['resolution'])
else:
# Thumbnails for Micrograph Table
numMicsDone = len(self.thumbPaths[MIC_THUMBS])
self.getThumbPaths(thumbsDone=numMicsDone)
numMics = len(self.thumbPaths[MIC_PATH])
numMicsToDo = numMics - numMicsDone
if numMicsToDo <= 10:
# we have few new images, eg streaming mode, generate thumbnails now
self.generateReportImages(firstThumbIndex=numMicsDone)
else:
# we have many images, generate thumbs in a separate process
process = multiprocessing.Process(target=self.generateReportImages,
args=(numMicsDone, ))
process.start()
# send over only thumbnails of the mics that have been fully processed
self.thumbsReady = self.checkNewThumbsReady()
thumbsLoading = numMics - self.thumbsReady
for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]:
if k in self.thumbPaths:
data[k] = self.thumbPaths[k][:self.thumbsReady] + [
''
] * thumbsLoading
data[MIC_ID] = self.thumbPaths[MIC_ID]
reportFinished = self.thumbsReady == numMics
ctfData = json.dumps(data)
# Movie gain monitor chart data
data = [] if self.movieGainMonitor is None else self.movieGainMonitor.getData(
)
movieGainData = json.dumps(data)
# system monitor chart data
data = self.sysMonitor.getData()
systemData = json.dumps(data)
tnow = datetime.now()
args = {
'projectName':
projName,
'startTime':
pwutils.dateStr(project.getCreationTime(), secs=True),
'dateStr':
pwutils.prettyTime(dt=tnow, secs=True),
'projectDuration':
pwutils.prettyDelta(tnow - project.getCreationTime()),
'projectStatus':
"FINISHED" if finished else "RUNNING",
'scipionVersion':
os.environ['SCIPION_VERSION'],
'acquisitionLines':
acquisitionLines,
'runLines':
runLines,
'ctfData':
ctfData,
'movieGainData':
movieGainData,
'systemData':
systemData,
'refresh':
self.refreshSecs
}
self.info("Writing report html to: %s" % abspath(self.reportPath))
pwutils.cleanPath(self.reportPath)
reportFile = open(self.reportPath, 'w')
reportTemplate = reportTemplate % args
reportFile.write(reportTemplate.encode('utf-8'))
reportFile.close()
if self.publishCmd:
self.info("Publishing the report:")
cmd = self.publishCmd % {'REPORT_FOLDER': self.reportDir}
self.info(cmd)
os.system(cmd)
return reportFinished
def importImagesStreamStep(self, pattern, voltage, sphericalAberration,
amplitudeContrast, magnification):
""" Copy images matching the filename pattern
Register other parameters.
"""
self.info("Using pattern: '%s'" % pattern)
imgSet = self._getOutputSet() if self.isContinued() else None
self.importedFiles = set()
if imgSet is None:
createSetFunc = getattr(self, '_create' + self._outputClassName)
imgSet = createSetFunc()
elif imgSet.getSize() > 0: # in case of continue
imgSet.loadAllProperties()
self._fillImportedFiles(imgSet)
imgSet.enableAppend()
pointerExcludedMovs = getattr(self, 'moviesToExclude', None)
if pointerExcludedMovs is not None:
excludedMovs = pointerExcludedMovs.get()
self._fillImportedFiles(excludedMovs)
imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
acquisition = imgSet.getAcquisition()
self.fillAcquisition(acquisition)
# Call a function that should be implemented by each subclass
self.setSamplingRate(imgSet)
outFiles = [imgSet.getFileName()]
imgh = ImageHandler()
img = imgSet.ITEM_TYPE()
img.setAcquisition(acquisition)
n = 1
copyOrLink = self.getCopyOrLink()
outputName = self._getOutputName()
alreadyWarned = False # Use this flag to warn only once
finished = False
# this is only used when creating stacks from frame files
self.createdStacks = set()
i = 0
lastDetectedChange = datetime.now()
# Ignore the timeout variables if we are not really in streaming mode
if self.dataStreaming:
timeout = timedelta(seconds=self.timeout.get())
fileTimeout = timedelta(seconds=self.fileTimeout.get())
else:
timeout = timedelta(seconds=5)
fileTimeout = timedelta(seconds=5)
while not finished:
time.sleep(3) # wait 3 seconds before check for new files
someNew = False
someAdded = False
for fileName, uniqueFn, fileId in self.iterNewInputFiles():
someNew = True
if self.fileModified(fileName, fileTimeout):
continue
dst = self._getExtraPath(uniqueFn)
self.importedFiles.add(uniqueFn)
if ' ' in dst:
if not alreadyWarned:
self.warning('Warning: your file names have white spaces!')
self.warning('Removing white spaces from copies/symlinks.')
alreadyWarned = True
dst = dst.replace(' ', '')
copyOrLink(fileName, dst)
self.debug('Importing file: %s' % fileName)
self.debug("uniqueFn: %s" % uniqueFn)
self.debug("dst Fn: %s" % dst)
if self._checkStacks:
_, _, _, n = imgh.getDimensions(dst)
someAdded = True
self.debug('Appending file to DB...')
if self.importedFiles: # enable append after first append
imgSet.enableAppend()
if n > 1:
for index in range(1, n+1):
img.cleanObjId()
img.setMicId(fileId)
img.setFileName(dst)
img.setIndex(index)
self._addImageToSet(img, imgSet)
else:
img.setObjId(fileId)
img.setFileName(dst)
# Fill the micName if img is either a Micrograph or a Movie
uniqueFn = uniqueFn.replace(' ', '')
self.debug("FILENAME TO fillMicName: %s" % uniqueFn)
self._fillMicName(img, uniqueFn)
self._addImageToSet(img, imgSet)
outFiles.append(dst)
self.debug('After append. Files: %d' % len(outFiles))
if someAdded:
self.debug('Updating output...')
self._updateOutputSet(outputName, imgSet,
state=imgSet.STREAM_OPEN)
self.debug('Update Done.')
self.debug('Checking if finished...someNew: %s' % someNew)
now = datetime.now()
if not someNew:
# If there are no new detected files, we should check the
# inactivity time elapsed (from last event to now) and
# if it is greater than the defined timeout, we conclude
# the import and close the output set
# Another option is to check if the protocol have some
# special stop condition, this can be used to manually stop
# some protocols such as import movies
finished = (now - lastDetectedChange > timeout or
self.streamingHasFinished())
self.debug("Checking if finished:")
self.debug(" Now - Last Change: %s"
% pwutils.prettyDelta(now - lastDetectedChange))
self.debug("Finished: %s" % finished)
else:
# If we have detected some files, we should update
# the timestamp of the last event
lastDetectedChange = now
self._updateOutputSet(outputName, imgSet,
state=imgSet.STREAM_CLOSED)
self._cleanUp()
return outFiles
def generate(self, finished):
reportTemplate = self.getHTMLReportText()
if not reportTemplate:
raise Exception("HTML template file '%s' not found. "
% self.template)
project = self.protocol.getProject()
projName = project.getShortName()
acquisitionLines = ''
self.provider.refreshObjects()
for item in self.provider.acquisition:
if not acquisitionLines == '':
acquisitionLines += ','
acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item
runLines = ''
wasProtocol = None
for obj in self.provider.getObjects():
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
if runLines != '': runLines += ']},'
runLines += '{protocolName: "%s", output:[' % obj.name
else:
if not wasProtocol: runLines += ','
runLines += '{name: "%s", size:"%s"}' % (obj.output,
obj.outSize)
wasProtocol = isProtocol
# End the runLines JSON object
runLines += ']}'
# Ctf monitor chart data
data = {} if self.ctfMonitor is None else self.ctfMonitor.getData()
if data:
numMicsDone = len(self.thumbPaths[PSD_THUMBS])
numMics = len(data[PSD_PATH])
numMicsToDo = numMics - numMicsDone
self.getThumbPaths(ctfData=data, thumbsDone=numMicsDone, micIdSet=data['idValues'])
if len(data['defocusU']) < 100:
data['defocusCoverage'] = self.processDefocusValues(data['defocusU'])
else:
data['defocusCoverage'] = self.processDefocusValues(data['defocusU'][:-50])
data['defocusCoverageLast50'] = self.processDefocusValues(data['defocusU'][-50:])
data['resolutionHistogram'] = self.getResolutionHistogram(data['resolution'])
else:
# Thumbnails for Micrograph Table
numMicsDone = len(self.thumbPaths[MIC_THUMBS])
self.getThumbPaths(thumbsDone=numMicsDone)
numMics = len(self.thumbPaths[MIC_PATH])
numMicsToDo = numMics - numMicsDone
if numMicsToDo <= 10:
# we have few new images, eg streaming mode, generate thumbnails now
self.generateReportImages(firstThumbIndex=numMicsDone)
else:
# we have many images, generate thumbs in a separate process
process = multiprocessing.Process(target=self.generateReportImages,
args=(numMicsDone,))
process.start()
# send over only thumbnails of the mics that have been fully processed
self.thumbsReady = self.checkNewThumbsReady()
thumbsLoading = numMics - self.thumbsReady
for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]:
if k in self.thumbPaths:
data[k] = self.thumbPaths[k][:self.thumbsReady] + ['']*thumbsLoading
data[MIC_ID] = self.thumbPaths[MIC_ID]
reportFinished = self.thumbsReady == numMics
ctfData = json.dumps(data)
# Movie gain monitor chart data
data = [] if self.movieGainMonitor is None else self.movieGainMonitor.getData()
movieGainData = json.dumps(data)
# system monitor chart data
data = self.sysMonitor.getData()
systemData = json.dumps(data)
tnow = datetime.now()
args = {'projectName': projName,
'startTime': pwutils.dateStr(project.getCreationTime(), secs=True),
'dateStr': pwutils.prettyTime(dt=tnow, secs=True),
'projectDuration': pwutils.prettyDelta(tnow-project.getCreationTime()),
'projectStatus': "FINISHED" if finished else "RUNNING",
'scipionVersion': os.environ['SCIPION_VERSION'],
'acquisitionLines': acquisitionLines,
'runLines': runLines,
'ctfData': ctfData,
'movieGainData': movieGainData,
'systemData': systemData,
'refresh': self.refreshSecs
}
self.info("Writing report html to: %s" % abspath(self.reportPath))
pwutils.cleanPath(self.reportPath)
reportFile = open(self.reportPath, 'w')
reportTemplate = reportTemplate % args
reportFile.write(reportTemplate.encode('utf-8'))
reportFile.close()
if self.publishCmd:
self.info("Publishing the report:")
cmd = self.publishCmd % {'REPORT_FOLDER': self.reportDir}
self.info(cmd)
os.system(cmd)
return reportFinished
def generate(self, finished):
reportTemplate = self.getHTMLReportText()
if not reportTemplate:
raise Exception("HTML template file '%s' not found. "
% self.template)
project = self.protocol.getProject()
reportName = 'index.html'
projName = project.getShortName()
reportDir = abspath(self.protocol._getExtraPath(projName))
self.info("Creating report directory: %s" % reportDir)
pwutils.cleanPath(reportDir)
pwutils.makePath(reportDir)
reportPath = join(reportDir, reportName)
acquisitionLines = ''
self.provider.refreshObjects()
for item in self.provider.acquisition:
if not acquisitionLines == '':
acquisitionLines += ','
acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item
runLines = ''
wasProtocol = None
for obj in self.provider.getObjects():
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
if runLines != '': runLines += ']},'
runLines += '{protocolName: "%s", output:[' % obj.name
else:
if not wasProtocol: runLines += ','
runLines += '{name: "%s", size:"%s"}' % (obj.output,
obj.outSize)
wasProtocol = isProtocol
# End the runLines JSON object
runLines += ']}'
# Ctf monitor chart data
data = [] if self.ctfMonitor is None else self.ctfMonitor.getData()
ctfData = json.dumps(data)
# system monitor chart data
data = self.sysMonitor.getData()
systemData = json.dumps(data)
args = {'projectName': projName,
'startTime': pwutils.dateStr(project.getCreationTime(), secs=True),
'dateStr': pwutils.prettyTime(secs=True),
'projectDuration': pwutils.prettyDelta(project.getElapsedTime()),
'projectStatus': "FINISHED" if finished else "RUNNING",
'scipionVersion': os.environ['SCIPION_VERSION'],
'acquisitionLines': acquisitionLines,
'runLines': runLines,
'ctfData': ctfData,
'systemData': systemData,
'refresh': '<META http-equiv="refresh" content="%s" >' % self.refreshSecs if not finished else ''
}
self.info("Writing report html to: %s" % abspath(reportPath))
reportFile = open(reportPath, 'w')
reportTemplate = reportTemplate % args
reportFile.write(reportTemplate.encode('utf-8'))
reportFile.close()
if self.publishCmd:
self.info("Publishing the report:")
cmd = self.publishCmd % {'REPORT_FOLDER': reportDir}
self.info(cmd)
os.system(cmd)
return reportPath
def generate(self, finished):
project = self.protocol.getProject()
# Project Properties Section
# Do not delete this variables. We are using them
# in an eval command
self.projectName = project.getShortName()
startTime = pwutils.dateStr(project.getCreationTime(), secs=True),
tnow = datetime.now()
_now = project.getCreationTime()
dateStr = pwutils.prettyTime(dt=tnow, secs=True),
projectDuration = pwutils.prettyDelta(tnow -
project.getCreationTime()),
projectStatus = "FINISHED" if finished else "RUNNING",
scipionVersion = os.environ['SCIPION_VERSION'],
# create structure with information related with the properties section
# the primmary key is time, projectionName, section
# if we insert two points with these three values idential
# the second updates the first
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'properties'
pointsDict['tags'] = tags
fields = {}
firstTime = self.confParser.getint("project", "properties")
fieldKeys = {'dateStr': 4, 'projectDuration': 3, 'projectStatus': 2}
fieldNames = {
'dateStr': "<b>Last Update</b>",
'projectDuration': '<b>Duration</b>',
'projectStatus': '<b>Status</b>'
}
if firstTime:
fieldKeys['startTime'] = 5
fieldKeys['scipionVersion'] = 1
fieldNames['startTime'] = '<b>Start Time</b>'
fieldNames['scipionVersion'] = '<b>Scipion Version</b>'
for metric, delta in fieldKeys.items():
localNow = _now + timedelta(seconds=delta)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
fields['metric'] = fieldNames[metric]
# str is need because all values must have the same type
fields['valueStr'] = str(eval(metric)[0])
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
self.confParser.set("project", "properties", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# acquisition section
self.provider.refreshObjects()
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'acquisition'
pointsDict['tags'] = tags
fields = {}
firstTime = self.confParser.getint("project", "acquisition")
if firstTime:
delta = 0
for metricName, value in self.provider.acquisition:
localNow = _now + timedelta(seconds=delta)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
fields['metric'] = "<b>" + metricName + "</b>"
fields['valueNum'] = float(value)
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
delta = delta - 1
# update first time only if some date has been read.
# do not place this upside the loop
self.confParser.set("project", "acquisition", "0")
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# send summary section
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'summary'
pointsDict['tags'] = tags
localNow = _now
for obj in self.provider.getObjects():
fields = {}
localNow = localNow + timedelta(seconds=-1)
pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
# If it's a protocol
isProtocol = True if obj.name else False
if isProtocol:
fields['protocolName'] = '<b>' + obj.name + '</b>'
fields['output'] = ""
fields['size'] = ""
else:
fields['protocolName'] = ""
fields['output'] = obj.output
fields['size'] = str(obj.outSize)
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
# Ctf monitor chart data
last_id = self.confParser.getint("ctf", "lastId")
listDictionaryCTF = {} if self.ctfMonitor is None else \
self.ctfMonitor.getData(lastId=last_id)
if listDictionaryCTF:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'ctf'
for ctf in listDictionaryCTF:
fields = {}
for key in ctf:
if key == 'timestamp':
pointsDict['time'] = ctf['timestamp']
# id must be a tag since tw CTF may have the same timestamp
# and time + tags must be unique
elif key == 'id':
tags['id'] = ctf['id']
# elif key == 'ctfID':
# fields['ctfID'] = "<b>" + str(ctf[key]) + "</b>"
elif key == 'shiftPlotPath':
temp = os.path.join(self.apacheImgDir,
self.projectName,
basename(ctf[key]))
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, basename(ctf[key]))
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
elif key == 'psdPath':
# convert to pǹg
baseName = basename(ctf[key])
baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
temp = os.path.join(self.apacheImgDir,
self.projectName, baseName)
temp = pwutils.replaceExt(temp, "png")
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, baseNamePng)
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
fields[key + 'LocalPng'] = pwutils.replaceExt(
ctf[key], "png")
elif key == 'micPath':
# convert to pǹg
baseName = basename(ctf[key])
baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
temp = os.path.join(self.apacheImgDir,
self.projectName, baseName)
temp = pwutils.replaceExt(temp, "png")
popUpStr = """<a href = "%s" target = "_blank"> <img src="%s" alt="%s" width="128px" height="128px"> </a>""" % \
(temp, temp, baseNamePng)
fields[key] = popUpStr
fields[key + 'Local'] = ctf[key]
fields[key + 'LocalPng'] = pwutils.replaceExt(
ctf[key], "png")
else:
fields[key] = ctf[key]
# while be use to control image creation
fields["transferImage"] = False
pointsDict['fields'] = fields
pointsDict['tags'] = tags
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("ctf", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# GAIN Section
last_id = self.confParser.getint("gain", "lastId")
listDictionaryGain = {} if self.movieGainMonitor is None else \
self.movieGainMonitor.getData(lastId=last_id)
if listDictionaryGain:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'gain'
pointsDict['tags'] = tags
counter = 1
tnow = datetime.now()
for gain in listDictionaryGain:
fields = {}
for key in gain:
fields[key] = gain[key]
pointsDict['fields'] = fields
# gain has no time information so I just
# put current time
localNow = tnow + timedelta(seconds=counter)
pointsDict[
'time'] = localNow # .strftime('%Y-%m-%dT%H:%M:%SZ')
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("gain", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
# SYSTEM data
last_id = self.confParser.getint("system", "lastId")
listDictionarySystem = {} if self.sysMonitor is None else \
self.sysMonitor.getData(lastId=last_id)
if listDictionarySystem:
pointsDict = {} # dictionary for data points
pointsDict['measurement'] = self.projectName
tags = {}
tags['section'] = 'system'
pointsDict['tags'] = tags
for system in listDictionarySystem:
fields = {}
for key in system:
if key == 'timestamp':
pointsDict['time'] = system['timestamp']
elif key == 'id':
continue
else:
fields[key] = system[key]
pointsDict['fields'] = fields
self.client.write_points([pointsDict])
last_id += 1
self.confParser.set("system", "lastId", str(last_id))
with open(self.confFileName, 'w') as confFile:
self.confParser.write(confFile)
self.transferFiles()
return last_id # reportFinished