def refreshObjects(self):
objects = []
objIds = [] # need to store ids too to avoid duplication in runs table
def addObj(objId, name, output='', size='', parent=None):
if objId not in objIds:
obj = pwobj.Object(objId=objId)
obj.name = name
obj.output = output
obj.outSize = size
obj._objParent = parent
objIds.append(objId)
objects.append(obj)
return obj
else:
return None
prots = [getUpdatedProtocol(p) for p in self.protocol.getInputProtocols()]
for prot in prots:
pobj = addObj(prot.getObjId(),
'%s (id=%s)' % (prot.getRunName(), prot.strId()))
for outName, outSet in prot.iterOutputAttributes(pwobj.Set):
outSet.load()
outSet.loadAllProperties()
# outSetId needs to be compound id to avoid duplicate ids
outSetId = '%s.%s' % (outSet.getObjId(), prot.getObjId())
addObj(outSetId, '', outName, outSet.getSize(), pobj)
outSet.close()
# Store acquisition parameters in case of the import protocol
from pyworkflow.em import ProtImportImages
# NOTE by Yaiza: we force the string containing the Å to be unicode
# because this is the encoding used when generating report in report_html.py
if isinstance(prot, ProtImportImages):
self.acquisition = [("Microscope Voltage (kV): ",
prot.voltage.get()),
("Spherical aberration (mm): ",
prot.sphericalAberration.get()),
("Magnification: ",
prot.magnification.get()),
(u"Pixel Size (Å/px): ",
round(outSet.getSamplingRate(),2))
]
if prot.dosePerFrame.get() is not None:
self.acquisition.append((u"Dose per frame (e/Ų):",
prot.dosePerFrame.get()))
self._objects = objects
def refreshObjects(self):
objects = []
objIds = [] # need to store ids too to avoid duplication in runs table
def addObj(objId, name, output='', size='', parent=None):
if objId not in objIds:
obj = pwobj.Object(objId=objId)
obj.name = name
obj.output = output
obj.outSize = size
obj._objParent = parent
objIds.append(objId)
objects.append(obj)
return obj
else:
return None
prots = [getUpdatedProtocol(p) for p in self.protocol.getInputProtocols()]
for prot in prots:
pobj = addObj(prot.getObjId(),
'%s (id=%s)' % (prot.getRunName(), prot.strId()))
for outName, outSet in prot.iterOutputAttributes(pwobj.Set):
outSet.load()
outSet.loadAllProperties()
# outSetId needs to be compound id to avoid duplicate ids
outSetId = '%s.%s' % (outSet.getObjId(), prot.getObjId())
addObj(outSetId, '', outName, outSet.getSize(), pobj)
outSet.close()
# Store acquisition parameters in case of the import protocol
from pyworkflow.em import ProtImportImages
# NOTE by Yaiza: we force the string containing the Å to be unicode
# because this is the encoding used when generating report in report_html.py
if isinstance(prot, ProtImportImages):
self.acquisition = [("Microscope Voltage (kV): ",
prot.voltage.get()),
("Spherical aberration (mm): ",
prot.sphericalAberration.get()),
("Magnification: ",
prot.magnification.get()),
(u"Pixel Size (Å/px): ",
round(outSet.getSamplingRate(),2))
]
self._objects = objects
def getThumbPaths(self,
thumbsDone=0,
ctfData=None,
ext='png',
micIdSet=None):
"""Adds to self.thumbPaths the paths to the report thumbnails
that come from the alignment and/or ctf protocol.
===== Params =====
- thumbsDone: how many thumbnails have already been generated.
we will get paths starting from this index
- ctfData: dict resulting from ctfMonitor.getData()
- ext: extension of the thumbnail images. Defaults to png.
- micIdSet: mic indexes to use
"""
# get psd thumbs from ctfData
if ctfData is not None:
for i in range(thumbsDone, len(ctfData[PSD_PATH])):
psdPath = ctfData[PSD_PATH][i]
movie = basename(os.path.dirname(psdPath))
psdThumb = join(
PSD_THUMBS, "%s_%s" %
(movie, pwutils.replaceExt(basename(psdPath), ext)))
self.thumbPaths[PSD_THUMBS].append(psdThumb)
self.thumbPaths[PSD_PATH].append(psdPath)
# get alignment and mic thumbs
if self.alignProtocol is not None:
getMicFromCTF = False
updatedProt = getUpdatedProtocol(self.alignProtocol)
if hasattr(updatedProt, 'outputMicrographs'):
outputSet = updatedProt.outputMicrographs
if micIdSet is None:
micIdSet = list(outputSet.getIdSet())
else:
return
elif self.ctfProtocol is not None:
getMicFromCTF = True
updatedProt = getUpdatedProtocol(self.ctfProtocol)
if hasattr(updatedProt, 'outputCTF'):
outputSet = updatedProt.outputCTF
if micIdSet is None:
micIdSet = list(outputSet.getIdSet())
else:
return
else:
return
for micId in micIdSet[thumbsDone:]:
mic = outputSet[micId]
if getMicFromCTF:
mic = mic.getMicrograph()
if hasattr(mic, 'thumbnail'):
srcMicFn = abspath(mic.thumbnail.getFileName())
else:
srcMicFn = abspath(mic.getFileName())
micThumbFn = join(MIC_THUMBS,
pwutils.replaceExt(basename(srcMicFn), ext))
self.thumbPaths[MIC_PATH].append(srcMicFn)
self.thumbPaths[MIC_THUMBS].append(micThumbFn)
shiftPlot = (getattr(mic, 'plotCart', None)
or getattr(mic, 'plotGlobal', None))
if shiftPlot is not None:
shiftPath = "" if shiftPlot is None else abspath(
shiftPlot.getFileName())
shiftCopy = "" if shiftPlot is None else join(
SHIFT_THUMBS, pwutils.replaceExt(basename(shiftPath), ext))
self.thumbPaths[SHIFT_PATH].append(shiftPath)
self.thumbPaths[SHIFT_THUMBS].append(shiftCopy)
else:
if SHIFT_PATH in self.thumbPaths:
self.thumbPaths.pop(SHIFT_PATH, None)
if SHIFT_THUMBS in self.thumbPaths:
self.thumbPaths.pop(SHIFT_THUMBS, None)
self.thumbPaths[MIC_ID].append(micId)
if self.ctfProtocol is None:
psdPath = mic.psdJpeg.getFileName() if hasattr(
mic, 'psdJpeg') else mic.psdCorr.getFileName()
psdThumb = join(PSD_THUMBS,
pwutils.replaceExt(basename(psdPath), ext))
self.thumbPaths[PSD_THUMBS].append(psdThumb)
self.thumbPaths[PSD_PATH].append(psdPath)
def step(self):
prot = getUpdatedProtocol(self.protocol)
# Create set of processed CTF from CTF protocol
if hasattr(prot, 'outputCTF'):
CTFset = prot.outputCTF.getIdSet()
else:
return False
# find difference
sys.stdout.flush()
diffSet = CTFset - self.readCTFs
setOfCTFs = prot.outputCTF
astigmatism = self.astigmatism
for ctfID in diffSet:
ctf = setOfCTFs[ctfID]
defocusU = ctf.getDefocusU()
defocusV = ctf.getDefocusV()
defocusAngle = ctf.getDefocusAngle()
if defocusAngle > 360 or defocusAngle< -360:
defocusAngle = 0
astig = abs(defocusU - defocusV)
resolution = ctf.getResolution()
if isinf(resolution):
resolution = 0.
fitQuality = ctf.getFitQuality()
if fitQuality is None or isinf(fitQuality):
fitQuality = 0.
psdPath = os.path.abspath(ctf.getPsdFile())
micPath = os.path.abspath(ctf.getMicrograph().getFileName())
shiftPlot = (getattr(ctf.getMicrograph(), 'plotCart', None)
or getattr(ctf.getMicrograph(), 'plotGlobal', None))
if shiftPlot is not None:
shiftPlotPath = os.path.abspath(shiftPlot.getFileName())
else:
shiftPlotPath = ""
if defocusU < defocusV:
aux = defocusV
defocusV = defocusU
defocusU = aux
# TODO: check if this is always true
defocusAngle = 180. - defocusAngle
print("ERROR: defocusU should be greater than defocusV")
# get CTFs with this ids a fill table
# do not forget to compute astigmatism
# sql = """INSERT INTO %s(defocusU,defocusV,astigmatism,ratio,psdPath)
# VALUES(%f,%f,%f,%f,"%s");""" % (self._tableName, defocusU,
# defocusV, defocusAngle, defocusU / defocusV, psdPath)
sql = """INSERT INTO %s(ctfID, defocusU,defocusV,astigmatism,ratio, resolution, fitQuality, micPath,psdPath,shiftPlotPath )
VALUES(%d,%f,%f,%f,%f,%f,%f,"%s","%s","%s");""" % (self._tableName, ctfID, defocusU,
defocusV, astig, defocusU / defocusV, resolution, fitQuality, micPath, psdPath, shiftPlotPath)
try:
self.cur.execute(sql)
except Exception as e:
print("ERROR: saving one data point (CTF monitor). I continue")
print e
if abs(defocusU - defocusV) > astigmatism:
self.warning("Astigmatism (defocusU - defocusV) = %f."
% abs(defocusU - defocusV))
if defocusU > self.maxDefocus:
self.warning("DefocusU (%f) is larger than defocus "
"maximum (%f)" % (defocusU, self.maxDefocus))
self.maxDefocus = defocusU
if defocusV < self.minDefocus:
self.warning("DefocusV (%f) is smaller than defocus "
"minumum (%f)" % (defocusV, self.maxDefocus))
self.minDefocus = defocusV
self.readCTFs.update(diffSet)
# Finish when protocol is not longer running
return prot.getStatus() != STATUS_RUNNING
def step(self):
self.info("MonitorISPyB: only one step")
prots = [getUpdatedProtocol(p) for p in self.inputProtocols]
finished = [] # store True if protocol not running
updateImageIds = [] # Store obj ids that have changes
updateAlignIds = [] # Store obj ids that have changes
updateCTFIds = [] # Store obj ids that have changes
for prot in prots:
self.info("protocol: %s" % prot.getRunName())
if isinstance(prot, ProtImportMovies) and hasattr(prot, 'outputMovies'):
self.create_movie_params(prot, updateImageIds)
elif isinstance(prot, ProtAlignMovies) and hasattr(prot, 'outputMicrographs'):
self.update_align_params(prot, updateAlignIds)
elif isinstance(prot, ProtCTFMicrographs) and hasattr(prot, 'outputCTF'):
self.update_ctf_params(prot, updateCTFIds)
finished.append(prot.getStatus() != STATUS_RUNNING)
dcParams = self.ispybDb.get_data_collection_params()
self.safe_update(dcParams, self.dataCollection)
if self.dcId:
self.safe_update(dcParams, self.previousParams)
dcParams['id'] = self.dcId
dcParams['endtime'] = self.now()
self.info("writing datacollection: %s" + str(dcParams))
self.ispybDb.update_data_collection(dcParams)
else:
dcParams['starttime'] = self.now()
dcParams['endtime'] = self.now()
self.info("writing datacollection: %s" + str(dcParams))
self.dcId = self.ispybDb.update_data_collection(dcParams)
self.previousParams = dcParams
for imageId in set(updateImageIds):
movieParams = self.ispybDb.get_movie_params()
self.movies[imageId]['dataCollectionId'] = self.dcId
if 'movieNumber' not in self.movies[imageId]:
self.movies[imageId]['movieNumber'] = self.imageNumber
self.imageNumber += 1
self.safe_update(movieParams, self.movies[imageId])
self.movies[imageId]['movieId'] = self.ispybDb.update_movie(movieParams)
for itemId in set(updateAlignIds):
if 'autoProcProgramId' not in self.motion_corrections[itemId]:
program = self.ispybDb.get_program_params()
self.safe_update(program, self.motion_corrections[itemId])
program['starttime'] = self.now()
program_id = self.ispybDb.update_program(program)
self.motion_corrections[itemId]['autoProcProgramId'] = program_id
motionParams = self.ispybDb.get_motion_correction_params()
self.safe_update(motionParams, self.motion_corrections[itemId])
motionParams['movieId'] = self.movies[itemId]['movieId']
self.info("writing motion correction: %s" + str(motionParams))
motionCorrectionId = self.ispybDb.update_motion_correction(motionParams)
self.info("wrote motion correction: %s" + str(motionCorrectionId))
self.motion_corrections[itemId]['motionCorrectionId'] = motionCorrectionId
drift = self.motion_correction_drift[itemId]
xs = drift['xs']
ys = drift['ys']
for (i, (x, y)) in enumerate(zip(xs, ys)):
drift_params = self.ispybDb.get_motion_correction_drift_params()
drift_params['motionCorrectionId'] = self.motion_corrections[itemId]['motionCorrectionId']
drift_params['frameNumber'] = i + 1
drift_params['deltaX'] = x
drift_params['deltaY'] = y
self.ispybDb.update_motion_correction_drift(drift_params)
for itemId in set(updateCTFIds):
if 'autoProcProgramId' not in self.ctfs[itemId]:
program = self.ispybDb.get_program_params()
self.safe_update(program, self.ctfs[itemId])
program['starttime'] = self.now()
program_id = self.ispybDb.update_program(program)
self.ctfs[itemId]['autoProcProgramId'] = program_id
ctfParams = self.ispybDb.get_ctf_params()
self.safe_update(ctfParams, self.ctfs[itemId])
ctfParams['motionCorrectionId'] = self.motion_corrections[itemId]['motionCorrectionId']
self.info("writing ctf: %s" + str(ctfParams))
ctfId = self.ispybDb.update_ctf(ctfParams)
self.info("wrote ctf: %s" + str(ctfId))
self.ctfs[itemId]['ctfId'] = ctfId
if all(finished):
self.info("All finished, closing ISPyBDb connection")
self.ispybDb.disconnect()
return all(finished)
def step(self):
prot = getUpdatedProtocol(self.protocol)
# Create set of processed CTF from CTF protocol
if hasattr(prot, 'outputCTF'):
CTFset = prot.outputCTF.getIdSet()
else:
return False
# find difference
sys.stdout.flush()
diffSet = CTFset - self.readCTFs
setOfCTFs = prot.outputCTF
astigmatism = self.astigmatism
for ctfID in diffSet:
ctf = setOfCTFs[ctfID]
defocusU = ctf.getDefocusU()
defocusV = ctf.getDefocusV()
defocusAngle = ctf.getDefocusAngle()
if defocusAngle > 360 or defocusAngle < -360:
defocusAngle = 0
astig = abs(defocusU - defocusV)
resolution = ctf.getResolution()
if isinf(resolution):
resolution = 0.
fitQuality = ctf.getFitQuality()
if fitQuality is None or isinf(fitQuality):
fitQuality = 0.
psdPath = os.path.abspath(ctf.getPsdFile())
micPath = os.path.abspath(ctf.getMicrograph().getFileName())
shiftPlot = (getattr(ctf.getMicrograph(), 'plotCart', None)
or getattr(ctf.getMicrograph(), 'plotGlobal', None))
if shiftPlot is not None:
shiftPlotPath = os.path.abspath(shiftPlot.getFileName())
else:
shiftPlotPath = ""
if defocusU < defocusV:
aux = defocusV
defocusV = defocusU
defocusU = aux
# TODO: check if this is always true
defocusAngle = 180. - defocusAngle
print("ERROR: defocusU should be greater than defocusV")
# get CTFs with this ids a fill table
# do not forget to compute astigmatism
# sql = """INSERT INTO %s(defocusU,defocusV,astigmatism,ratio,psdPath)
# VALUES(%f,%f,%f,%f,"%s");""" % (self._tableName, defocusU,
# defocusV, defocusAngle, defocusU / defocusV, psdPath)
sql = """INSERT INTO %s(ctfID, defocusU,defocusV,astigmatism,ratio, resolution, fitQuality, micPath,psdPath,shiftPlotPath )
VALUES(%d,%f,%f,%f,%f,%f,%f,"%s","%s","%s");""" % (
self._tableName, ctfID, defocusU, defocusV, astig,
defocusU / defocusV, resolution, fitQuality, micPath, psdPath,
shiftPlotPath)
try:
self.cur.execute(sql)
except Exception as e:
print("ERROR: saving one data point (CTF monitor). I continue")
print e
if abs(defocusU - defocusV) > astigmatism:
self.warning("Astigmatism (defocusU - defocusV) = %f." %
abs(defocusU - defocusV))
if defocusU > self.maxDefocus:
self.warning("DefocusU (%f) is larger than defocus "
"maximum (%f)" % (defocusU, self.maxDefocus))
self.maxDefocus = defocusU
if defocusV < self.minDefocus:
self.warning("DefocusV (%f) is smaller than defocus "
"minumum (%f)" % (defocusV, self.maxDefocus))
self.minDefocus = defocusV
self.readCTFs.update(diffSet)
# Finish when protocol is not longer running
return prot.getStatus() != STATUS_RUNNING
def step(self):
valuesDict = {}
valuesDict['table'] = self._tableName
cpu = valuesDict['cpu'] = psutil.cpu_percent(interval=0)
mem = valuesDict['mem'] = psutil.virtual_memory().percent
swap = valuesDict['swap'] = psutil.swap_memory().percent
# some code examples:
# https://github.com/ngi644/datadog_nvml/blob/master/nvml.py
if self.doGpu:
for i in self.gpusToUse:
try:
handle = nvmlDeviceGetHandleByIndex(i)
memInfo = nvmlDeviceGetMemoryInfo(handle)
valuesDict["gpuMem_%d" % i] = \
float(memInfo.used)*100./float(memInfo.total)
util = nvmlDeviceGetUtilizationRates(handle)
valuesDict["gpuUse_%d" % i] = util.gpu
temp = nvmlDeviceGetTemperature(handle,
NVML_TEMPERATURE_GPU)
valuesDict["gpuTem_%d" % i] = temp
except NVMLError as err:
handle = nvmlDeviceGetHandleByIndex(i)
msg = "Device %d -> %s not suported\n" \
"Remove device %d from FORM" % \
(i, nvmlDeviceGetName(handle), i)
errorWindow(None, msg)
if self.doNetwork:
try:
# measure a sort interval
pnic_before = psutil.net_io_counters(pernic=True)[self.nif]
time.sleep(self.samplingTime) # sec
pnic_after = psutil.net_io_counters(pernic=True)[self.nif]
bytes_sent = pnic_after.bytes_sent - pnic_before.bytes_sent
bytes_recv = pnic_after.bytes_recv - pnic_before.bytes_recv
valuesDict["%s_send" % self.nif] = \
bytes_sent * self.samplingTime / 1048576
valuesDict["%s_recv" % self.nif] = \
bytes_recv * self.samplingTime / 1048576
except:
msg = "cannot get information of network interface %s" % \
self.nif
if self.doDiskIO:
try:
# measure a sort interval
disk_before = psutil.disk_io_counters(perdisk=False)
time.sleep(self.samplingTime) # sec
disk_after = psutil.disk_io_counters(perdisk=False)
bytes_read = disk_after.read_bytes - disk_before.read_bytes
bytes_write = disk_after.write_bytes - disk_before.write_bytes
valuesDict["disk_read"] = \
self.samplingTime * bytes_read / self.mega
valuesDict["disk_write"] = \
self.samplingTime * bytes_write / self.mega
except:
msg = "cannot get information of disk usage "
if self.cpuAlert < 100 and cpu > self.cpuAlert:
self.warning("CPU allocation =%f." % cpu)
self.cpuAlert = cpu
if self.memAlert < 100 and mem.percent > self.memAlert:
self.warning("Memory allocation =%f." % mem)
self.memAlert = mem
if self.swapAlert < 100 and swap.percent > self.swapAlert:
self.warning("SWAP allocation =%f." % swap)
self.swapAlert = swap
sqlCommand = "INSERT INTO %(table)s ("
for label in self.labelList:
sqlCommand += "%s, " % label
# remove last comma
sqlCommand = sqlCommand[:-2]
sqlCommand += ") VALUES("
for label in self.labelList:
sqlCommand += "%" + "(%s)f, " % label
# remove last comma
sqlCommand = sqlCommand[:-2]
sqlCommand += ");"
sql = sqlCommand % valuesDict
try:
self.cur.execute(sql)
except Exception as e:
print("ERROR: saving one data point (monitor). I continue")
# Return finished = True if all protocols have finished
finished = []
for prot in self.protocols:
updatedProt = getUpdatedProtocol(prot)
finished.append(updatedProt.getStatus() != STATUS_RUNNING)
return all(finished)
def getThumbPaths(self, thumbsDone=0, ctfData=None, ext='png', micIdSet=None):
"""Adds to self.thumbPaths the paths to the report thumbnails
that come from the alignment and/or ctf protocol.
===== Params =====
- thumbsDone: how many thumbnails have already been generated.
we will get paths starting from this index
- ctfData: dict resulting from ctfMonitor.getData()
- ext: extension of the thumbnail images. Defaults to png.
- micIdSet: mic indexes to use
"""
# get psd thumbs from ctfData
if ctfData is not None:
for i in range(thumbsDone, len(ctfData[PSD_PATH])):
psdPath = ctfData[PSD_PATH][i]
movie = basename(os.path.dirname(psdPath))
psdThumb = join(PSD_THUMBS, "%s_%s" % (movie, pwutils.replaceExt(basename(psdPath), ext)))
self.thumbPaths[PSD_THUMBS].append(psdThumb)
self.thumbPaths[PSD_PATH].append(psdPath)
# get alignment and mic thumbs
if self.alignProtocol is not None:
getMicFromCTF = False
updatedProt = getUpdatedProtocol(self.alignProtocol)
if hasattr(updatedProt, 'outputMicrographs'):
outputSet = updatedProt.outputMicrographs
if micIdSet is None:
micIdSet = list(outputSet.getIdSet())
else:
return
elif self.ctfProtocol is not None:
getMicFromCTF = True
updatedProt = getUpdatedProtocol(self.ctfProtocol)
if hasattr(updatedProt, 'outputCTF'):
outputSet = updatedProt.outputCTF
if micIdSet is None:
micIdSet = list(outputSet.getIdSet())
else:
return
else:
return
for micId in micIdSet[thumbsDone:]:
mic = outputSet[micId]
if getMicFromCTF:
mic = mic.getMicrograph()
if hasattr(mic, 'thumbnail'):
srcMicFn = abspath(mic.thumbnail.getFileName())
else:
srcMicFn = abspath(mic.getFileName())
micThumbFn = join(MIC_THUMBS, pwutils.replaceExt(basename(srcMicFn), ext))
self.thumbPaths[MIC_PATH].append(srcMicFn)
self.thumbPaths[MIC_THUMBS].append(micThumbFn)
shiftPlot = (getattr(mic, 'plotCart', None) or getattr(mic, 'plotGlobal', None))
if shiftPlot is not None:
shiftPath = "" if shiftPlot is None else abspath(shiftPlot.getFileName())
shiftCopy = "" if shiftPlot is None else join(SHIFT_THUMBS, pwutils.replaceExt(basename(shiftPath), ext))
self.thumbPaths[SHIFT_PATH].append(shiftPath)
self.thumbPaths[SHIFT_THUMBS].append(shiftCopy)
else:
if SHIFT_PATH in self.thumbPaths:
self.thumbPaths.pop(SHIFT_PATH, None)
if SHIFT_THUMBS in self.thumbPaths:
self.thumbPaths.pop(SHIFT_THUMBS, None)
self.thumbPaths[MIC_ID].append(micId)
if self.ctfProtocol is None:
def getMicPSDPath(mic):
if hasattr(mic, 'psdJpeg'):
return mic.psdJpeg.getFileName()
elif hasattr(mic,'psdCorr'):
return mic.psdCorr.getFileName()
else:
return None
psdPath = getMicPSDPath(mic)
psdThumb = None
if psdPath ==None:
psdThumb = join(PSD_THUMBS, pwutils.replaceExt(basename(psdPath), ext))
self.thumbPaths[PSD_THUMBS].append(psdThumb)
self.thumbPaths[PSD_PATH].append(psdPath)
else:
if PSD_THUMBS in self.thumbPaths:
self.thumbPaths.pop(PSD_THUMBS, None)
if PSD_PATH in self.thumbPaths:
self.thumbPaths.pop(PSD_PATH, None)
def step(self):
valuesDict = {}
valuesDict['table'] = self._tableName
cpu = valuesDict['cpu'] = psutil.cpu_percent(interval=0)
mem = valuesDict['mem'] = psutil.virtual_memory().percent
swap = valuesDict['swap'] = psutil.swap_memory().percent
# some code examples:
# https://github.com/ngi644/datadog_nvml/blob/master/nvml.py
if self.doGpu:
for i in self.gpusToUse:
try:
handle = nvmlDeviceGetHandleByIndex(i)
memInfo = nvmlDeviceGetMemoryInfo(handle)
valuesDict["gpuMem_%d" % i] = \
float(memInfo.used)*100./float(memInfo.total)
util = nvmlDeviceGetUtilizationRates(handle)
valuesDict["gpuUse_%d" % i] = util.gpu
temp = nvmlDeviceGetTemperature(handle,
NVML_TEMPERATURE_GPU)
valuesDict["gpuTem_%d" % i] = temp
except NVMLError as err:
handle = nvmlDeviceGetHandleByIndex(i)
msg = "Device %d -> %s not suported\n" \
"Remove device %d from FORM" % \
(i, nvmlDeviceGetName(handle), i)
errorWindow(None, msg)
if self.doNetwork:
try:
# measure a sort interval
pnic_before = psutil.net_io_counters(pernic=True)[self.nif]
time.sleep(self.samplingTime) # sec
pnic_after = psutil.net_io_counters(pernic=True)[self.nif]
bytes_sent = pnic_after.bytes_sent - pnic_before.bytes_sent
bytes_recv = pnic_after.bytes_recv - pnic_before.bytes_recv
valuesDict["%s_send" % self.nif] = \
bytes_sent * self.samplingTime / 1048576
valuesDict["%s_recv" % self.nif] = \
bytes_recv * self.samplingTime / 1048576
except:
msg = "cannot get information of network interface %s" % \
self.nif
if self.doDiskIO:
try:
# measure a sort interval
disk_before = psutil.disk_io_counters(perdisk=False)
time.sleep(self.samplingTime) # sec
disk_after = psutil.disk_io_counters(perdisk=False)
bytes_read = disk_after.read_bytes - disk_before.read_bytes
bytes_write = disk_after.write_bytes - disk_before.write_bytes
valuesDict["disk_read"] = \
self.samplingTime * bytes_read / self.mega
valuesDict["disk_write"] = \
self.samplingTime * bytes_write / self.mega
except:
msg = "cannot get information of disk usage "
if self.cpuAlert < 100 and cpu > self.cpuAlert:
self.warning("CPU allocation =%f." % cpu)
self.cpuAlert = cpu
if self.memAlert < 100 and mem.percent > self.memAlert:
self.warning("Memory allocation =%f." % mem)
self.memAlert = mem
if self.swapAlert < 100 and swap.percent > self.swapAlert:
self.warning("SWAP allocation =%f." % swap)
self.swapAlert = swap
sqlCommand = "INSERT INTO %(table)s ("
for label in self.labelList:
sqlCommand += "%s, " % label
# remove last comma
sqlCommand = sqlCommand[:-2]
sqlCommand += ") VALUES("
for label in self.labelList:
sqlCommand += "%"+"(%s)f, " % label
# remove last comma
sqlCommand = sqlCommand[:-2]
sqlCommand += ");"
sql = sqlCommand % valuesDict
try:
self.cur.execute(sql)
except Exception as e:
print("ERROR: saving one data point (monitor). I continue")
# Return finished = True if all protocols have finished
finished = []
for prot in self.protocols:
updatedProt = getUpdatedProtocol(prot)
finished.append(updatedProt.getStatus() != STATUS_RUNNING)
return all(finished)
