def getFinishedJobs(self):
"""
Compare last and latest job list, pick up the new DONE/EXIT jobs on latest sampling.
"""
if len(self.specifiedJobs) > 0:
self.finishedJobList = self.specifiedJobs
else:
self.finishedJobList = []
lastJobList = self.getLastJobList()
latestJobList = self.getLatestJobList()
self.finishedJobList = [
job for job in lastJobList if job not in latestJobList
]
finishedJobString = ' '.join(self.finishedJobList)
self.finishedJobDic = openlava_common.getBjobsUfInfo(
'bjobs -UF ' + str(finishedJobString))
if len(self.specifiedJobs) == 0:
for job in self.finishedJobList:
if job in self.finishedJobDic:
jobStatus = self.finishedJobDic[job]['status']
if (jobStatus != 'DONE') and (jobStatus != 'EXIT'):
self.finishedJobList.remove(job)
def checkJob(self):
"""
Get job information with "bjobs -UF <jobId>", save the infomation into dict self.jobInfoDic.
Update self.jobTabFrame1 and self.jobTabFrame3.
"""
self.currentJob = self.jobTabJobLine.text().strip()
print('* Checking job "' + str(self.currentJob) + '".')
# Initicalization
self.updateJobTabFrame1(init=True)
self.updateJobTabFrame2(init=True)
self.updateJobTabFrame3(init=True)
# Job name must be a string of numbers.
if not re.match('^[0-9]+$', self.currentJob):
warningMessage = '*Warning*: No valid job is specified!'
self.guiWarning(warningMessage)
return
# Get job info
print('Getting job information for job "' + str(self.currentJob) +
'".')
self.jobInfoDic = openlava_common.getBjobsUfInfo(command='bjobs -UF ' +
str(self.currentJob))
# Update the related frames with the job info.
self.updateJobTabFrame1()
self.updateJobTabFrame2()
self.updateJobTabFrame3()
def getJobPidListDic(self, pidsDic):
print('>>> Getting openlava job related pids ...')
bjobsUfDic = openlava_common.getBjobsUfInfo(command='bjobs -u all -r -m ' + str(hostname) + ' -UF')
jobPidListDic = {}
for job in bjobsUfDic.keys():
if ('pids' in bjobsUfDic[job]) and (len(bjobsUfDic[job]['pids']) > 0):
jobPidListDic[job] = [int(x) for x in bjobsUfDic[job]['pids']]
else:
jobPidList = self.getJobPidListDicFromProcessTree(pidsDic, job)
if len(jobPidList) > 0:
jobPidListDic[job] = self.getJobPidListDicFromProcessTree(pidsDic, job)
return(jobPidListDic)
def sampleJobInfo(self):
"""
Sample job info, especially the memory usage info.
"""
self.getDateInfo()
print('>>> Sampling job info ...')
bjobsDic = openlava_common.getBjobsUfInfo()
jobList = list(bjobsDic.keys())
jobRangeDic = common.getJobRangeDic(jobList)
jobSqlDic = {}
keyList = ['sampleTime', 'mem']
for jobRange in jobRangeDic.keys():
jobDbFile = str(self.dbPath) + '/job/' + str(jobRange) + '.db'
(result, jobDbConn) = sqlite3_common.connectDbFile(jobDbFile,
mode='read')
if result == 'passed':
jobTableList = sqlite3_common.getSqlTableList(
jobDbFile, jobDbConn)
else:
jobTableList = []
for job in jobRangeDic[jobRange]:
jobTableName = 'job_' + str(job)
print(' Sampling for job "' + str(job) + '" ...')
jobSqlDic[job] = {
'drop': False,
'keyString': '',
'valueString': '',
}
# If job table (with old data) has been on the jobDbFile, drop it.
if jobTableName in jobTableList:
dataDic = sqlite3_common.getSqlTableData(
jobDbFile, jobDbConn, jobTableName, ['sampleTime'])
if dataDic:
if len(dataDic['sampleTime']) > 0:
lastSampleTime = dataDic['sampleTime'][-1]
lastSeconds = int(
time.mktime(
datetime.datetime.strptime(
str(lastSampleTime),
"%Y%m%d_%H%M%S").timetuple()))
if self.currentSeconds - lastSeconds > 3600:
common.printWarning(
' *Warning*: table "' +
str(jobTableName) +
'" already existed even one hour ago, will drop it.'
)
jobSqlDic[job]['drop'] = True
jobTableList.remove(jobTableName)
# If job table is not on the jobDbFile, create it.
if jobTableName not in jobTableList:
keyString = sqlite3_common.genSqlTableKeyString(keyList)
jobSqlDic[job]['keyString'] = keyString
# Insert sql table value.
valueList = [self.sampleTime, bjobsDic[job]['mem']]
valueString = sqlite3_common.genSqlTableValueString(valueList)
jobSqlDic[job]['valueString'] = valueString
if result == 'passed':
jobDbConn.commit()
jobDbConn.close()
for jobRange in jobRangeDic.keys():
jobDbFile = str(self.dbPath) + '/job/' + str(jobRange) + '.db'
(result, jobDbConn) = sqlite3_common.connectDbFile(jobDbFile,
mode='write')
if result != 'passed':
return
for job in jobRangeDic[jobRange]:
jobTableName = 'job_' + str(job)
if jobSqlDic[job]['drop']:
sqlite3_common.dropSqlTable(jobDbFile,
jobDbConn,
jobTableName,
commit=False)
if jobSqlDic[job]['keyString'] != '':
sqlite3_common.createSqlTable(jobDbFile,
jobDbConn,
jobTableName,
jobSqlDic[job]['keyString'],
commit=False)
if jobSqlDic[job]['valueString'] != '':
sqlite3_common.insertIntoSqlTable(
jobDbFile,
jobDbConn,
jobTableName,
jobSqlDic[job]['valueString'],
commit=False)
jobDbConn.commit()
jobDbConn.close()
print(' Committing the update to sqlite3 ...')
print(' Done (' + str(len(jobList)) + ' jobs).')
def genJobsTabTable(self):
self.jobsTabTable.setShowGrid(True)
self.jobsTabTable.setSortingEnabled(True)
self.jobsTabTable.setColumnCount(12)
self.jobsTabTable.setHorizontalHeaderLabels([
'Job', 'User', 'Status', 'Queue', 'Host', 'Started', 'Project',
'Processers', 'cpuTime', 'Rusage (G)', 'Mem (G)', 'Command'
])
command = 'bjobs -UF '
user = self.jobsTabUserLine.text().strip()
if re.match('^\s*$', user):
command = str(command) + ' -u all'
else:
command = str(command) + ' -u ' + str(user)
queue = self.jobsTabQueueCombo.currentText().strip()
if queue != 'ALL':
command = str(command) + ' -q ' + str(queue)
status = self.jobsTabStatusCombo.currentText().strip()
if status == 'RUN':
command = str(command) + ' -r'
elif status == 'PEND':
command = str(command) + ' -p'
elif status == 'ALL':
command = str(command) + ' -a'
startedOn = self.jobsTabStartedOnCombo.currentText().strip()
if startedOn != 'ALL':
command = str(command) + ' -m ' + str(startedOn)
jobDic = openlava_common.getBjobsUfInfo(command)
self.jobsTabTable.setRowCount(len(jobDic.keys()))
jobs = list(jobDic.keys())
for i in range(len(jobs)):
job = jobs[i]
j = 0
self.jobsTabTable.setItem(i, j, QTableWidgetItem(job))
j = j + 1
item = QTableWidgetItem()
item.setText(jobDic[job]['user'])
self.jobsTabTable.setItem(i, j, item)
j = j + 1
item = QTableWidgetItem()
item.setText(jobDic[job]['status'])
self.jobsTabTable.setItem(i, j, item)
j = j + 1
item = QTableWidgetItem()
item.setText(jobDic[job]['queue'])
self.jobsTabTable.setItem(i, j, item)
j = j + 1
item = QTableWidgetItem()
item.setText(jobDic[job]['startedOn'])
self.jobsTabTable.setItem(i, j, item)
j = j + 1
item = QTableWidgetItem()
item.setText(jobDic[job]['startedTime'])
self.jobsTabTable.setItem(i, j, item)
j = j + 1
if str(jobDic[job]['project']) != '':
item = QTableWidgetItem()
item.setData(Qt.DisplayRole, jobDic[job]['project'])
self.jobsTabTable.setItem(i, j, item)
j = j + 1
if str(jobDic[job]['processorsRequested']) != '':
item = QTableWidgetItem()
item.setData(Qt.DisplayRole,
int(jobDic[job]['processorsRequested']))
self.jobsTabTable.setItem(i, j, item)
j = j + 1
if str(jobDic[job]['cpuTime']) != '':
item = QTableWidgetItem()
item.setData(Qt.DisplayRole, int(jobDic[job]['cpuTime']))
self.jobsTabTable.setItem(i, j, item)
j = j + 1
if str(jobDic[job]['rusageMem']) != '':
item = QTableWidgetItem()
rusageMemValue = int(jobDic[job]['rusageMem']) / 1024
item.setData(Qt.DisplayRole, int(rusageMemValue))
self.jobsTabTable.setItem(i, j, item)
j = j + 1
if str(jobDic[job]['mem']) != '':
item = QTableWidgetItem()
memValue = int(jobDic[job]['mem']) / 1024
item.setData(Qt.DisplayRole, int(memValue))
self.jobsTabTable.setItem(i, j, item)
j = j + 1
item = QTableWidgetItem()
item.setText(jobDic[job]['command'])
self.jobsTabTable.setItem(i, j, item)