def jobs(self, *args, **kw):
result = '<body>'
result += str(CPProgressBar(0, min(100, self.counter), 100, 300))
if 'job' in kw:
jobNum = int(kw['job'])
info = self.task.getJobConfig(jobNum)
result += str(TabularHTML(zip(sorted(info), sorted(info)), [info], top = False))
def getJobObjs():
for jobNum in self.jobMgr.jobDB.getJobs():
result = self.jobMgr.jobDB.get(jobNum).__dict__
result['jobNum'] = jobNum
result.update(result['dict'])
yield result
fmtTime = lambda t: time.strftime('%Y-%m-%d %T', time.localtime(t))
result += str(TabularHTML([
('jobNum', 'Job'), ('state', 'Status'), ('attempt', 'Attempt'),
('wmsId', 'WMS ID'), ('dest', 'Destination'), ('submitted', 'Submitted')
], getJobObjs(),
fmt = {
'jobNum': lambda x: '<a href="jobs?job=%s">%s</a>' % (x, x),
'state': lambda s: Job.enum2str(s),
'submitted': fmtTime
}, top = True))
result += '</body>'
return result
def _update_job_states(self, job_db, jobnum_list):
states = {}
for state_name in Job.enum_name_list:
states[state_name] = 0
for jobnum in jobnum_list:
states[Job.enum2str(job_db.get_job_transient(jobnum).state)] += 1
self._task_info['states'] = states
def show_report(self, job_db, jobnum_list):
report_dict_list = []
for jobnum in jobnum_list:
job_obj = job_db.get_job_transient(jobnum)
if job_obj.state != Job.INIT:
report_dict_list.append({0: jobnum, 1: Job.enum2str(job_obj.state), 2: job_obj.gc_id})
self._fill_report_dict_list(report_dict_list, job_obj)
header_list = ['Job', 'Status / Attempt', 'Id / Destination']
self._show_table(lzip(irange(3), header_list), report_dict_list, 'rcl')
def resetState(jobs, newState):
jobSet = set(jobs)
for jobNum in jobs:
jobObj = self.jobDB.get(jobNum)
if jobObj and jobObj.state in [ Job.INIT, Job.DISABLED, Job.ABORTED, Job.CANCELLED, Job.DONE, Job.FAILED, Job.SUCCESS ]:
self._update(jobObj, jobNum, newState)
jobSet.remove(jobNum)
jobObj.attempt = 0
if len(jobSet) > 0:
output = (Job.enum2str(newState), str.join(', ', imap(str, jobSet)))
raise JobError('For the following jobs it was not possible to reset the state to %s:\n%s' % output)
def _serialize_job_obj(self, job_obj):
data = dict(job_obj.dict)
data['status'] = Job.enum2str(job_obj.state)
data['attempt'] = job_obj.attempt
data['submitted'] = job_obj.submitted
data['changed'] = job_obj.changed
for key, value in job_obj.history.items():
data['history_' + str(key)] = value
if job_obj.gcID is not None:
data['id'] = job_obj.dict.get('legacy_gcID', None) or job_obj.gcID # store legacy gcID
return data
def _serialize_job_obj(self, job_obj):
data = dict(job_obj.dict)
data['status'] = Job.enum2str(job_obj.state)
data['attempt'] = job_obj.attempt
data['submitted'] = job_obj.submitted
data['changed'] = job_obj.changed
for key, value in job_obj.history.items():
data['history_' + str(key)] = value
if job_obj.gcID is not None:
data['id'] = job_obj.dict.get(
'legacy_gcID', None) or job_obj.gcID # store legacy gcID
return data
def _reset_state(jobnum_list, state_new):
jobnum_listet = set(jobnum_list)
for jobnum in jobnum_list:
job_obj = self.job_db.get_job_persistent(jobnum)
if job_obj.state in resetable_state_list:
self._update(task, job_obj, jobnum, state_new)
jobnum_listet.remove(jobnum)
job_obj.attempt = 0
if len(jobnum_listet) > 0:
raise JobError('For the following jobs it was not possible to reset the state to %s:\n%s' % (
Job.enum2str(state_new), str.join(', ', imap(str, jobnum_listet))))
def _update(self, jobObj, jobNum, state, showWMS=False):
if jobObj.state == state:
return
oldState = jobObj.state
jobObj.update(state)
self.jobDB.commit(jobNum, jobObj)
jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1)
jobStatus = [
'Job %s state changed from %s to %s ' %
(str(jobNum).ljust(jobNumLen), Job.enum2str(oldState),
Job.enum2str(state))
]
if showWMS and jobObj.wmsId:
jobStatus.append('(WMS:%s)' % jobObj.wmsId.split('.')[1])
if (state == Job.SUBMITTED) and (jobObj.attempt > 1):
jobStatus.append('(retry #%s)' % (jobObj.attempt - 1))
elif (state == Job.QUEUED) and (jobObj.get('dest') != 'N/A'):
jobStatus.append('(%s)' % jobObj.get('dest'))
elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED
]) and jobObj.get('reason'):
jobStatus.append('(%s)' % jobObj.get('reason'))
elif (state == Job.SUCCESS) and (jobObj.get('runtime', None)
is not None):
jobStatus.append('(runtime %s)' %
strTime(jobObj.get('runtime') or 0))
elif state == Job.FAILED:
msg = []
retCode = jobObj.get('retcode')
if retCode:
msg.append('error code: %d' % retCode)
if (utils.verbosity() > 0) and (retCode
in self._task.errorDict):
msg.append(self._task.errorDict[retCode])
if jobObj.get('dest'):
msg.append(jobObj.get('dest'))
if len(msg):
jobStatus.append('(%s)' % str.join(' - ', msg))
self._log_user_time.info(str.join(' ', jobStatus))
def show_report(self, job_db, jobnum_list):
report_dict_list = []
for jobnum in jobnum_list:
job_obj = job_db.get_job_transient(jobnum)
if job_obj.state != Job.INIT:
report_dict_list.append({
0: jobnum,
1: Job.enum2str(job_obj.state),
2: job_obj.gc_id
})
self._fill_report_dict_list(report_dict_list, job_obj)
header_list = ['Job', 'Status / Attempt', 'Id / Destination']
self._show_table(lzip(irange(3), header_list), report_dict_list, 'rcl')
def on_job_state_change(self,
job_db_len,
jobnum,
job_obj,
old_state,
new_state,
reason=None):
jobnum_len = int(math.log10(max(1, job_db_len)) + 1)
job_status_str_list = [
'Job %s state changed from %s to %s' %
(str(jobnum).ljust(jobnum_len), Job.enum2str(old_state),
Job.enum2str(new_state))
]
if reason:
job_status_str_list.append('(%s)' % reason)
if self._show_wms and job_obj.gc_id:
job_status_str_list.append('(WMS:%s)' %
job_obj.gc_id.split('.')[1])
if (new_state == Job.SUBMITTED) and (job_obj.attempt > 1):
job_status_str_list.append('(retry #%s)' % (job_obj.attempt - 1))
elif (new_state
== Job.QUEUED) and (job_obj.get_job_location() != 'N/A'):
job_status_str_list.append('(%s)' % job_obj.get_job_location())
elif (new_state in [Job.WAITING, Job.ABORTED, Job.DISABLED
]) and job_obj.get('reason'):
job_status_str_list.append('(%s)' % job_obj.get('reason'))
elif (new_state == Job.SUCCESS) and (job_obj.get('runtime')
is not None):
if (job_obj.get('runtime') or 0) >= 0:
job_status_str_list.append(
'(runtime %s)' %
str_time_long(job_obj.get('runtime') or 0))
elif new_state == Job.FAILED:
fail_msg = self._explain_failure(job_obj)
if fail_msg:
job_status_str_list.append('(%s)' % fail_msg)
self._log_status.log_time(logging.INFO,
str.join(' ', job_status_str_list))
def _reset_state(jobnum_list, state_new):
jobnum_listet = set(jobnum_list)
for jobnum in jobnum_list:
job_obj = self.job_db.get_job_persistent(jobnum)
if job_obj.state in resetable_state_list:
self._update(task, job_obj, jobnum, state_new)
jobnum_listet.remove(jobnum)
job_obj.attempt = 0
if len(jobnum_listet) > 0:
raise JobError(
'For the following jobs it was not possible to reset the state to %s:\n%s'
% (Job.enum2str(state_new),
str.join(', ', imap(str, jobnum_listet))))
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.getJob(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({
0: jobNum,
1: Job.enum2str(jobObj.state),
2: jobObj.gcID
})
self._add_details(reports, jobObj)
utils.printTabular(
lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']),
reports, 'rcl')
def on_job_state_change(self, task, job_db_len,
jobnum, job_obj, old_state, new_state, reason=None):
jobnum_len = int(math.log10(max(1, job_db_len)) + 1)
job_status_str_list = ['Job %s state changed from %s to %s' % (
str(jobnum).ljust(jobnum_len), Job.enum2str(old_state), Job.enum2str(new_state))]
if reason:
job_status_str_list.append('(%s)' % reason)
if self._show_wms and job_obj.gc_id:
job_status_str_list.append('(WMS:%s)' % job_obj.gc_id.split('.')[1])
if (new_state == Job.SUBMITTED) and (job_obj.attempt > 1):
job_status_str_list.append('(retry #%s)' % (job_obj.attempt - 1))
elif (new_state == Job.QUEUED) and (job_obj.get_job_location() != 'N/A'):
job_status_str_list.append('(%s)' % job_obj.get_job_location())
elif (new_state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and job_obj.get('reason'):
job_status_str_list.append('(%s)' % job_obj.get('reason'))
elif (new_state == Job.SUCCESS) and (job_obj.get('runtime') is not None):
if (job_obj.get('runtime') or 0) >= 0:
job_status_str_list.append('(runtime %s)' % str_time_long(job_obj.get('runtime') or 0))
elif new_state == Job.FAILED:
fail_msg = self._explain_failure(task, job_obj)
if fail_msg:
job_status_str_list.append('(%s)' % fail_msg)
self._log_status.log_time(logging.INFO, str.join(' ', job_status_str_list))
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.get(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({0: jobNum, 1: Job.enum2str(jobObj.state), 2: jobObj.wmsId})
if utils.verbosity() > 0:
history = jobObj.history.items()
history.reverse()
for at, dest in history:
if dest != 'N/A':
reports.append({1: at, 2: ' -> ' + dest})
elif jobObj.get('dest', 'N/A') != 'N/A':
reports.append({2: ' -> ' + jobObj.get('dest')})
utils.printTabular(lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']), reports, 'rcl')
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.get(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({0: jobNum, 1: Job.enum2str(jobObj.state), 2: jobObj.wmsId})
if utils.verbosity() > 0:
history = jobObj.history.items()
history.reverse()
for at, dest in history:
if dest != "N/A":
reports.append({1: at, 2: " -> " + dest})
elif jobObj.get("dest", "N/A") != "N/A":
reports.append({2: " -> " + jobObj.get("dest")})
utils.printTabular(zip(range(3), ["Job", "Status / Attempt", "Id / Destination"]), reports, "rcl")
utils.vprint()
def resetState(jobs, newState):
jobSet = set(jobs)
for jobNum in jobs:
jobObj = self.jobDB.get(jobNum)
if jobObj and jobObj.state in [
Job.INIT, Job.DISABLED, Job.ABORTED, Job.CANCELLED,
Job.DONE, Job.FAILED, Job.SUCCESS
]:
self._update(jobObj, jobNum, newState)
jobSet.remove(jobNum)
jobObj.attempt = 0
if len(jobSet) > 0:
output = (Job.enum2str(newState),
str.join(', ', imap(str, jobSet)))
raise JobError(
'For the following jobs it was not possible to reset the state to %s:\n%s'
% output)
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.get(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({
0: jobNum,
1: Job.enum2str(jobObj.state),
2: jobObj.wmsId
})
if utils.verbosity() > 0:
history = jobObj.history.items()
history.reverse()
for at, dest in history:
if dest != 'N/A':
reports.append({1: at, 2: ' -> ' + dest})
elif jobObj.get('dest', 'N/A') != 'N/A':
reports.append({2: ' -> ' + jobObj.get('dest')})
utils.printTabular(
lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']),
reports, 'rcl')
def _update(self, jobObj, jobNum, state, showWMS = False):
if jobObj.state == state:
return
oldState = jobObj.state
jobObj.update(state)
self.jobDB.commit(jobNum, jobObj)
jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1)
utils.vprint('Job %s state changed from %s to %s ' % (str(jobNum).ljust(jobNumLen), Job.enum2str(oldState), Job.enum2str(state)), -1, True, False)
if showWMS and jobObj.wmsId:
print '(WMS:%s)' % jobObj.wmsId.split('.')[1],
if (state == Job.SUBMITTED) and (jobObj.attempt > 1):
print '(retry #%s)' % (jobObj.attempt - 1)
elif (state == Job.QUEUED) and jobObj.get('dest') != 'N/A':
print '(%s)' % jobObj.get('dest')
elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and jobObj.get('reason'):
print '(%s)' % jobObj.get('reason')
elif (state == Job.SUCCESS) and jobObj.get('runtime', None) != None:
print '(runtime %s)' % utils.strTime(utils.QM(jobObj.get('runtime') != '', jobObj.get('runtime'), 0))
elif (state == Job.FAILED):
msg = []
if jobObj.get('retcode'):
msg.append('error code: %d' % jobObj.get('retcode'))
try:
if utils.verbosity() > 0:
msg.append(self._task.errorDict[jobObj.get('retcode')])
except Exception:
pass
if jobObj.get('dest'):
msg.append(jobObj.get('dest'))
if len(msg):
print '(%s)' % str.join(' - ', msg),
print
else:
print
def onJobUpdate(self, wms, jobObj, jobNum, data):
self._runInBackground(self._evtStatus, jobNum, jobObj, {'STATUS': Job.enum2str(jobObj.state)})
def _update(self, jobObj, jobNum, state, showWMS = False):
if jobObj.state == state:
return
oldState = jobObj.state
jobObj.update(state)
self.jobDB.commit(jobNum, jobObj)
jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1)
jobStatus = ['Job %s state changed from %s to %s ' % (str(jobNum).ljust(jobNumLen), Job.enum2str(oldState), Job.enum2str(state))]
if showWMS and jobObj.wmsId:
jobStatus.append('(WMS:%s)' % jobObj.wmsId.split('.')[1])
if (state == Job.SUBMITTED) and (jobObj.attempt > 1):
jobStatus.append('(retry #%s)' % (jobObj.attempt - 1))
elif (state == Job.QUEUED) and (jobObj.get('dest') != 'N/A'):
jobStatus.append('(%s)' % jobObj.get('dest'))
elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and jobObj.get('reason'):
jobStatus.append('(%s)' % jobObj.get('reason'))
elif (state == Job.SUCCESS) and (jobObj.get('runtime', None) is not None):
jobStatus.append('(runtime %s)' % strTime(jobObj.get('runtime') or 0))
elif state == Job.FAILED:
msg = []
retCode = jobObj.get('retcode')
if retCode:
msg.append('error code: %d' % retCode)
if (utils.verbosity() > 0) and (retCode in self._task.errorDict):
msg.append(self._task.errorDict[retCode])
if jobObj.get('dest'):
msg.append(jobObj.get('dest'))
if len(msg):
jobStatus.append('(%s)' % str.join(' - ', msg))
self._log_user_time.info(str.join(' ', jobStatus))
def onJobUpdate(self, wms, jobObj, jobNum, data):
self._runInBackground(self._evtStatus, jobNum, jobObj,
{'STATUS': Job.enum2str(jobObj.state)})
