def _create_job_obj(self, name, data):
try:
job = Job()
job.state = Job.str2enum(data.pop('status'), Job.UNKNOWN)
if 'id' in data:
gc_id = data.pop('id')
if not gc_id.startswith('WMSID'): # Legacy support
data['legacy_gc_id'] = gc_id
if gc_id.startswith('https'):
gc_id = 'WMSID.GLITEWMS.%s' % gc_id
else:
wms_id, wms_name = tuple(gc_id.split('.', 1))
gc_id = 'WMSID.%s.%s' % (wms_name, wms_id)
job.gc_id = gc_id
for key in ['attempt', 'submitted', 'changed']:
if key in data:
setattr(job, key, data[key])
if 'runtime' not in data:
if 'submitted' in data and (job.submitted > 0):
data['runtime'] = time.time() - float(job.submitted)
else:
data['runtime'] = 0
for key in irange(1, job.attempt + 1):
if ('history_' + str(key)).strip() in data:
job.history[key] = data['history_' + str(key)]
job.set_dict(data)
except Exception:
raise JobError('Unable to parse data in %s:\n%r' % (name, data))
return job
def jobs(self, *args, **kw):
result = '<body>'
result += str(CPProgressBar(0, min(100, self.counter), 100, 300))
if 'job' in kw:
jobNum = int(kw['job'])
info = self.task.getJobConfig(jobNum)
result += str(TabularHTML(zip(sorted(info), sorted(info)), [info], top = False))
def getJobObjs():
for jobNum in self.jobMgr.jobDB.getJobs():
result = self.jobMgr.jobDB.get(jobNum).__dict__
result['jobNum'] = jobNum
result.update(result['dict'])
yield result
fmtTime = lambda t: time.strftime('%Y-%m-%d %T', time.localtime(t))
result += str(TabularHTML([
('jobNum', 'Job'), ('state', 'Status'), ('attempt', 'Attempt'),
('wmsId', 'WMS ID'), ('dest', 'Destination'), ('submitted', 'Submitted')
], getJobObjs(),
fmt = {
'jobNum': lambda x: '<a href="jobs?job=%s">%s</a>' % (x, x),
'state': lambda s: Job.enum2str(s),
'submitted': fmtTime
}, top = True))
result += '</body>'
return result
def _getReportInfos(self):
result = []
defaultJob = Job()
t_now = time.time()
for jobNum in self._jobs:
jobObj = self._jobDB.get(jobNum, defaultJob)
runtime = parseStr(jobObj.get('runtime'), int, 0)
for attempt in jobObj.history:
if (attempt != jobObj.attempt) and not self._useHistory:
continue
if (attempt == jobObj.attempt) and (jobObj.state == Job.SUCCESS):
time_info = runtime
elif (attempt == jobObj.attempt - 1) and (jobObj.state != Job.SUCCESS):
time_info = runtime
elif attempt == jobObj.attempt:
time_info = t_now - float(jobObj.submitted)
else:
time_info = 0
state = jobObj.state
if attempt != jobObj.attempt:
state = Job.FAILED
dest = jobObj.history[attempt]
if dest == 'N/A':
dest_info = [dest]
else:
dest_info = dest.split('/')
wmsName = jobObj.wmsId.split('.')[1]
endpoint = 'N/A'
if 'http:' in jobObj.wmsId:
endpoint = jobObj.wmsId.split(':')[1].split('/')[0]
result.append([state, time_info, wmsName, endpoint] + dest_info)
return result
def _create_job_obj(self, name, data):
try:
job = Job()
job.state = Job.str2enum(data.pop('status'), Job.UNKNOWN)
if 'id' in data:
gc_id = data.pop('id')
if not gc_id.startswith('WMSID'): # Legacy support
data['legacy_gc_id'] = gc_id
if gc_id.startswith('https'):
gc_id = 'WMSID.GLITEWMS.%s' % gc_id
else:
wms_id, wms_name = tuple(gc_id.split('.', 1))
gc_id = 'WMSID.%s.%s' % (wms_name, wms_id)
job.gc_id = gc_id
for key in ['attempt', 'submitted', 'changed']:
if key in data:
setattr(job, key, data[key])
if 'runtime' not in data:
if 'submitted' in data and (job.submitted > 0):
data['runtime'] = time.time() - float(job.submitted)
else:
data['runtime'] = 0
for key in irange(1, job.attempt + 1):
if ('history_' + str(key)).strip() in data:
job.history[key] = data['history_' + str(key)]
job.set_dict(data)
except Exception:
raise JobError('Unable to parse data in %s:\n%r' % (name, data))
return job
def _update_job_states(self, job_db, jobnum_list):
states = {}
for state_name in Job.enum_name_list:
states[state_name] = 0
for jobnum in jobnum_list:
states[Job.enum2str(job_db.get_job_transient(jobnum).state)] += 1
self._task_info['states'] = states
def show_report(self, job_db, jobnum_list):
report_dict_list = []
for jobnum in jobnum_list:
job_obj = job_db.get_job_transient(jobnum)
if job_obj.state != Job.INIT:
report_dict_list.append({0: jobnum, 1: Job.enum2str(job_obj.state), 2: job_obj.gc_id})
self._fill_report_dict_list(report_dict_list, job_obj)
header_list = ['Job', 'Status / Attempt', 'Id / Destination']
self._show_table(lzip(irange(3), header_list), report_dict_list, 'rcl')
def _getCategoryStateSummary(self):
catStateDict = {}
defaultJob = Job()
for jobNum in self._jobs:
jobState = self._jobDB.get(jobNum, defaultJob).state
catKey = self._job2cat[jobNum]
catStateDict[catKey][jobState] = catStateDict.setdefault(
catKey, dict()).get(jobState, 0) + 1
return (catStateDict, dict(self._catDescDict), {}
) # (<state overview>, <descriptions>, <#subcategories>)
def _serialize_job_obj(self, job_obj):
data = dict(job_obj.dict)
data['status'] = Job.enum2str(job_obj.state)
data['attempt'] = job_obj.attempt
data['submitted'] = job_obj.submitted
data['changed'] = job_obj.changed
for key, value in job_obj.history.items():
data['history_' + str(key)] = value
if job_obj.gcID is not None:
data['id'] = job_obj.dict.get('legacy_gcID', None) or job_obj.gcID # store legacy gcID
return data
def resetState(jobs, newState):
jobSet = set(jobs)
for jobNum in jobs:
jobObj = self.jobDB.get(jobNum)
if jobObj and jobObj.state in [ Job.INIT, Job.DISABLED, Job.ABORTED, Job.CANCELLED, Job.DONE, Job.FAILED, Job.SUCCESS ]:
self._update(jobObj, jobNum, newState)
jobSet.remove(jobNum)
jobObj.attempt = 0
if len(jobSet) > 0:
output = (Job.enum2str(newState), str.join(', ', imap(str, jobSet)))
raise JobError('For the following jobs it was not possible to reset the state to %s:\n%s' % output)
def _serialize_job_obj(self, job_obj):
data = dict(job_obj.dict)
data['status'] = Job.enum2str(job_obj.state)
data['attempt'] = job_obj.attempt
data['submitted'] = job_obj.submitted
data['changed'] = job_obj.changed
for key, value in job_obj.history.items():
data['history_' + str(key)] = value
if job_obj.gcID is not None:
data['id'] = job_obj.dict.get(
'legacy_gcID', None) or job_obj.gcID # store legacy gcID
return data
def _reset_state(jobnum_list, state_new):
jobnum_listet = set(jobnum_list)
for jobnum in jobnum_list:
job_obj = self.job_db.get_job_persistent(jobnum)
if job_obj.state in resetable_state_list:
self._update(task, job_obj, jobnum, state_new)
jobnum_listet.remove(jobnum)
job_obj.attempt = 0
if len(jobnum_listet) > 0:
raise JobError('For the following jobs it was not possible to reset the state to %s:\n%s' % (
Job.enum2str(state_new), str.join(', ', imap(str, jobnum_listet))))
def _update(self, jobObj, jobNum, state, showWMS=False):
if jobObj.state == state:
return
oldState = jobObj.state
jobObj.update(state)
self.jobDB.commit(jobNum, jobObj)
jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1)
jobStatus = [
'Job %s state changed from %s to %s ' %
(str(jobNum).ljust(jobNumLen), Job.enum2str(oldState),
Job.enum2str(state))
]
if showWMS and jobObj.wmsId:
jobStatus.append('(WMS:%s)' % jobObj.wmsId.split('.')[1])
if (state == Job.SUBMITTED) and (jobObj.attempt > 1):
jobStatus.append('(retry #%s)' % (jobObj.attempt - 1))
elif (state == Job.QUEUED) and (jobObj.get('dest') != 'N/A'):
jobStatus.append('(%s)' % jobObj.get('dest'))
elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED
]) and jobObj.get('reason'):
jobStatus.append('(%s)' % jobObj.get('reason'))
elif (state == Job.SUCCESS) and (jobObj.get('runtime', None)
is not None):
jobStatus.append('(runtime %s)' %
strTime(jobObj.get('runtime') or 0))
elif state == Job.FAILED:
msg = []
retCode = jobObj.get('retcode')
if retCode:
msg.append('error code: %d' % retCode)
if (utils.verbosity() > 0) and (retCode
in self._task.errorDict):
msg.append(self._task.errorDict[retCode])
if jobObj.get('dest'):
msg.append(jobObj.get('dest'))
if len(msg):
jobStatus.append('(%s)' % str.join(' - ', msg))
self._log_user_time.info(str.join(' ', jobStatus))
def on_job_state_change(self,
job_db_len,
jobnum,
job_obj,
old_state,
new_state,
reason=None):
jobnum_len = int(math.log10(max(1, job_db_len)) + 1)
job_status_str_list = [
'Job %s state changed from %s to %s' %
(str(jobnum).ljust(jobnum_len), Job.enum2str(old_state),
Job.enum2str(new_state))
]
if reason:
job_status_str_list.append('(%s)' % reason)
if self._show_wms and job_obj.gc_id:
job_status_str_list.append('(WMS:%s)' %
job_obj.gc_id.split('.')[1])
if (new_state == Job.SUBMITTED) and (job_obj.attempt > 1):
job_status_str_list.append('(retry #%s)' % (job_obj.attempt - 1))
elif (new_state
== Job.QUEUED) and (job_obj.get_job_location() != 'N/A'):
job_status_str_list.append('(%s)' % job_obj.get_job_location())
elif (new_state in [Job.WAITING, Job.ABORTED, Job.DISABLED
]) and job_obj.get('reason'):
job_status_str_list.append('(%s)' % job_obj.get('reason'))
elif (new_state == Job.SUCCESS) and (job_obj.get('runtime')
is not None):
if (job_obj.get('runtime') or 0) >= 0:
job_status_str_list.append(
'(runtime %s)' %
str_time_long(job_obj.get('runtime') or 0))
elif new_state == Job.FAILED:
fail_msg = self._explain_failure(job_obj)
if fail_msg:
job_status_str_list.append('(%s)' % fail_msg)
self._log_status.log_time(logging.INFO,
str.join(' ', job_status_str_list))
def show_report(self, job_db, jobnum_list):
report_dict_list = []
for jobnum in jobnum_list:
job_obj = job_db.get_job_transient(jobnum)
if job_obj.state != Job.INIT:
report_dict_list.append({
0: jobnum,
1: Job.enum2str(job_obj.state),
2: job_obj.gc_id
})
self._fill_report_dict_list(report_dict_list, job_obj)
header_list = ['Job', 'Status / Attempt', 'Id / Destination']
self._show_table(lzip(irange(3), header_list), report_dict_list, 'rcl')
def reset(self, wms, select):
jobs = self.jobDB.getJobs(JobSelector.create(select, task=self._task))
if jobs:
self._log_user.warning('Resetting the following jobs:')
self._reportClass(self.jobDB, self._task, jobs).display()
if utils.getUserBool(
'Are you sure you want to reset the state of these jobs?',
False):
self.cancel(
wms,
self.jobDB.getJobs(ClassSelector(JobClass.PROCESSING),
jobs), False, False)
for jobNum in jobs:
self.jobDB.commit(jobNum, Job())
def _reset_state(jobnum_list, state_new):
jobnum_listet = set(jobnum_list)
for jobnum in jobnum_list:
job_obj = self.job_db.get_job_persistent(jobnum)
if job_obj.state in resetable_state_list:
self._update(task, job_obj, jobnum, state_new)
jobnum_listet.remove(jobnum)
job_obj.attempt = 0
if len(jobnum_listet) > 0:
raise JobError(
'For the following jobs it was not possible to reset the state to %s:\n%s'
% (Job.enum2str(state_new),
str.join(', ', imap(str, jobnum_listet))))
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.getJob(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({
0: jobNum,
1: Job.enum2str(jobObj.state),
2: jobObj.gcID
})
self._add_details(reports, jobObj)
utils.printTabular(
lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']),
reports, 'rcl')
def on_job_state_change(self, task, job_db_len,
jobnum, job_obj, old_state, new_state, reason=None):
jobnum_len = int(math.log10(max(1, job_db_len)) + 1)
job_status_str_list = ['Job %s state changed from %s to %s' % (
str(jobnum).ljust(jobnum_len), Job.enum2str(old_state), Job.enum2str(new_state))]
if reason:
job_status_str_list.append('(%s)' % reason)
if self._show_wms and job_obj.gc_id:
job_status_str_list.append('(WMS:%s)' % job_obj.gc_id.split('.')[1])
if (new_state == Job.SUBMITTED) and (job_obj.attempt > 1):
job_status_str_list.append('(retry #%s)' % (job_obj.attempt - 1))
elif (new_state == Job.QUEUED) and (job_obj.get_job_location() != 'N/A'):
job_status_str_list.append('(%s)' % job_obj.get_job_location())
elif (new_state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and job_obj.get('reason'):
job_status_str_list.append('(%s)' % job_obj.get('reason'))
elif (new_state == Job.SUCCESS) and (job_obj.get('runtime') is not None):
if (job_obj.get('runtime') or 0) >= 0:
job_status_str_list.append('(runtime %s)' % str_time_long(job_obj.get('runtime') or 0))
elif new_state == Job.FAILED:
fail_msg = self._explain_failure(task, job_obj)
if fail_msg:
job_status_str_list.append('(%s)' % fail_msg)
self._log_status.log_time(logging.INFO, str.join(' ', job_status_str_list))
def __init__(self, config, name):
glite_path = os.environ.get('GLITE_WMS_LOCATION',
os.environ.get('GLITE_LOCATION', ''))
stored_sys_path = list(sys.path)
for p in [
'lib', 'lib64',
os.path.join('lib', 'python'),
os.path.join('lib64', 'python')
]:
sys.path.append(os.path.join(glite_path, p))
try: # gLite 3.2
import wmsui_api
glStates = wmsui_api.states_names
def getStatusDirect(wmsID):
try: # new parameter json
jobStatus = wmsui_api.getStatus(
wmsui_api.getJobIdfromList(None, [wmsID])[0], 0)
except Exception:
jobStatus = wmsui_api.getStatus(
wmsui_api.getJobIdfromList([wmsID])[0], 0)
return lmap(
lambda name: (name.lower(
), jobStatus.getAttribute(glStates.index(name))), glStates)
except Exception: # gLite 3.1
try:
from glite_wmsui_LbWrapper import Status
import Job
wrStatus = Status()
jobStatus = Job.JobStatus(wrStatus)
def getStatusDirect(wmsID):
wrStatus.getStatus(wmsID, 0)
err, apiMsg = wrStatus.get_error()
if err:
raise BackendError(apiMsg)
info = wrStatus.loadStatus()
return lzip(imap(str.lower, jobStatus.states_names),
info[0:jobStatus.ATTR_MAX])
except Exception:
getStatusDirect = None
sys.path = stored_sys_path
checkExecutor = None
if getStatusDirect:
checkExecutor = GliteWMSDirect_CheckJobs(config, getStatusDirect)
GliteWMS.__init__(self, config, name, checkExecutor=checkExecutor)
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.get(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({0: jobNum, 1: Job.enum2str(jobObj.state), 2: jobObj.wmsId})
if utils.verbosity() > 0:
history = jobObj.history.items()
history.reverse()
for at, dest in history:
if dest != 'N/A':
reports.append({1: at, 2: ' -> ' + dest})
elif jobObj.get('dest', 'N/A') != 'N/A':
reports.append({2: ' -> ' + jobObj.get('dest')})
utils.printTabular(lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']), reports, 'rcl')
def resetState(jobs, newState):
jobSet = set(jobs)
for jobNum in jobs:
jobObj = self.jobDB.get(jobNum)
if jobObj and jobObj.state in [
Job.INIT, Job.DISABLED, Job.ABORTED, Job.CANCELLED,
Job.DONE, Job.FAILED, Job.SUCCESS
]:
self._update(jobObj, jobNum, newState)
jobSet.remove(jobNum)
jobObj.attempt = 0
if len(jobSet) > 0:
output = (Job.enum2str(newState),
str.join(', ', imap(str, jobSet)))
raise JobError(
'For the following jobs it was not possible to reset the state to %s:\n%s'
% output)
def reset(self, task, wms, select):
jobnum_list = self.job_db.get_job_list(
JobSelector.create(select, task=task))
if jobnum_list:
self._log.warning('Resetting the following jobs:')
self._abort_report.show_report(self.job_db, jobnum_list)
ask_user_msg = 'Are you sure you want to reset the state of these jobs?'
if self._interactive_reset or self._uii.prompt_bool(
ask_user_msg, False):
self.cancel(wms,
self.job_db.get_job_list(
ClassSelector(JobClass.PROCESSING),
jobnum_list),
interactive=False,
show_jobs=False)
for jobnum in jobnum_list:
self.job_db.commit(jobnum, Job())
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.get(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({0: jobNum, 1: Job.enum2str(jobObj.state), 2: jobObj.wmsId})
if utils.verbosity() > 0:
history = jobObj.history.items()
history.reverse()
for at, dest in history:
if dest != "N/A":
reports.append({1: at, 2: " -> " + dest})
elif jobObj.get("dest", "N/A") != "N/A":
reports.append({2: " -> " + jobObj.get("dest")})
utils.printTabular(zip(range(3), ["Job", "Status / Attempt", "Id / Destination"]), reports, "rcl")
utils.vprint()
def readJobs(self, jobLimit):
jobMap = {}
maxJobs = 0
if os.path.exists(self._dbFile):
try:
tar = zipfile.ZipFile(self._dbFile, 'r', zipfile.ZIP_DEFLATED)
except Exception: # Try to recover job archive
utils.eprint('=' * 40 + '\nStarting recovery of broken job database')
utils.eprint(' => Answer "y" if asked "Is this a single-disk archive?"!\n' + '=' * 40)
os.system('zip -FF %s --out %s.tmp 2> /dev/null' % (self._dbFile, self._dbFile))
os.rename(self._dbFile, self._dbFile + '.broken')
os.rename(self._dbFile + '.tmp', self._dbFile)
tar = zipfile.ZipFile(self._dbFile, 'r', zipfile.ZIP_DEFLATED)
utils.removeFiles([self._dbFile + '.broken'])
brokenList = []
for idx, fnTarInfo in enumerate(tar.namelist()):
(jobNum, tid) = tuple(imap(lambda s: int(s[1:]), fnTarInfo.split('_', 1)))
try:
fp = tar.open(fnTarInfo)
try:
fp.read()
finally:
fp.close()
except Exception:
pass
for broken in brokenList:
os.system('zip %s -d %s' % (self._dbFile, broken))
utils.eprint('Recover completed!')
activity = utils.ActivityLog('Reading job transactions ...')
maxJobs = len(tar.namelist())
tMap = {}
for idx, fnTarInfo in enumerate(tar.namelist()):
(jobNum, tid) = tuple(imap(lambda s: int(s[1:]), fnTarInfo.split('_', 1)))
if tid < tMap.get(jobNum, 0):
continue
data = utils.DictFormat(escapeString = True).parse(tar.open(fnTarInfo).read())
jobMap[jobNum] = Job.loadData(fnTarInfo, data)
tMap[jobNum] = tid
if idx % 100 == 0:
activity.finish()
activity = utils.ActivityLog('Reading job transactions ... %d [%d%%]' % (idx, (100.0 * idx) / maxJobs))
self._serial = maxJobs
return jobMap
def display(self):
summary = lmap(lambda x: 0.0, Job.enumNames)
defaultJob = Job()
for jobNum in self._jobs:
summary[self._jobDB.get(jobNum, defaultJob).state] += 1
makeSum = lambda *states: sum(imap(lambda z: summary[z], states))
makePer = lambda *states: [
makeSum(*states),
round(makeSum(*states) / len(self._jobDB) * 100.0)
]
# Print report summary
self._printHeader('REPORT SUMMARY:')
njobs_total = len(self._jobDB)
jobov_succ = makePer(Job.SUCCESS)
utils.vprint(
'Total number of jobs:%9d Successful jobs:%8d %3d%%' %
tuple([njobs_total] + jobov_succ), -1)
njobs_assigned = makeSum(Job.SUBMITTED, Job.WAITING, Job.READY,
Job.QUEUED, Job.RUNNING)
jobov_fail = makePer(Job.ABORTED, Job.CANCELLED, Job.FAILED)
utils.vprint(
'Jobs assigned to WMS:%9d Failing jobs:%8d %3d%%' %
tuple([njobs_assigned] + jobov_fail), -1)
utils.vprint(' ' * 65 + '\nDetailed Status Information: ',
-1,
newline=False)
ignored = len(self._jobDB) - sum(summary)
if ignored:
utils.vprint(
'(Jobs IGNORED:%8d %3d%%)' %
(ignored, ignored / len(self._jobDB) * 100.0), -1)
else:
utils.vprint(' ' * 31, -1)
for stateNum, category in enumerate(Job.enumNames):
utils.vprint('Jobs %9s:%8d %3d%% ' %
tuple([category] + makePer(stateNum)),
-1,
newline=stateNum % 2)
utils.vprint('-' * 65, -1)
return 0
def _getSubmissionJobs(self, maxsample):
# Get list of submittable jobs
readyList = self.jobDB.getJobs(ClassSelector(JobClass.READY))
retryOK = readyList
defaultJob = Job()
if self._job_retries >= 0:
retryOK = lfilter(
lambda x: self.jobDB.get(x, defaultJob).attempt - 1 < self.
_job_retries, readyList)
modOK = lfilter(self._task.canSubmit, readyList)
jobList = set.intersection(set(retryOK), set(modOK))
if self._showBlocker and readyList and not jobList: # No submission but ready jobs
err = []
err += utils.QM((len(retryOK) > 0) and (len(modOK) == 0), [],
['have hit their maximum number of retries'])
err += utils.QM((len(retryOK) == 0) and (len(modOK) > 0), [],
['are vetoed by the task module'])
self._log_user_time.warning(
'All remaining jobs %s!',
str.join(utils.QM(retryOK or modOK, ' or ', ' and '), err))
self._showBlocker = not (len(readyList) > 0 and len(jobList) == 0)
# Determine number of jobs to submit
submit = len(jobList)
if self._njobs_inqueue > 0:
submit = min(
submit, self._njobs_inqueue -
self.jobDB.getJobsN(ClassSelector(JobClass.ATWMS)))
if self._njobs_inflight > 0:
submit = min(
submit, self._njobs_inflight -
self.jobDB.getJobsN(ClassSelector(JobClass.PROCESSING)))
if self._chunks_enabled and (maxsample > 0):
submit = min(submit, maxsample)
submit = max(submit, 0)
if self._do_shuffle:
return self._sample(jobList, submit)
return sorted(jobList)[:submit]
def readJobs(self, jobLimit):
jobMap = {}
maxJobs = 0
if os.path.exists(self._dbFile):
try:
tar = zipfile.ZipFile(self._dbFile, "r", zipfile.ZIP_DEFLATED)
except Exception: # Try to recover job archive
utils.eprint("=" * 40 + "\nStarting recovery of broken job database")
utils.eprint(' => Answer "y" if asked "Is this a single-disk archive?"!\n' + "=" * 40)
os.system("zip -FF %s --out %s.tmp 2> /dev/null" % (self._dbFile, self._dbFile))
os.rename(self._dbFile, self._dbFile + ".broken")
os.rename(self._dbFile + ".tmp", self._dbFile)
tar = zipfile.ZipFile(self._dbFile, "r", zipfile.ZIP_DEFLATED)
utils.removeFiles([self._dbFile + ".broken"])
brokenList = []
for idx, fnTarInfo in enumerate(tar.namelist()):
(jobNum, tid) = tuple(map(lambda s: int(s[1:]), fnTarInfo.split("_", 1)))
try:
rawData = tar.open(fnTarInfo).read()
except Exception:
pass
for broken in brokenList:
os.system("zip %s -d %s" % (self._dbFile, broken))
utils.eprint("Recover completed!")
log = None
maxJobs = len(tar.namelist())
tMap = {}
for idx, fnTarInfo in enumerate(tar.namelist()):
(jobNum, tid) = tuple(map(lambda s: int(s[1:]), fnTarInfo.split("_", 1)))
if tid < tMap.get(jobNum, 0):
continue
data = utils.DictFormat(escapeString=True).parse(tar.open(fnTarInfo).read())
jobMap[jobNum] = Job.loadData(fnTarInfo, data)
tMap[jobNum] = tid
if idx % 100 == 0:
del log
log = utils.ActivityLog("Reading job transactions ... %d [%d%%]" % (idx, (100.0 * idx) / maxJobs))
self._serial = maxJobs
return jobMap
def display(self):
reports = []
for jobNum in self._jobs:
jobObj = self._jobDB.get(jobNum)
if not jobObj or (jobObj.state == Job.INIT):
continue
reports.append({
0: jobNum,
1: Job.enum2str(jobObj.state),
2: jobObj.wmsId
})
if utils.verbosity() > 0:
history = jobObj.history.items()
history.reverse()
for at, dest in history:
if dest != 'N/A':
reports.append({1: at, 2: ' -> ' + dest})
elif jobObj.get('dest', 'N/A') != 'N/A':
reports.append({2: ' -> ' + jobObj.get('dest')})
utils.printTabular(
lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']),
reports, 'rcl')
def onJobUpdate(self, wms, jobObj, jobNum, data):
self._runInBackground(self._evtStatus, jobNum, jobObj,
{'STATUS': Job.enum2str(jobObj.state)})
def get_job_persistent(self, jobnum):
return self._job_map.get(jobnum, Job())
def _update(self, jobObj, jobNum, state, showWMS = False):
if jobObj.state == state:
return
oldState = jobObj.state
jobObj.update(state)
self.jobDB.commit(jobNum, jobObj)
jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1)
utils.vprint('Job %s state changed from %s to %s ' % (str(jobNum).ljust(jobNumLen), Job.enum2str(oldState), Job.enum2str(state)), -1, True, False)
if showWMS and jobObj.wmsId:
print '(WMS:%s)' % jobObj.wmsId.split('.')[1],
if (state == Job.SUBMITTED) and (jobObj.attempt > 1):
print '(retry #%s)' % (jobObj.attempt - 1)
elif (state == Job.QUEUED) and jobObj.get('dest') != 'N/A':
print '(%s)' % jobObj.get('dest')
elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and jobObj.get('reason'):
print '(%s)' % jobObj.get('reason')
elif (state == Job.SUCCESS) and jobObj.get('runtime', None) != None:
print '(runtime %s)' % utils.strTime(utils.QM(jobObj.get('runtime') != '', jobObj.get('runtime'), 0))
elif (state == Job.FAILED):
msg = []
if jobObj.get('retcode'):
msg.append('error code: %d' % jobObj.get('retcode'))
try:
if utils.verbosity() > 0:
msg.append(self._task.errorDict[jobObj.get('retcode')])
except Exception:
pass
if jobObj.get('dest'):
msg.append(jobObj.get('dest'))
if len(msg):
print '(%s)' % str.join(' - ', msg),
print
else:
print
def getJobPersistent(self, jobNum):
return self._jobMap.get(jobNum, Job())
def _update(self, jobObj, jobNum, state, showWMS = False):
if jobObj.state == state:
return
oldState = jobObj.state
jobObj.update(state)
self.jobDB.commit(jobNum, jobObj)
jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1)
jobStatus = ['Job %s state changed from %s to %s ' % (str(jobNum).ljust(jobNumLen), Job.enum2str(oldState), Job.enum2str(state))]
if showWMS and jobObj.wmsId:
jobStatus.append('(WMS:%s)' % jobObj.wmsId.split('.')[1])
if (state == Job.SUBMITTED) and (jobObj.attempt > 1):
jobStatus.append('(retry #%s)' % (jobObj.attempt - 1))
elif (state == Job.QUEUED) and (jobObj.get('dest') != 'N/A'):
jobStatus.append('(%s)' % jobObj.get('dest'))
elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and jobObj.get('reason'):
jobStatus.append('(%s)' % jobObj.get('reason'))
elif (state == Job.SUCCESS) and (jobObj.get('runtime', None) is not None):
jobStatus.append('(runtime %s)' % strTime(jobObj.get('runtime') or 0))
elif state == Job.FAILED:
msg = []
retCode = jobObj.get('retcode')
if retCode:
msg.append('error code: %d' % retCode)
if (utils.verbosity() > 0) and (retCode in self._task.errorDict):
msg.append(self._task.errorDict[retCode])
if jobObj.get('dest'):
msg.append(jobObj.get('dest'))
if len(msg):
jobStatus.append('(%s)' % str.join(' - ', msg))
self._log_user_time.info(str.join(' ', jobStatus))
def onJobUpdate(self, wms, jobObj, jobNum, data):
self._runInBackground(self._evtStatus, jobNum, jobObj, {'STATUS': Job.enum2str(jobObj.state)})