def jobs(self, *args, **kw): result = '<body>' result += str(CPProgressBar(0, min(100, self.counter), 100, 300)) if 'job' in kw: jobNum = int(kw['job']) info = self.task.getJobConfig(jobNum) result += str(TabularHTML(zip(sorted(info), sorted(info)), [info], top = False)) def getJobObjs(): for jobNum in self.jobMgr.jobDB.getJobs(): result = self.jobMgr.jobDB.get(jobNum).__dict__ result['jobNum'] = jobNum result.update(result['dict']) yield result fmtTime = lambda t: time.strftime('%Y-%m-%d %T', time.localtime(t)) result += str(TabularHTML([ ('jobNum', 'Job'), ('state', 'Status'), ('attempt', 'Attempt'), ('wmsId', 'WMS ID'), ('dest', 'Destination'), ('submitted', 'Submitted') ], getJobObjs(), fmt = { 'jobNum': lambda x: '<a href="jobs?job=%s">%s</a>' % (x, x), 'state': lambda s: Job.enum2str(s), 'submitted': fmtTime }, top = True)) result += '</body>' return result
def _update_job_states(self, job_db, jobnum_list): states = {} for state_name in Job.enum_name_list: states[state_name] = 0 for jobnum in jobnum_list: states[Job.enum2str(job_db.get_job_transient(jobnum).state)] += 1 self._task_info['states'] = states
def show_report(self, job_db, jobnum_list): report_dict_list = [] for jobnum in jobnum_list: job_obj = job_db.get_job_transient(jobnum) if job_obj.state != Job.INIT: report_dict_list.append({0: jobnum, 1: Job.enum2str(job_obj.state), 2: job_obj.gc_id}) self._fill_report_dict_list(report_dict_list, job_obj) header_list = ['Job', 'Status / Attempt', 'Id / Destination'] self._show_table(lzip(irange(3), header_list), report_dict_list, 'rcl')
def resetState(jobs, newState): jobSet = set(jobs) for jobNum in jobs: jobObj = self.jobDB.get(jobNum) if jobObj and jobObj.state in [ Job.INIT, Job.DISABLED, Job.ABORTED, Job.CANCELLED, Job.DONE, Job.FAILED, Job.SUCCESS ]: self._update(jobObj, jobNum, newState) jobSet.remove(jobNum) jobObj.attempt = 0 if len(jobSet) > 0: output = (Job.enum2str(newState), str.join(', ', imap(str, jobSet))) raise JobError('For the following jobs it was not possible to reset the state to %s:\n%s' % output)
def _serialize_job_obj(self, job_obj): data = dict(job_obj.dict) data['status'] = Job.enum2str(job_obj.state) data['attempt'] = job_obj.attempt data['submitted'] = job_obj.submitted data['changed'] = job_obj.changed for key, value in job_obj.history.items(): data['history_' + str(key)] = value if job_obj.gcID is not None: data['id'] = job_obj.dict.get('legacy_gcID', None) or job_obj.gcID # store legacy gcID return data
def _serialize_job_obj(self, job_obj): data = dict(job_obj.dict) data['status'] = Job.enum2str(job_obj.state) data['attempt'] = job_obj.attempt data['submitted'] = job_obj.submitted data['changed'] = job_obj.changed for key, value in job_obj.history.items(): data['history_' + str(key)] = value if job_obj.gcID is not None: data['id'] = job_obj.dict.get( 'legacy_gcID', None) or job_obj.gcID # store legacy gcID return data
def _reset_state(jobnum_list, state_new): jobnum_listet = set(jobnum_list) for jobnum in jobnum_list: job_obj = self.job_db.get_job_persistent(jobnum) if job_obj.state in resetable_state_list: self._update(task, job_obj, jobnum, state_new) jobnum_listet.remove(jobnum) job_obj.attempt = 0 if len(jobnum_listet) > 0: raise JobError('For the following jobs it was not possible to reset the state to %s:\n%s' % ( Job.enum2str(state_new), str.join(', ', imap(str, jobnum_listet))))
def _update(self, jobObj, jobNum, state, showWMS=False): if jobObj.state == state: return oldState = jobObj.state jobObj.update(state) self.jobDB.commit(jobNum, jobObj) jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1) jobStatus = [ 'Job %s state changed from %s to %s ' % (str(jobNum).ljust(jobNumLen), Job.enum2str(oldState), Job.enum2str(state)) ] if showWMS and jobObj.wmsId: jobStatus.append('(WMS:%s)' % jobObj.wmsId.split('.')[1]) if (state == Job.SUBMITTED) and (jobObj.attempt > 1): jobStatus.append('(retry #%s)' % (jobObj.attempt - 1)) elif (state == Job.QUEUED) and (jobObj.get('dest') != 'N/A'): jobStatus.append('(%s)' % jobObj.get('dest')) elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED ]) and jobObj.get('reason'): jobStatus.append('(%s)' % jobObj.get('reason')) elif (state == Job.SUCCESS) and (jobObj.get('runtime', None) is not None): jobStatus.append('(runtime %s)' % strTime(jobObj.get('runtime') or 0)) elif state == Job.FAILED: msg = [] retCode = jobObj.get('retcode') if retCode: msg.append('error code: %d' % retCode) if (utils.verbosity() > 0) and (retCode in self._task.errorDict): msg.append(self._task.errorDict[retCode]) if jobObj.get('dest'): msg.append(jobObj.get('dest')) if len(msg): jobStatus.append('(%s)' % str.join(' - ', msg)) self._log_user_time.info(str.join(' ', jobStatus))
def show_report(self, job_db, jobnum_list): report_dict_list = [] for jobnum in jobnum_list: job_obj = job_db.get_job_transient(jobnum) if job_obj.state != Job.INIT: report_dict_list.append({ 0: jobnum, 1: Job.enum2str(job_obj.state), 2: job_obj.gc_id }) self._fill_report_dict_list(report_dict_list, job_obj) header_list = ['Job', 'Status / Attempt', 'Id / Destination'] self._show_table(lzip(irange(3), header_list), report_dict_list, 'rcl')
def on_job_state_change(self, job_db_len, jobnum, job_obj, old_state, new_state, reason=None): jobnum_len = int(math.log10(max(1, job_db_len)) + 1) job_status_str_list = [ 'Job %s state changed from %s to %s' % (str(jobnum).ljust(jobnum_len), Job.enum2str(old_state), Job.enum2str(new_state)) ] if reason: job_status_str_list.append('(%s)' % reason) if self._show_wms and job_obj.gc_id: job_status_str_list.append('(WMS:%s)' % job_obj.gc_id.split('.')[1]) if (new_state == Job.SUBMITTED) and (job_obj.attempt > 1): job_status_str_list.append('(retry #%s)' % (job_obj.attempt - 1)) elif (new_state == Job.QUEUED) and (job_obj.get_job_location() != 'N/A'): job_status_str_list.append('(%s)' % job_obj.get_job_location()) elif (new_state in [Job.WAITING, Job.ABORTED, Job.DISABLED ]) and job_obj.get('reason'): job_status_str_list.append('(%s)' % job_obj.get('reason')) elif (new_state == Job.SUCCESS) and (job_obj.get('runtime') is not None): if (job_obj.get('runtime') or 0) >= 0: job_status_str_list.append( '(runtime %s)' % str_time_long(job_obj.get('runtime') or 0)) elif new_state == Job.FAILED: fail_msg = self._explain_failure(job_obj) if fail_msg: job_status_str_list.append('(%s)' % fail_msg) self._log_status.log_time(logging.INFO, str.join(' ', job_status_str_list))
def _reset_state(jobnum_list, state_new): jobnum_listet = set(jobnum_list) for jobnum in jobnum_list: job_obj = self.job_db.get_job_persistent(jobnum) if job_obj.state in resetable_state_list: self._update(task, job_obj, jobnum, state_new) jobnum_listet.remove(jobnum) job_obj.attempt = 0 if len(jobnum_listet) > 0: raise JobError( 'For the following jobs it was not possible to reset the state to %s:\n%s' % (Job.enum2str(state_new), str.join(', ', imap(str, jobnum_listet))))
def display(self): reports = [] for jobNum in self._jobs: jobObj = self._jobDB.getJob(jobNum) if not jobObj or (jobObj.state == Job.INIT): continue reports.append({ 0: jobNum, 1: Job.enum2str(jobObj.state), 2: jobObj.gcID }) self._add_details(reports, jobObj) utils.printTabular( lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']), reports, 'rcl')
def on_job_state_change(self, task, job_db_len, jobnum, job_obj, old_state, new_state, reason=None): jobnum_len = int(math.log10(max(1, job_db_len)) + 1) job_status_str_list = ['Job %s state changed from %s to %s' % ( str(jobnum).ljust(jobnum_len), Job.enum2str(old_state), Job.enum2str(new_state))] if reason: job_status_str_list.append('(%s)' % reason) if self._show_wms and job_obj.gc_id: job_status_str_list.append('(WMS:%s)' % job_obj.gc_id.split('.')[1]) if (new_state == Job.SUBMITTED) and (job_obj.attempt > 1): job_status_str_list.append('(retry #%s)' % (job_obj.attempt - 1)) elif (new_state == Job.QUEUED) and (job_obj.get_job_location() != 'N/A'): job_status_str_list.append('(%s)' % job_obj.get_job_location()) elif (new_state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and job_obj.get('reason'): job_status_str_list.append('(%s)' % job_obj.get('reason')) elif (new_state == Job.SUCCESS) and (job_obj.get('runtime') is not None): if (job_obj.get('runtime') or 0) >= 0: job_status_str_list.append('(runtime %s)' % str_time_long(job_obj.get('runtime') or 0)) elif new_state == Job.FAILED: fail_msg = self._explain_failure(task, job_obj) if fail_msg: job_status_str_list.append('(%s)' % fail_msg) self._log_status.log_time(logging.INFO, str.join(' ', job_status_str_list))
def display(self): reports = [] for jobNum in self._jobs: jobObj = self._jobDB.get(jobNum) if not jobObj or (jobObj.state == Job.INIT): continue reports.append({0: jobNum, 1: Job.enum2str(jobObj.state), 2: jobObj.wmsId}) if utils.verbosity() > 0: history = jobObj.history.items() history.reverse() for at, dest in history: if dest != 'N/A': reports.append({1: at, 2: ' -> ' + dest}) elif jobObj.get('dest', 'N/A') != 'N/A': reports.append({2: ' -> ' + jobObj.get('dest')}) utils.printTabular(lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']), reports, 'rcl')
def display(self): reports = [] for jobNum in self._jobs: jobObj = self._jobDB.get(jobNum) if not jobObj or (jobObj.state == Job.INIT): continue reports.append({0: jobNum, 1: Job.enum2str(jobObj.state), 2: jobObj.wmsId}) if utils.verbosity() > 0: history = jobObj.history.items() history.reverse() for at, dest in history: if dest != "N/A": reports.append({1: at, 2: " -> " + dest}) elif jobObj.get("dest", "N/A") != "N/A": reports.append({2: " -> " + jobObj.get("dest")}) utils.printTabular(zip(range(3), ["Job", "Status / Attempt", "Id / Destination"]), reports, "rcl") utils.vprint()
def resetState(jobs, newState): jobSet = set(jobs) for jobNum in jobs: jobObj = self.jobDB.get(jobNum) if jobObj and jobObj.state in [ Job.INIT, Job.DISABLED, Job.ABORTED, Job.CANCELLED, Job.DONE, Job.FAILED, Job.SUCCESS ]: self._update(jobObj, jobNum, newState) jobSet.remove(jobNum) jobObj.attempt = 0 if len(jobSet) > 0: output = (Job.enum2str(newState), str.join(', ', imap(str, jobSet))) raise JobError( 'For the following jobs it was not possible to reset the state to %s:\n%s' % output)
def display(self): reports = [] for jobNum in self._jobs: jobObj = self._jobDB.get(jobNum) if not jobObj or (jobObj.state == Job.INIT): continue reports.append({ 0: jobNum, 1: Job.enum2str(jobObj.state), 2: jobObj.wmsId }) if utils.verbosity() > 0: history = jobObj.history.items() history.reverse() for at, dest in history: if dest != 'N/A': reports.append({1: at, 2: ' -> ' + dest}) elif jobObj.get('dest', 'N/A') != 'N/A': reports.append({2: ' -> ' + jobObj.get('dest')}) utils.printTabular( lzip(irange(3), ['Job', 'Status / Attempt', 'Id / Destination']), reports, 'rcl')
def _update(self, jobObj, jobNum, state, showWMS = False): if jobObj.state == state: return oldState = jobObj.state jobObj.update(state) self.jobDB.commit(jobNum, jobObj) jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1) utils.vprint('Job %s state changed from %s to %s ' % (str(jobNum).ljust(jobNumLen), Job.enum2str(oldState), Job.enum2str(state)), -1, True, False) if showWMS and jobObj.wmsId: print '(WMS:%s)' % jobObj.wmsId.split('.')[1], if (state == Job.SUBMITTED) and (jobObj.attempt > 1): print '(retry #%s)' % (jobObj.attempt - 1) elif (state == Job.QUEUED) and jobObj.get('dest') != 'N/A': print '(%s)' % jobObj.get('dest') elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and jobObj.get('reason'): print '(%s)' % jobObj.get('reason') elif (state == Job.SUCCESS) and jobObj.get('runtime', None) != None: print '(runtime %s)' % utils.strTime(utils.QM(jobObj.get('runtime') != '', jobObj.get('runtime'), 0)) elif (state == Job.FAILED): msg = [] if jobObj.get('retcode'): msg.append('error code: %d' % jobObj.get('retcode')) try: if utils.verbosity() > 0: msg.append(self._task.errorDict[jobObj.get('retcode')]) except Exception: pass if jobObj.get('dest'): msg.append(jobObj.get('dest')) if len(msg): print '(%s)' % str.join(' - ', msg), print else: print
def onJobUpdate(self, wms, jobObj, jobNum, data): self._runInBackground(self._evtStatus, jobNum, jobObj, {'STATUS': Job.enum2str(jobObj.state)})
def _update(self, jobObj, jobNum, state, showWMS = False): if jobObj.state == state: return oldState = jobObj.state jobObj.update(state) self.jobDB.commit(jobNum, jobObj) jobNumLen = int(math.log10(max(1, len(self.jobDB))) + 1) jobStatus = ['Job %s state changed from %s to %s ' % (str(jobNum).ljust(jobNumLen), Job.enum2str(oldState), Job.enum2str(state))] if showWMS and jobObj.wmsId: jobStatus.append('(WMS:%s)' % jobObj.wmsId.split('.')[1]) if (state == Job.SUBMITTED) and (jobObj.attempt > 1): jobStatus.append('(retry #%s)' % (jobObj.attempt - 1)) elif (state == Job.QUEUED) and (jobObj.get('dest') != 'N/A'): jobStatus.append('(%s)' % jobObj.get('dest')) elif (state in [Job.WAITING, Job.ABORTED, Job.DISABLED]) and jobObj.get('reason'): jobStatus.append('(%s)' % jobObj.get('reason')) elif (state == Job.SUCCESS) and (jobObj.get('runtime', None) is not None): jobStatus.append('(runtime %s)' % strTime(jobObj.get('runtime') or 0)) elif state == Job.FAILED: msg = [] retCode = jobObj.get('retcode') if retCode: msg.append('error code: %d' % retCode) if (utils.verbosity() > 0) and (retCode in self._task.errorDict): msg.append(self._task.errorDict[retCode]) if jobObj.get('dest'): msg.append(jobObj.get('dest')) if len(msg): jobStatus.append('(%s)' % str.join(' - ', msg)) self._log_user_time.info(str.join(' ', jobStatus))