def _publish(self, jobObj, jobNum, taskId, usermsg):
(_, backend, rawId) = jobObj.gcID.split('.', 2)
dashId = '%s_%s' % (jobNum, rawId)
if 'http' not in jobObj.gcID:
dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId)
msg = mergeDicts([{'taskId': taskId, 'jobId': dashId, 'sid': rawId}] + usermsg)
DashboardAPI(taskId, dashId).publish(**filterDict(msg, vF = lambda v: v is not None))
def publish(self, jobObj, jobNum, taskId, usermsg):
(header, backend, rawId) = jobObj.wmsId.split('.', 2)
dashId = '%s_%s' % (jobNum, rawId)
if "http" not in jobObj.wmsId:
dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId)
msg = utils.mergeDicts([{'taskId': taskId, 'jobId': dashId, 'sid': rawId}] + usermsg)
DashboardAPI(taskId, dashId).publish(**utils.filterDict(msg, vF = lambda v: v != None))
def _submitJob(self, jobNum, module):
fd, jdl = tempfile.mkstemp('.jdl')
try:
jdlData = self.makeJDL(jobNum, module)
utils.safeWrite(os.fdopen(fd, 'w'), jdlData)
except Exception:
utils.removeFiles([jdl])
raise BackendError('Could not write jdl data to %s.' % jdl)
try:
submitArgs = []
for key_value in utils.filterDict(self._submitParams, vF = lambda v: v).items():
submitArgs.extend(key_value)
submitArgs.append(jdl)
activity = Activity('submitting job %d' % jobNum)
proc = LocalProcess(self._submitExec, '--nomsg', '--noint', '--logfile', '/dev/stderr', *submitArgs)
gcID = None
for line in ifilter(lambda x: x.startswith('http'), imap(str.strip, proc.stdout.iter(timeout = 60))):
gcID = line
retCode = proc.status(timeout = 0, terminate = True)
activity.finish()
if (retCode != 0) or (gcID is None):
if self.explainError(proc, retCode):
pass
else:
self._log.log_process(proc, files = {'jdl': SafeFile(jdl).read()})
finally:
utils.removeFiles([jdl])
return (jobNum, utils.QM(gcID, self._createId(gcID), None), {'jdl': str.join('', jdlData)})
def _submitJob(self, jobNum, module):
fd, jdl = tempfile.mkstemp('.jdl')
try:
data = self.makeJDL(jobNum, module)
utils.safeWrite(os.fdopen(fd, 'w'), data)
except Exception:
utils.removeFiles([jdl])
raise BackendError('Could not write jdl data to %s.' % jdl)
try:
tmp = utils.filterDict(self._submitParams, vF = lambda v: v)
params = str.join(' ', map(lambda (x, y): '%s %s' % (x, y), tmp.items()))
log = tempfile.mktemp('.log')
activity = utils.ActivityLog('submitting jobs')
proc = utils.LoggedProcess(self._submitExec, '%s --nomsg --noint --logfile "%s" "%s"' % (params, log, jdl))
wmsId = None
for line in filter(lambda x: x.startswith('http'), map(str.strip, proc.iter())):
wmsId = line
retCode = proc.wait()
del activity
if (retCode != 0) or (wmsId == None):
if self.explainError(proc, retCode):
pass
else:
proc.logError(self.errorLog, log = log, jdl = jdl)
finally:
utils.removeFiles([log, jdl])
return (jobNum, utils.QM(wmsId, self._createId(wmsId), None), {'jdl': str.join('', data)})
def _parseStatusX(self, lines):
adder = lambda a, b: utils.QM('=====' not in b and b != '\n', a + b, a)
remap = {
'destination': 'dest',
'status reason': 'reason',
'status info for the job': 'id',
'current status': 'status',
'submitted': 'timestamp',
'reached': 'timestamp',
'exit code': 'gridexit'
}
for section in utils.accumulate(lines,
lambda x, buf: ('=' * 70) in x,
'',
opAdd=adder):
data = utils.DictFormat(':').parse(
str.join('', section),
keyParser={None: lambda k: remap.get(k, str)})
data = utils.filterDict(data, vF=lambda v: v)
if data:
try:
if 'failed' in data['status']:
data['status'] = 'failed'
else:
data['status'] = data['status'].split()[0].lower()
except Exception:
pass
try:
data['timestamp'] = int(
time.mktime(parsedate(data['timestamp'])))
except Exception:
pass
yield data
def report(self, jobNum):
info = self.source.getJobInfo(jobNum)
keys = filter(lambda k: k.untracked == False, self.source.getJobKeys())
result = utils.filterDict(info, kF = lambda k: k in keys)
if self.dataSplitter:
result.pop('DATASETSPLIT')
result['Dataset'] = info.get('DATASETNICK', info.get('DATASETPATH', None))
elif not keys:
result[' '] = 'All jobs'
return result
def getJobInfo(self, jobNum, pNum = None):
if pNum is None:
pNum = jobNum
if jobNum is None:
raise APIError('Unable to process jobNum None!')
result = {ParameterInfo.ACTIVE: True, ParameterInfo.REQS: []}
result['GC_JOB_ID'] = jobNum
result['GC_PARAM'] = pNum
self._source.fillParameterInfo(pNum, result)
return utils.filterDict(result, vF = lambda v: v != '')
def report(self, jobNum):
info = self.source.getJobInfo(jobNum)
keys = lfilter(lambda k: not k.untracked, self.source.getJobKeys())
result = utils.filterDict(info, kF = lambda k: k in keys)
if self.dataSplitter:
result.pop('DATASETSPLIT')
result['Dataset'] = info.get('DATASETNICK', info.get('DATASETPATH', None))
elif not keys:
result[' '] = 'All jobs'
return result
def getEntries(self, path, metadata, events, seList, objStore): newVerbosity = utils.verbosity(utils.verbosity() - 3) if 'GC_TASK' in objStore: tmp = dict(objStore['GC_TASK'].getTaskConfig()) if 'GC_JOBNUM' in metadata: tmp.update(objStore['GC_TASK'].getJobConfig(metadata['GC_JOBNUM'])) for (newKey, oldKey) in objStore['GC_TASK'].getVarMapping().items(): tmp[newKey] = tmp.get(oldKey) metadata.update(utils.filterDict(tmp, kF = lambda k: k not in self.ignoreVars)) utils.verbosity(newVerbosity + 3) yield (path, metadata, events, seList, objStore)
def getEntries(self, path, metadata, events, seList, objStore):
if 'GC_TASK' in objStore:
tmp = dict(objStore['GC_TASK'].getTaskConfig())
if 'GC_JOBNUM' in metadata:
tmp.update(objStore['GC_TASK'].getJobConfig(
metadata['GC_JOBNUM']))
for (newKey,
oldKey) in objStore['GC_TASK'].getVarMapping().items():
tmp[newKey] = tmp.get(oldKey)
metadata.update(
utils.filterDict(tmp, kF=lambda k: k not in self._ignoreVars))
yield (path, metadata, events, seList, objStore)
def getJobConfig(self, jobNum):
data = CMSSW.getJobConfig(self, jobNum)
nickdata = self.getVarsForNick(data.get("DATASETNICK"))
data.update(nickdata)
data["LUMI_RANGE"] = self.getActiveLumiFilter(data["LUMI_RANGE"], jobNum)
if utils.verbosity() > 0:
utils.vprint("Nickname: %s" % data.get("DATASETNICK"), 1)
utils.vprint(" * Config files: %s" % data["CMSSW_CONFIG"], 1)
utils.vprint(" * Lumi range: %s" % data["LUMI_RANGE"], 1)
utils.vprint(
" * Variables: %s" % utils.filterDict(nickdata, lambda k: k not in ["CMSSW_CONFIG", "LUMI_RANGE"]), 1
)
return data
def _publish(self, jobObj, jobNum, taskId, usermsg):
(_, backend, rawId) = jobObj.gcID.split('.', 2)
dashId = '%s_%s' % (jobNum, rawId)
if 'http' not in jobObj.gcID:
dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId)
msg = mergeDicts([{
'taskId': taskId,
'jobId': dashId,
'sid': rawId
}] + usermsg)
DashboardAPI(
taskId,
dashId).publish(**filterDict(msg, vF=lambda v: v is not None))
def execute(self, wmsIDs): # yields list of (wmsID, job_status, job_info)
ec = ExceptionCollector()
for wmsID in wmsIDs:
try:
job_info = utils.filterDict(dict(self._status_fun(wmsID)), vF = lambda v: v not in ['', '0'])
job_info[CheckInfo.RAW_STATUS] = job_info.pop('status', '').lower()
if 'destination' in job_info:
try:
dest_info = job_info['destination'].split('/', 1)
job_info[CheckInfo.SITE] = dest_info[0].strip()
job_info[CheckInfo.QUEUE] = dest_info[1].strip()
except Exception:
clear_current_exception()
yield (wmsID, self._status_map.get(job_info[CheckInfo.RAW_STATUS], Job.UNKNOWN), job_info)
except Exception:
ec.collect()
if utils.abort():
break
ec.raise_any(BackendError('Encountered errors while checking job status'))
def checkJobsDirect(self, ids):
if len(ids) == 0:
raise StopIteration
activity = utils.ActivityLog('checking job status')
errors = []
for (wmsId, jobNum) in ids:
try:
data = utils.filterDict(dict(getStatusDirect(self._splitId(wmsId)[0])), vF = lambda v: (v != '') and (v != '0'))
data['id'] = self._createId(data.get('jobid', wmsId))
data['dest'] = data.get('destination', 'N/A')
yield (jobNum, data['id'], self._statusMap[data['status'].lower()], data)
except:
errors.append(repr(sys.exc_info()[1]))
if utils.abort():
break
del activity
if errors:
utils.eprint('The following glite errors have occured:\n%s' % str.join('\n', errors))
def _parseStatusX(self, lines):
adder = lambda a, b: utils.QM('=====' not in b and b != '\n', a + b, a)
remap = { 'destination': 'dest', 'status reason': 'reason',
'status info for the job': 'id', 'current status': 'status',
'submitted': 'timestamp', 'reached': 'timestamp', 'exit code': 'gridexit' }
for section in utils.accumulate(lines, lambda x, buf: ('='*70) in x, '', opAdd = adder):
data = utils.DictFormat(':').parse(str.join('', section), keyParser = {None: lambda k: remap.get(k, str)})
data = utils.filterDict(data, vF = lambda v: v)
if data:
try:
if 'failed' in data['status']:
data['status'] = 'failed'
else:
data['status'] = data['status'].split()[0].lower()
except Exception:
pass
try:
data['timestamp'] = int(time.mktime(parsedate(data['timestamp'])))
except Exception:
pass
yield data
def _submitJob(self, jobNum, module):
fd, jdl = tempfile.mkstemp('.jdl')
try:
jdlData = self.makeJDL(jobNum, module)
utils.safeWrite(os.fdopen(fd, 'w'), jdlData)
except Exception:
utils.removeFiles([jdl])
raise BackendError('Could not write jdl data to %s.' % jdl)
try:
submitArgs = []
for key_value in utils.filterDict(self._submitParams,
vF=lambda v: v).items():
submitArgs.extend(key_value)
submitArgs.append(jdl)
activity = Activity('submitting job %d' % jobNum)
proc = LocalProcess(self._submitExec, '--nomsg', '--noint',
'--logfile', '/dev/stderr', *submitArgs)
gcID = None
for line in ifilter(lambda x: x.startswith('http'),
imap(str.strip, proc.stdout.iter(timeout=60))):
gcID = line
retCode = proc.status(timeout=0, terminate=True)
activity.finish()
if (retCode != 0) or (gcID is None):
if self.explainError(proc, retCode):
pass
else:
self._log.log_process(proc,
files={'jdl': SafeFile(jdl).read()})
finally:
utils.removeFiles([jdl])
return (jobNum, utils.QM(gcID, self._createId(gcID), None), {
'jdl': str.join('', jdlData)
})
def checkJobsDirect(self, ids):
if len(ids) == 0:
raise StopIteration
activity = utils.ActivityLog('checking job status')
errors = []
for (wmsId, jobNum) in ids:
try:
data = utils.filterDict(dict(
getStatusDirect(self._splitId(wmsId)[0])),
vF=lambda v: (v != '') and (v != '0'))
data['id'] = self._createId(data.get('jobid', wmsId))
data['dest'] = data.get('destination', 'N/A')
yield (jobNum, data['id'],
self._statusMap[data['status'].lower()], data)
except Exception:
errors.append(repr(sys.exc_info()[1]))
if utils.abort():
break
del activity
if errors:
utils.eprint('The following glite errors have occured:\n%s' %
str.join('\n', errors))
def execute(self, wmsIDs): # yields list of (wmsID, job_status, job_info)
ec = ExceptionCollector()
for wmsID in wmsIDs:
try:
job_info = utils.filterDict(dict(self._status_fun(wmsID)),
vF=lambda v: v not in ['', '0'])
job_info[CheckInfo.RAW_STATUS] = job_info.pop('status',
'').lower()
if 'destination' in job_info:
try:
dest_info = job_info['destination'].split('/', 1)
job_info[CheckInfo.SITE] = dest_info[0].strip()
job_info[CheckInfo.QUEUE] = dest_info[1].strip()
except Exception:
clear_current_exception()
yield (wmsID,
self._status_map.get(job_info[CheckInfo.RAW_STATUS],
Job.UNKNOWN), job_info)
except Exception:
ec.collect()
if utils.abort():
break
ec.raise_any(
BackendError('Encountered errors while checking job status'))
def report(self, jobNum): keys = lfilter(lambda k: not k.untracked, self.source.getJobKeys()) return utils.filterDict(self.source.getJobInfo(jobNum), kF = lambda k: k in keys)
def report(self, jobNum):
keys = lfilter(lambda k: not k.untracked, self.source.getJobKeys())
return utils.filterDict(self.source.getJobInfo(jobNum),
kF=lambda k: k in keys)