def publish(self, jobObj, jobNum, taskId, usermsg):
(header, backend, rawId) = jobObj.wmsId.split('.', 2)
dashId = '%s_%s' % (jobNum, rawId)
if "http" not in jobObj.wmsId:
dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId)
msg = utils.mergeDicts([{'taskId': taskId, 'jobId': dashId, 'sid': rawId}] + usermsg)
DashboardAPI(taskId, dashId).publish(**utils.filterDict(msg, vF = lambda v: v != None))
def getTaskConfig(self):
taskConfig = {
# Storage element
'SE_MINFILESIZE':
self.seMinSize,
# Sandbox
'SB_OUTPUT_FILES':
str.join(' ', self.getSBOutFiles()),
'SB_INPUT_FILES':
str.join(' ', imap(lambda x: x.pathRel, self.getSBInFiles())),
# Runtime
'GC_JOBTIMEOUT':
self.nodeTimeout,
'GC_RUNTIME':
self.getCommand(),
# Seeds and substitutions
'SUBST_FILES':
str.join(' ', imap(os.path.basename, self.getSubstFiles())),
'GC_SUBST_OLD_STYLE':
str('__' in self._varCheck.markers).lower(),
# Task infos
'GC_TASK_CONF':
self.taskConfigName,
'GC_TASK_DATE':
self.taskDate,
'GC_TASK_ID':
self.taskID,
'GC_VERSION':
utils.getVersion(),
}
return utils.mergeDicts([taskConfig, self.taskVariables])
def getVarMapping(self):
if self._dataSplitter:
return utils.mergeDicts(
[TaskModule.getVarMapping(self), {
'NICK': 'DATASETNICK'
}])
return TaskModule.getVarMapping(self)
def substVars(self, inp, jobNum = None, addDict = {}, check = True):
allVars = utils.mergeDicts([addDict, self.getTaskConfig()])
if jobNum != None:
allVars.update(self.getJobConfig(jobNum))
subst = lambda x: utils.replaceDict(x, allVars, self.getVarMapping().items() + zip(addDict, addDict))
result = subst(subst(str(inp)))
return utils.checkVar(result, "'%s' contains invalid variable specifiers: '%s'" % (inp, result), check)
def _publish(self, jobObj, jobNum, taskId, usermsg):
(_, backend, rawId) = jobObj.gcID.split('.', 2)
dashId = '%s_%s' % (jobNum, rawId)
if 'http' not in jobObj.gcID:
dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId)
msg = mergeDicts([{'taskId': taskId, 'jobId': dashId, 'sid': rawId}] + usermsg)
DashboardAPI(taskId, dashId).publish(**filterDict(msg, vF = lambda v: v is not None))
def getTaskConfig(self):
tmp = {
'GC_MONITORING':
str.join(' ', imap(os.path.basename, self.getScript()))
}
return utils.mergeDicts(
lmap(lambda m: m.getTaskConfig(), self._handlers) + [tmp])
def _writeJobConfig(self, cfgPath, jobNum, task, extras):
try:
jobEnv = utils.mergeDicts([task.getJobConfig(jobNum), extras])
jobEnv['GC_ARGS'] = task.getJobArguments(jobNum).strip()
content = utils.DictFormat(escapeString = True).format(jobEnv, format = 'export %s%s%s\n')
utils.safeWrite(open(cfgPath, 'w'), content)
except Exception:
raise BackendError('Could not write job config data to %s.' % cfgPath)
def _writeJobConfig(self, cfgPath, jobNum, module, extras = {}):
try:
jobEnv = utils.mergeDicts([module.getJobConfig(jobNum), extras])
jobEnv['GC_ARGS'] = module.getJobArguments(jobNum).strip()
content = utils.DictFormat(escapeString = True).format(jobEnv, format = 'export %s%s%s\n')
utils.safeWrite(open(cfgPath, 'w'), content)
except:
raise RethrowError('Could not write job config data to %s.' % cfgPath)
def report():
for nick in sorted(set(self.nmCfg.keys() + self.nmConst.keys() + self.nmLumi.keys())):
tmp = {
0: nick,
1: str.join(", ", map(os.path.basename, self.nmCfg.get(nick, ""))),
2: self.displayLumi(self.nmLumi.get(nick, "")),
}
yield utils.mergeDicts([tmp, self.nmConst.get(nick, {})])
def getSubmitInfo(self, jobNum):
jobInfo = self.source.getJobInfo(jobNum)
submitInfo = {
'nevtJob': jobInfo.get('MAX_EVENTS', 0),
'datasetFull': jobInfo.get('DATASETPATH', 'none')
}
return utils.mergeDicts(
[TaskModule.getSubmitInfo(self, jobNum), submitInfo])
def getVarsForNick(self, nick):
data = {"CMSSW_CONFIG": str.join(" ", map(os.path.basename, utils.flatten(fromNM(self.nmCfg, nick, ""))))}
constants = utils.mergeDicts(fromNM(self.nmConst, None, {}) + fromNM(self.nmConst, nick, {}))
constants = dict(map(lambda var: (var, constants.get(var, "")), self.nmCName))
data.update(constants)
lumifilter = utils.flatten(fromNM(self.nmLumi, nick, ""))
if lumifilter:
data["LUMI_RANGE"] = parseLumiFilter(str.join(",", lumifilter))
return data
def substVars(self, inp, jobNum = None, addDict = None, check = True):
addDict = addDict or {}
allVars = utils.mergeDicts([addDict, self.getTaskConfig()])
if jobNum is not None:
allVars.update(self.getJobConfig(jobNum))
subst = lambda x: utils.replaceDict(x, allVars, ichain([self.getVarMapping().items(), izip(addDict, addDict)]))
result = subst(subst(str(inp)))
if check and self._varCheck.check(result):
raise ConfigError("'%s' contains invalid variable specifiers: '%s'" % (inp, result))
return result
def substVars(self, name, inp, jobNum = None, addDict = None, check = True):
addDict = addDict or {}
allVars = utils.mergeDicts([addDict, self.getTaskConfig()])
if jobNum is not None:
allVars.update(self.getJobConfig(jobNum))
subst = lambda x: utils.replaceDict(x, allVars, ichain([self.getVarMapping().items(), izip(addDict, addDict)]))
result = subst(subst(str(inp)))
if check and self._varCheck.check(result):
raise ConfigError('%s references unknown variables: %s' % (name, result))
return result
def _writeJobConfig(self, cfgPath, jobNum, task, extras):
try:
jobEnv = utils.mergeDicts([task.getJobConfig(jobNum), extras])
jobEnv['GC_ARGS'] = task.getJobArguments(jobNum).strip()
content = utils.DictFormat(escapeString=True).format(
jobEnv, format='export %s%s%s\n')
utils.safeWrite(open(cfgPath, 'w'), content)
except Exception:
raise BackendError('Could not write job config data to %s.' %
cfgPath)
def _publish(self, jobObj, jobNum, taskId, usermsg):
(_, backend, rawId) = jobObj.gcID.split('.', 2)
dashId = '%s_%s' % (jobNum, rawId)
if 'http' not in jobObj.gcID:
dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId)
msg = mergeDicts([{
'taskId': taskId,
'jobId': dashId,
'sid': rawId
}] + usermsg)
DashboardAPI(
taskId,
dashId).publish(**filterDict(msg, vF=lambda v: v is not None))
def _getSandboxFiles(self, task, monitor, smList):
# Prepare all input files
depList = set(
ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg),
os.listdir(utils.pathPKG()))
depFiles = lmap(
lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths),
depList)
taskEnv = utils.mergeDicts(
imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
taskEnv.update({
'GC_DEPFILES': str.join(' ', depList),
'GC_USERNAME': self._token.getUsername(),
'GC_WMS_NAME': self.wmsName
})
taskConfig = sorted(
utils.DictFormat(escapeString=True).format(
taskEnv, format='export %s%s%s\n'))
varMappingDict = dict(
izip(monitor.getTaskConfig().keys(),
monitor.getTaskConfig().keys()))
varMappingDict.update(task.getVarMapping())
varMapping = sorted(
utils.DictFormat(delimeter=' ').format(varMappingDict,
format='%s%s%s\n'))
# Resolve wildcards in task input files
def getTaskFiles():
for f in task.getSBInFiles():
matched = glob.glob(f.pathAbs)
if matched != []:
for match in matched:
yield match
else:
yield f.pathAbs
return lchain([
monitor.getFiles(), depFiles,
getTaskFiles(),
[
VirtualFile('_config.sh', taskConfig),
VirtualFile('_varmap.dat', varMapping)
]
])
def _writeJobConfig(self, cfgPath, jobNum, task, extras):
try:
jobEnv = utils.mergeDicts([task.getJobConfig(jobNum), extras])
jobEnv['GC_ARGS'] = task.getJobArguments(jobNum).strip()
# write $FILE_NAMES into file in case [wms] file names environment = False
# This can help avoiding too large environments
# TODO: send fileNames.job_%d.txt together with the sandbox
if ("FILE_NAMES" in jobEnv) and (not self.fileNamesEnvironment):
fileNamesList = os.path.join(self.config.getPath('sandbox path', self.config.getWorkPath('sandbox')), "fileNames.job_%d.txt" % jobNum)
with open(fileNamesList, "w") as fileNamesListFile:
fileNamesListFile.write("\n".join(shlex.split(jobEnv.pop("FILE_NAMES"))))
jobEnv["FILE_NAMES"] = fileNamesList
content = utils.DictFormat(escapeString = True).format(jobEnv, format = 'export %s%s%s\n')
utils.safeWrite(open(cfgPath, 'w'), content)
except Exception:
raise BackendError('Could not write job config data to %s.' % cfgPath)
def getTaskConfig(self):
taskConfig = {
# Storage element
'SE_MINFILESIZE': self.seMinSize,
# Sandbox
'SB_OUTPUT_FILES': str.join(' ', self.getSBOutFiles()),
'SB_INPUT_FILES': str.join(' ', map(lambda x: x.pathRel, self.getSBInFiles())),
# Runtime
'GC_JOBTIMEOUT': self.nodeTimeout,
'GC_RUNTIME': self.getCommand(),
# Seeds and substitutions
'SUBST_FILES': str.join(' ', map(os.path.basename, self.getSubstFiles())),
# Task infos
'GC_TASK_CONF': self.taskConfigName,
'GC_TASK_DATE': self.taskDate,
'GC_TASK_ID': self.taskID,
'GC_VERSION': utils.getVersion(),
}
return utils.mergeDicts([taskConfig, self.taskVariables])
def _getSandboxFiles(self, module, monitor, smList):
# Prepare all input files
depList = set(itertools.chain(*map(lambda x: x.getDependencies(), [module] + smList)))
depPaths = map(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathGC('packages')))
depFiles = map(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
taskEnv = list(itertools.chain(map(lambda x: x.getTaskConfig(), [monitor, module] + smList)))
taskEnv.append({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self.proxy.getUsername(),
'GC_WMS_NAME': self.wmsName})
taskConfig = sorted(utils.DictFormat(escapeString = True).format(utils.mergeDicts(taskEnv), format = 'export %s%s%s\n'))
varMappingDict = dict(zip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
varMappingDict.update(module.getVarMapping())
varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
# Resolve wildcards in module input files
def getModuleFiles():
for f in module.getSBInFiles():
matched = glob.glob(f)
if matched != []:
for match in matched:
yield match
else:
yield f
return list(itertools.chain(monitor.getFiles(), depFiles, getModuleFiles(),
[utils.VirtualFile('_config.sh', taskConfig), utils.VirtualFile('_varmap.dat', varMapping)]))
def _getSandboxFiles(self, task, monitor, smList):
# Prepare all input files
depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG()))
depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(),
'GC_WMS_NAME': self._name})
taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n'))
varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
varMappingDict.update(task.getVarMapping())
varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
# Resolve wildcards in task input files
def getTaskFiles():
for f in task.getSBInFiles():
matched = glob.glob(f.pathAbs)
if matched != []:
for match in matched:
yield match
else:
yield f.pathAbs
return lchain([monitor.getFiles(), depFiles, getTaskFiles(),
[VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
def _writeJobConfig(self, cfgPath, jobNum, task, extras):
try:
jobEnv = utils.mergeDicts([task.getJobConfig(jobNum), extras])
jobEnv['GC_ARGS'] = task.getJobArguments(jobNum).strip()
# write $FILE_NAMES into file in case [wms] file names environment = False
# This can help avoiding too large environments
# TODO: send fileNames.job_%d.txt together with the sandbox
if ("FILE_NAMES" in jobEnv) and (not self.fileNamesEnvironment):
fileNamesList = os.path.join(
self.config.getPath('sandbox path',
self.config.getWorkPath('sandbox')),
"fileNames.job_%d.txt" % jobNum)
with open(fileNamesList, "w") as fileNamesListFile:
fileNamesListFile.write("\n".join(
shlex.split(jobEnv.pop("FILE_NAMES"))))
jobEnv["FILE_NAMES"] = fileNamesList
content = utils.DictFormat(escapeString=True).format(
jobEnv, format='export %s%s%s\n')
utils.safeWrite(open(cfgPath, 'w'), content)
except Exception:
raise BackendError('Could not write job config data to %s.' %
cfgPath)
def getTaskConfig(self):
tmp = {'GC_MONITORING': str.join(' ', imap(os.path.basename, self.getScript()))}
return utils.mergeDicts(lmap(lambda m: m.getTaskConfig(), self._handlers) + [tmp])
def getTaskConfig(self):
tmp = {'GC_MONITORING': str.join(" ", map(os.path.basename, self.getScript()))}
return utils.mergeDicts(map(lambda m: m.getTaskConfig(), self.submodules) + [tmp])
def getTaskConfig(self):
return utils.mergeDicts(
[UserTask.getTaskConfig(self), {
'GC_ROOTSYS': self._rootpath
}])
def getTaskConfig(self):
return utils.mergeDicts([UserTask.getTaskConfig(self), {'GC_ROOTSYS': self._rootpath}])
def getTaskConfig(self): tmp = Monitoring.getTaskConfig(self) return utils.mergeDicts(lmap(lambda m: m.getTaskConfig(), self._handlers) + [tmp])
def getVarMapping(self):
if self.dataSplitter:
return utils.mergeDicts([TaskModule.getVarMapping(self), {'NICK': 'DATASETNICK'}])
return TaskModule.getVarMapping(self)
def getSubmitInfo(self, jobNum):
jobInfo = self.source.getJobInfo(jobNum)
submitInfo = {'nevtJob': jobInfo.get('MAX_EVENTS', 0),
'datasetFull': jobInfo.get('DATASETPATH', 'none')}
return utils.mergeDicts([TaskModule.getSubmitInfo(self, jobNum), submitInfo])
def getTaskConfig(self):
tmp = Monitoring.getTaskConfig(self)
return utils.mergeDicts(
lmap(lambda m: m.getTaskConfig(), self._handlers) + [tmp])