Пример #1
0
	def publish(self, jobObj, jobNum, taskId, usermsg):
		(header, backend, rawId) = jobObj.wmsId.split('.', 2)
		dashId = '%s_%s' % (jobNum, rawId)
		if "http" not in jobObj.wmsId:
			dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId)
		msg = utils.mergeDicts([{'taskId': taskId, 'jobId': dashId, 'sid': rawId}] + usermsg)
		DashboardAPI(taskId, dashId).publish(**utils.filterDict(msg, vF = lambda v: v != None))
Пример #2
0
 def getTaskConfig(self):
     taskConfig = {
         # Storage element
         'SE_MINFILESIZE':
         self.seMinSize,
         # Sandbox
         'SB_OUTPUT_FILES':
         str.join(' ', self.getSBOutFiles()),
         'SB_INPUT_FILES':
         str.join(' ', imap(lambda x: x.pathRel, self.getSBInFiles())),
         # Runtime
         'GC_JOBTIMEOUT':
         self.nodeTimeout,
         'GC_RUNTIME':
         self.getCommand(),
         # Seeds and substitutions
         'SUBST_FILES':
         str.join(' ', imap(os.path.basename, self.getSubstFiles())),
         'GC_SUBST_OLD_STYLE':
         str('__' in self._varCheck.markers).lower(),
         # Task infos
         'GC_TASK_CONF':
         self.taskConfigName,
         'GC_TASK_DATE':
         self.taskDate,
         'GC_TASK_ID':
         self.taskID,
         'GC_VERSION':
         utils.getVersion(),
     }
     return utils.mergeDicts([taskConfig, self.taskVariables])
Пример #3
0
 def getVarMapping(self):
     if self._dataSplitter:
         return utils.mergeDicts(
             [TaskModule.getVarMapping(self), {
                 'NICK': 'DATASETNICK'
             }])
     return TaskModule.getVarMapping(self)
Пример #4
0
	def substVars(self, inp, jobNum = None, addDict = {}, check = True):
		allVars = utils.mergeDicts([addDict, self.getTaskConfig()])
		if jobNum != None:
			allVars.update(self.getJobConfig(jobNum))
		subst = lambda x: utils.replaceDict(x, allVars, self.getVarMapping().items() + zip(addDict, addDict))
		result = subst(subst(str(inp)))
		return utils.checkVar(result, "'%s' contains invalid variable specifiers: '%s'" % (inp, result), check)
Пример #5
0
	def _publish(self, jobObj, jobNum, taskId, usermsg):
		(_, backend, rawId) = jobObj.gcID.split('.', 2)
		dashId = '%s_%s' % (jobNum, rawId)
		if 'http' not in jobObj.gcID:
			dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId)
		msg = mergeDicts([{'taskId': taskId, 'jobId': dashId, 'sid': rawId}] + usermsg)
		DashboardAPI(taskId, dashId).publish(**filterDict(msg, vF = lambda v: v is not None))
Пример #6
0
 def getTaskConfig(self):
     tmp = {
         'GC_MONITORING':
         str.join(' ', imap(os.path.basename, self.getScript()))
     }
     return utils.mergeDicts(
         lmap(lambda m: m.getTaskConfig(), self._handlers) + [tmp])
Пример #7
0
	def _writeJobConfig(self, cfgPath, jobNum, task, extras):
		try:
			jobEnv = utils.mergeDicts([task.getJobConfig(jobNum), extras])
			jobEnv['GC_ARGS'] = task.getJobArguments(jobNum).strip()
			content = utils.DictFormat(escapeString = True).format(jobEnv, format = 'export %s%s%s\n')
			utils.safeWrite(open(cfgPath, 'w'), content)
		except Exception:
			raise BackendError('Could not write job config data to %s.' % cfgPath)
Пример #8
0
	def _writeJobConfig(self, cfgPath, jobNum, module, extras = {}):
		try:
			jobEnv = utils.mergeDicts([module.getJobConfig(jobNum), extras])
			jobEnv['GC_ARGS'] = module.getJobArguments(jobNum).strip()
			content = utils.DictFormat(escapeString = True).format(jobEnv, format = 'export %s%s%s\n')
			utils.safeWrite(open(cfgPath, 'w'), content)
		except:
			raise RethrowError('Could not write job config data to %s.' % cfgPath)
Пример #9
0
 def report():
     for nick in sorted(set(self.nmCfg.keys() + self.nmConst.keys() + self.nmLumi.keys())):
         tmp = {
             0: nick,
             1: str.join(", ", map(os.path.basename, self.nmCfg.get(nick, ""))),
             2: self.displayLumi(self.nmLumi.get(nick, "")),
         }
         yield utils.mergeDicts([tmp, self.nmConst.get(nick, {})])
Пример #10
0
 def getSubmitInfo(self, jobNum):
     jobInfo = self.source.getJobInfo(jobNum)
     submitInfo = {
         'nevtJob': jobInfo.get('MAX_EVENTS', 0),
         'datasetFull': jobInfo.get('DATASETPATH', 'none')
     }
     return utils.mergeDicts(
         [TaskModule.getSubmitInfo(self, jobNum), submitInfo])
Пример #11
0
 def getVarsForNick(self, nick):
     data = {"CMSSW_CONFIG": str.join(" ", map(os.path.basename, utils.flatten(fromNM(self.nmCfg, nick, ""))))}
     constants = utils.mergeDicts(fromNM(self.nmConst, None, {}) + fromNM(self.nmConst, nick, {}))
     constants = dict(map(lambda var: (var, constants.get(var, "")), self.nmCName))
     data.update(constants)
     lumifilter = utils.flatten(fromNM(self.nmLumi, nick, ""))
     if lumifilter:
         data["LUMI_RANGE"] = parseLumiFilter(str.join(",", lumifilter))
     return data
Пример #12
0
	def substVars(self, inp, jobNum = None, addDict = None, check = True):
		addDict = addDict or {}
		allVars = utils.mergeDicts([addDict, self.getTaskConfig()])
		if jobNum is not None:
			allVars.update(self.getJobConfig(jobNum))
		subst = lambda x: utils.replaceDict(x, allVars, ichain([self.getVarMapping().items(), izip(addDict, addDict)]))
		result = subst(subst(str(inp)))
		if check and self._varCheck.check(result):
			raise ConfigError("'%s' contains invalid variable specifiers: '%s'" % (inp, result))
		return result
Пример #13
0
	def substVars(self, name, inp, jobNum = None, addDict = None, check = True):
		addDict = addDict or {}
		allVars = utils.mergeDicts([addDict, self.getTaskConfig()])
		if jobNum is not None:
			allVars.update(self.getJobConfig(jobNum))
		subst = lambda x: utils.replaceDict(x, allVars, ichain([self.getVarMapping().items(), izip(addDict, addDict)]))
		result = subst(subst(str(inp)))
		if check and self._varCheck.check(result):
			raise ConfigError('%s references unknown variables: %s' % (name, result))
		return result
Пример #14
0
 def _writeJobConfig(self, cfgPath, jobNum, task, extras):
     try:
         jobEnv = utils.mergeDicts([task.getJobConfig(jobNum), extras])
         jobEnv['GC_ARGS'] = task.getJobArguments(jobNum).strip()
         content = utils.DictFormat(escapeString=True).format(
             jobEnv, format='export %s%s%s\n')
         utils.safeWrite(open(cfgPath, 'w'), content)
     except Exception:
         raise BackendError('Could not write job config data to %s.' %
                            cfgPath)
Пример #15
0
 def _publish(self, jobObj, jobNum, taskId, usermsg):
     (_, backend, rawId) = jobObj.gcID.split('.', 2)
     dashId = '%s_%s' % (jobNum, rawId)
     if 'http' not in jobObj.gcID:
         dashId = '%s_https://%s:/%s' % (jobNum, backend, rawId)
     msg = mergeDicts([{
         'taskId': taskId,
         'jobId': dashId,
         'sid': rawId
     }] + usermsg)
     DashboardAPI(
         taskId,
         dashId).publish(**filterDict(msg, vF=lambda v: v is not None))
Пример #16
0
    def _getSandboxFiles(self, task, monitor, smList):
        # Prepare all input files
        depList = set(
            ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
        depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg),
                        os.listdir(utils.pathPKG()))
        depFiles = lmap(
            lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths),
            depList)
        taskEnv = utils.mergeDicts(
            imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
        taskEnv.update({
            'GC_DEPFILES': str.join(' ', depList),
            'GC_USERNAME': self._token.getUsername(),
            'GC_WMS_NAME': self.wmsName
        })
        taskConfig = sorted(
            utils.DictFormat(escapeString=True).format(
                taskEnv, format='export %s%s%s\n'))
        varMappingDict = dict(
            izip(monitor.getTaskConfig().keys(),
                 monitor.getTaskConfig().keys()))
        varMappingDict.update(task.getVarMapping())
        varMapping = sorted(
            utils.DictFormat(delimeter=' ').format(varMappingDict,
                                                   format='%s%s%s\n'))

        # Resolve wildcards in task input files
        def getTaskFiles():
            for f in task.getSBInFiles():
                matched = glob.glob(f.pathAbs)
                if matched != []:
                    for match in matched:
                        yield match
                else:
                    yield f.pathAbs

        return lchain([
            monitor.getFiles(), depFiles,
            getTaskFiles(),
            [
                VirtualFile('_config.sh', taskConfig),
                VirtualFile('_varmap.dat', varMapping)
            ]
        ])
Пример #17
0
	def _writeJobConfig(self, cfgPath, jobNum, task, extras):
		try:
			jobEnv = utils.mergeDicts([task.getJobConfig(jobNum), extras])
			jobEnv['GC_ARGS'] = task.getJobArguments(jobNum).strip()
			
			# write $FILE_NAMES into file in case [wms] file names environment = False
			# This can help avoiding too large environments
			# TODO: send fileNames.job_%d.txt together with the sandbox
			if ("FILE_NAMES" in jobEnv) and (not self.fileNamesEnvironment):
				fileNamesList = os.path.join(self.config.getPath('sandbox path', self.config.getWorkPath('sandbox')), "fileNames.job_%d.txt" % jobNum)
				with open(fileNamesList, "w") as fileNamesListFile:
					fileNamesListFile.write("\n".join(shlex.split(jobEnv.pop("FILE_NAMES"))))
				jobEnv["FILE_NAMES"] = fileNamesList
			
			content = utils.DictFormat(escapeString = True).format(jobEnv, format = 'export %s%s%s\n')
			utils.safeWrite(open(cfgPath, 'w'), content)
		except Exception:
			raise BackendError('Could not write job config data to %s.' % cfgPath)
Пример #18
0
	def getTaskConfig(self):
		taskConfig = {
			# Storage element
			'SE_MINFILESIZE': self.seMinSize,
			# Sandbox
			'SB_OUTPUT_FILES': str.join(' ', self.getSBOutFiles()),
			'SB_INPUT_FILES': str.join(' ', map(lambda x: x.pathRel, self.getSBInFiles())),
			# Runtime
			'GC_JOBTIMEOUT': self.nodeTimeout,
			'GC_RUNTIME': self.getCommand(),
			# Seeds and substitutions
			'SUBST_FILES': str.join(' ', map(os.path.basename, self.getSubstFiles())),
			# Task infos
			'GC_TASK_CONF': self.taskConfigName,
			'GC_TASK_DATE': self.taskDate,
			'GC_TASK_ID': self.taskID,
			'GC_VERSION': utils.getVersion(),
		}
		return utils.mergeDicts([taskConfig, self.taskVariables])
Пример #19
0
	def _getSandboxFiles(self, module, monitor, smList):
		# Prepare all input files
		depList = set(itertools.chain(*map(lambda x: x.getDependencies(), [module] + smList)))
		depPaths = map(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathGC('packages')))
		depFiles = map(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
		taskEnv = list(itertools.chain(map(lambda x: x.getTaskConfig(), [monitor, module] + smList)))
		taskEnv.append({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self.proxy.getUsername(),
			'GC_WMS_NAME': self.wmsName})
		taskConfig = sorted(utils.DictFormat(escapeString = True).format(utils.mergeDicts(taskEnv), format = 'export %s%s%s\n'))
		varMappingDict = dict(zip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
		varMappingDict.update(module.getVarMapping())
		varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
		# Resolve wildcards in module input files
		def getModuleFiles():
			for f in module.getSBInFiles():
				matched = glob.glob(f)
				if matched != []:
					for match in matched:
						yield match
				else:
					yield f
		return list(itertools.chain(monitor.getFiles(), depFiles, getModuleFiles(),
			[utils.VirtualFile('_config.sh', taskConfig), utils.VirtualFile('_varmap.dat', varMapping)]))
Пример #20
0
	def _getSandboxFiles(self, task, monitor, smList):
		# Prepare all input files
		depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
		depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG()))
		depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
		taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
		taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(),
			'GC_WMS_NAME': self._name})
		taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n'))
		varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
		varMappingDict.update(task.getVarMapping())
		varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
		# Resolve wildcards in task input files
		def getTaskFiles():
			for f in task.getSBInFiles():
				matched = glob.glob(f.pathAbs)
				if matched != []:
					for match in matched:
						yield match
				else:
					yield f.pathAbs
		return lchain([monitor.getFiles(), depFiles, getTaskFiles(),
			[VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
Пример #21
0
    def _writeJobConfig(self, cfgPath, jobNum, task, extras):
        try:
            jobEnv = utils.mergeDicts([task.getJobConfig(jobNum), extras])
            jobEnv['GC_ARGS'] = task.getJobArguments(jobNum).strip()

            # write $FILE_NAMES into file in case [wms] file names environment = False
            # This can help avoiding too large environments
            # TODO: send fileNames.job_%d.txt together with the sandbox
            if ("FILE_NAMES" in jobEnv) and (not self.fileNamesEnvironment):
                fileNamesList = os.path.join(
                    self.config.getPath('sandbox path',
                                        self.config.getWorkPath('sandbox')),
                    "fileNames.job_%d.txt" % jobNum)
                with open(fileNamesList, "w") as fileNamesListFile:
                    fileNamesListFile.write("\n".join(
                        shlex.split(jobEnv.pop("FILE_NAMES"))))
                jobEnv["FILE_NAMES"] = fileNamesList

            content = utils.DictFormat(escapeString=True).format(
                jobEnv, format='export %s%s%s\n')
            utils.safeWrite(open(cfgPath, 'w'), content)
        except Exception:
            raise BackendError('Could not write job config data to %s.' %
                               cfgPath)
Пример #22
0
	def getTaskConfig(self):
		tmp = {'GC_MONITORING': str.join(' ', imap(os.path.basename, self.getScript()))}
		return utils.mergeDicts(lmap(lambda m: m.getTaskConfig(), self._handlers) + [tmp])
Пример #23
0
	def getTaskConfig(self):
		tmp = {'GC_MONITORING': str.join(" ", map(os.path.basename, self.getScript()))}
		return utils.mergeDicts(map(lambda m: m.getTaskConfig(), self.submodules) + [tmp])
Пример #24
0
 def getTaskConfig(self):
     return utils.mergeDicts(
         [UserTask.getTaskConfig(self), {
             'GC_ROOTSYS': self._rootpath
         }])
Пример #25
0
	def getTaskConfig(self):
		return utils.mergeDicts([UserTask.getTaskConfig(self), {'GC_ROOTSYS': self._rootpath}])
Пример #26
0
	def getTaskConfig(self):
		tmp = Monitoring.getTaskConfig(self)
		return utils.mergeDicts(lmap(lambda m: m.getTaskConfig(), self._handlers) + [tmp])
Пример #27
0
	def getVarMapping(self):
		if self.dataSplitter:
			return utils.mergeDicts([TaskModule.getVarMapping(self), {'NICK': 'DATASETNICK'}])
		return TaskModule.getVarMapping(self)
Пример #28
0
	def getSubmitInfo(self, jobNum):
		jobInfo = self.source.getJobInfo(jobNum)
		submitInfo = {'nevtJob': jobInfo.get('MAX_EVENTS', 0),
			'datasetFull': jobInfo.get('DATASETPATH', 'none')}
		return utils.mergeDicts([TaskModule.getSubmitInfo(self, jobNum), submitInfo])
Пример #29
0
 def getTaskConfig(self):
     tmp = Monitoring.getTaskConfig(self)
     return utils.mergeDicts(
         lmap(lambda m: m.getTaskConfig(), self._handlers) + [tmp])