def setJobDetails(self): """ _setJobName_ Propagate Job Information from JobSpec to RunResDB """ self.config['JobSpecID'][0] = self.jobSpecNode.jobName self.jobSpecNode.loadConfiguration() cfgInt = self.jobSpecNode.cfgInterface inpSrc = cfgInt.sourceParams if self.config['Input'].has_key("MaxEvents"): del self.config['Input']['MaxEvents'] self.config['Input']['MaxEvents'] = [cfgInt.maxEvents['input']] if self.config['Input'].has_key("FirstRun"): del self.config['Input']['FirstRun'] if inpSrc.has_key('firstRun'): self.config['Input']['FirstRun'] = [inpSrc['firstRun']] if self.config['Input'].has_key("SourceType"): del self.config['Input']['SourceType'] self.config['Input']['SourceType'] = [cfgInt.sourceType] self.config['Input']['InputFiles'] = [] inpFileList = cfgInt.inputFiles for inpFile in inpFileList: self.config['Input']['InputFiles'].append( inpFile.replace("\'", "") ) for modName, item in cfgInt.outputModules.items(): if item.get('catalog', None) == None: continue catalog = unquote(item['catalog']) catalog = os.path.join(self.taskState.dir, catalog) if not self.config['Output']['Catalogs'].has_key(modName): self.config['Output']['Catalogs'][modName] = [] self.config['Output']['Catalogs'][modName].append(catalog) # // # // Now save the RunResDB with the updates #// newComponent = RunResComponent() dictRep = {"%s" % self.taskState.taskName() : self.config } newComponent.populate(dictRep) targetFile = os.path.join(self.taskState.dir, "RunResDB.xml") handle = open(targetFile, 'w') dom = newComponent.makeDOMElement() handle.write(dom.toprettyxml()) handle.close() return
def loadRunResDB(self): """ _loadRunResDB_ If the RunResDB file exists, load it """ if not os.path.exists(self.runresdb): return improvNode = loadIMProvFile(self.runresdb) self._RunResDB = RunResComponent() self._RunResDB.children = improvNode.children self.runresLoaded = True dbDict = self._RunResDB.toDictionary() self.taskAttrs['Name'] = dbDict.keys()[0] tName = self.taskAttrs['Name'] self.taskAttrs['WorkflowSpecID'] = \ dbDict[tName]['WorkflowSpecID'][0] self.taskAttrs['JobSpecID'] = \ dbDict[tName]['JobSpecID'][0] self.taskAttrs['JobType'] = \ dbDict[tName]['JobType'][0] if not dbDict[tName].has_key('SizeBasedMerge'): return doSizeMerge = dbDict[tName]['SizeBasedMerge'].get("DoSizeMerge", []) if len(doSizeMerge) > 0: if str(doSizeMerge[0]).lower() == "true": self.taskAttrs["DoSizeMerge"] = True mergeSize = dbDict[tName]['SizeBasedMerge'].get("MinMergeFileSize", []) if len(mergeSize) > 0: size = int(mergeSize[0]) self.taskAttrs["MinMergeFileSize"] = size return
class TaskState: """ _TaskState_ API object for extracting information from a CMSSW Task from the components of that task including the RunResDB and FrameworkJobReport """ def __init__(self, taskDirectory): self.dir = taskDirectory self.jobReport = os.path.join(self.dir, "FrameworkJobReport.xml") self.runresdb = os.path.join(self.dir, "RunResDB.xml") self.jobSpecNode = None self.jobSpec = None self.taskAttrs = {} self.taskAttrs.setdefault("Name", None) self.taskAttrs.setdefault("CfgFile", None) self.taskAttrs.setdefault("PyCfgFile", None) self.taskAttrs.setdefault("WorkflowSpecID", None) self.taskAttrs.setdefault("JobSpecID", None) self.taskAttrs.setdefault("JobType", None) self.taskAttrs.setdefault("DoSizeMerge", False) self.taskAttrs.setdefault("MinMergeFileSize", 2000000000) self._RunResDB = None self._JobReport = None self._CatalogEntries = None self._SiteConfig = None self.runresLoaded = False self.jobReportLoaded = False self.catalogsLoaded = False self.siteConfigLoaded = False self.jobSpecLoaded = False self.parentsForwarded = [] def taskName(self): """ _taskName_ get the task name attribute """ return self.taskAttrs['Name'] def loadRunResDB(self): """ _loadRunResDB_ If the RunResDB file exists, load it """ if not os.path.exists(self.runresdb): return improvNode = loadIMProvFile(self.runresdb) self._RunResDB = RunResComponent() self._RunResDB.children = improvNode.children self.runresLoaded = True dbDict = self._RunResDB.toDictionary() self.taskAttrs['Name'] = dbDict.keys()[0] tName = self.taskAttrs['Name'] self.taskAttrs['WorkflowSpecID'] = \ dbDict[tName]['WorkflowSpecID'][0] self.taskAttrs['JobSpecID'] = \ dbDict[tName]['JobSpecID'][0] self.taskAttrs['JobType'] = \ dbDict[tName]['JobType'][0] if not dbDict[tName].has_key('SizeBasedMerge'): return doSizeMerge = dbDict[tName]['SizeBasedMerge'].get("DoSizeMerge", []) if len(doSizeMerge) > 0: if str(doSizeMerge[0]).lower() == "true": self.taskAttrs["DoSizeMerge"] = True mergeSize = dbDict[tName]['SizeBasedMerge'].get("MinMergeFileSize", []) if len(mergeSize) > 0: size = int(mergeSize[0]) self.taskAttrs["MinMergeFileSize"] = size return def loadJobSpecNode(self): """ _loadJobSpecNode_ Load the job spec file referenced by PRODAGENT_JOB_SPEC env var and extract the node from it with the name provided """ if not os.environ.has_key("PRODAGENT_JOBSPEC"): print " No PRODAGENT_JOBSPEC set" return specFile = os.environ['PRODAGENT_JOBSPEC'] if not os.path.exists(specFile): print "Job Spec File %s does not exist" % specFile return jobSpec = JobSpec() jobSpec.load(specFile) self.jobSpec = jobSpec self.jobSpecNode = jobSpec.findNode(self.taskAttrs['Name']) self.jobSpecLoaded = True return def configurationDict(self): """ _configurationDict_ Return the RunResDB for this task name as a dictionary """ try: result = self._RunResDB.toDictionary()[self.taskName()] except StandardError, ex: result = {} return result
def __call__(self, taskObject): """ _operator()_ Install the RunResDB key with a RunResComponent instance as the value, add it to the list of XML IMProvNodes to be written out Create basic structure in RunResComponent common to all tasks """ newComponent = RunResComponent() objName = taskObject['Name'] workflow = taskObject['RequestName'] jobspec = taskObject['JobName'] jobtype = taskObject['JobType'] newComponent.addPath(objName) newComponent.addData("/%s/WorkflowSpecID" % objName, workflow) newComponent.addData("/%s/JobSpecID" % objName, jobspec) newComponent.addData("/%s/JobType" % objName, jobtype) newComponent.addPath("/%s/Type" % objName) newComponent.addPath("/%s/Configuration" % objName) newComponent.addPath("/%s/Application" % objName) newComponent.addPath("/%s/Input" % objName) newComponent.addPath("/%s/Output" % objName) newComponent.addPath("/%s/SizeBasedMerge" % objName) taskObject['RunResDB'] = newComponent taskObject['IMProvDocs'].append("RunResDB") return