def setJobDetails(self):
"""
_setJobName_
Propagate Job Information from JobSpec to RunResDB
"""
self.config['JobSpecID'][0] = self.jobSpecNode.jobName
self.jobSpecNode.loadConfiguration()
cfgInt = self.jobSpecNode.cfgInterface
inpSrc = cfgInt.sourceParams
if self.config['Input'].has_key("MaxEvents"):
del self.config['Input']['MaxEvents']
self.config['Input']['MaxEvents'] = [cfgInt.maxEvents['input']]
if self.config['Input'].has_key("FirstRun"):
del self.config['Input']['FirstRun']
if inpSrc.has_key('firstRun'):
self.config['Input']['FirstRun'] = [inpSrc['firstRun']]
if self.config['Input'].has_key("SourceType"):
del self.config['Input']['SourceType']
self.config['Input']['SourceType'] = [cfgInt.sourceType]
self.config['Input']['InputFiles'] = []
inpFileList = cfgInt.inputFiles
for inpFile in inpFileList:
self.config['Input']['InputFiles'].append(
inpFile.replace("\'", "")
)
for modName, item in cfgInt.outputModules.items():
if item.get('catalog', None) == None:
continue
catalog = unquote(item['catalog'])
catalog = os.path.join(self.taskState.dir, catalog)
if not self.config['Output']['Catalogs'].has_key(modName):
self.config['Output']['Catalogs'][modName] = []
self.config['Output']['Catalogs'][modName].append(catalog)
# //
# // Now save the RunResDB with the updates
#//
newComponent = RunResComponent()
dictRep = {"%s" % self.taskState.taskName() : self.config }
newComponent.populate(dictRep)
targetFile = os.path.join(self.taskState.dir, "RunResDB.xml")
handle = open(targetFile, 'w')
dom = newComponent.makeDOMElement()
handle.write(dom.toprettyxml())
handle.close()
return
def loadRunResDB(self):
"""
_loadRunResDB_
If the RunResDB file exists, load it
"""
if not os.path.exists(self.runresdb):
return
improvNode = loadIMProvFile(self.runresdb)
self._RunResDB = RunResComponent()
self._RunResDB.children = improvNode.children
self.runresLoaded = True
dbDict = self._RunResDB.toDictionary()
self.taskAttrs['Name'] = dbDict.keys()[0]
tName = self.taskAttrs['Name']
self.taskAttrs['WorkflowSpecID'] = \
dbDict[tName]['WorkflowSpecID'][0]
self.taskAttrs['JobSpecID'] = \
dbDict[tName]['JobSpecID'][0]
self.taskAttrs['JobType'] = \
dbDict[tName]['JobType'][0]
if not dbDict[tName].has_key('SizeBasedMerge'):
return
doSizeMerge = dbDict[tName]['SizeBasedMerge'].get("DoSizeMerge", [])
if len(doSizeMerge) > 0:
if str(doSizeMerge[0]).lower() == "true":
self.taskAttrs["DoSizeMerge"] = True
mergeSize = dbDict[tName]['SizeBasedMerge'].get("MinMergeFileSize", [])
if len(mergeSize) > 0:
size = int(mergeSize[0])
self.taskAttrs["MinMergeFileSize"] = size
return
def loadRunResDB(self):
"""
_loadRunResDB_
If the RunResDB file exists, load it
"""
if not os.path.exists(self.runresdb):
return
improvNode = loadIMProvFile(self.runresdb)
self._RunResDB = RunResComponent()
self._RunResDB.children = improvNode.children
self.runresLoaded = True
dbDict = self._RunResDB.toDictionary()
self.taskAttrs['Name'] = dbDict.keys()[0]
tName = self.taskAttrs['Name']
self.taskAttrs['WorkflowSpecID'] = \
dbDict[tName]['WorkflowSpecID'][0]
self.taskAttrs['JobSpecID'] = \
dbDict[tName]['JobSpecID'][0]
self.taskAttrs['JobType'] = \
dbDict[tName]['JobType'][0]
if not dbDict[tName].has_key('SizeBasedMerge'):
return
doSizeMerge = dbDict[tName]['SizeBasedMerge'].get("DoSizeMerge", [])
if len(doSizeMerge) > 0:
if str(doSizeMerge[0]).lower() == "true":
self.taskAttrs["DoSizeMerge"] = True
mergeSize = dbDict[tName]['SizeBasedMerge'].get("MinMergeFileSize", [])
if len(mergeSize) > 0:
size = int(mergeSize[0])
self.taskAttrs["MinMergeFileSize"] = size
return
class TaskState:
"""
_TaskState_
API object for extracting information from a CMSSW Task from the
components of that task including the RunResDB and FrameworkJobReport
"""
def __init__(self, taskDirectory):
self.dir = taskDirectory
self.jobReport = os.path.join(self.dir, "FrameworkJobReport.xml")
self.runresdb = os.path.join(self.dir, "RunResDB.xml")
self.jobSpecNode = None
self.jobSpec = None
self.taskAttrs = {}
self.taskAttrs.setdefault("Name", None)
self.taskAttrs.setdefault("CfgFile", None)
self.taskAttrs.setdefault("PyCfgFile", None)
self.taskAttrs.setdefault("WorkflowSpecID", None)
self.taskAttrs.setdefault("JobSpecID", None)
self.taskAttrs.setdefault("JobType", None)
self.taskAttrs.setdefault("DoSizeMerge", False)
self.taskAttrs.setdefault("MinMergeFileSize", 2000000000)
self._RunResDB = None
self._JobReport = None
self._CatalogEntries = None
self._SiteConfig = None
self.runresLoaded = False
self.jobReportLoaded = False
self.catalogsLoaded = False
self.siteConfigLoaded = False
self.jobSpecLoaded = False
self.parentsForwarded = []
def taskName(self):
"""
_taskName_
get the task name attribute
"""
return self.taskAttrs['Name']
def loadRunResDB(self):
"""
_loadRunResDB_
If the RunResDB file exists, load it
"""
if not os.path.exists(self.runresdb):
return
improvNode = loadIMProvFile(self.runresdb)
self._RunResDB = RunResComponent()
self._RunResDB.children = improvNode.children
self.runresLoaded = True
dbDict = self._RunResDB.toDictionary()
self.taskAttrs['Name'] = dbDict.keys()[0]
tName = self.taskAttrs['Name']
self.taskAttrs['WorkflowSpecID'] = \
dbDict[tName]['WorkflowSpecID'][0]
self.taskAttrs['JobSpecID'] = \
dbDict[tName]['JobSpecID'][0]
self.taskAttrs['JobType'] = \
dbDict[tName]['JobType'][0]
if not dbDict[tName].has_key('SizeBasedMerge'):
return
doSizeMerge = dbDict[tName]['SizeBasedMerge'].get("DoSizeMerge", [])
if len(doSizeMerge) > 0:
if str(doSizeMerge[0]).lower() == "true":
self.taskAttrs["DoSizeMerge"] = True
mergeSize = dbDict[tName]['SizeBasedMerge'].get("MinMergeFileSize", [])
if len(mergeSize) > 0:
size = int(mergeSize[0])
self.taskAttrs["MinMergeFileSize"] = size
return
def loadJobSpecNode(self):
"""
_loadJobSpecNode_
Load the job spec file referenced by PRODAGENT_JOB_SPEC env var and extract the node
from it with the name provided
"""
if not os.environ.has_key("PRODAGENT_JOBSPEC"):
print " No PRODAGENT_JOBSPEC set"
return
specFile = os.environ['PRODAGENT_JOBSPEC']
if not os.path.exists(specFile):
print "Job Spec File %s does not exist" % specFile
return
jobSpec = JobSpec()
jobSpec.load(specFile)
self.jobSpec = jobSpec
self.jobSpecNode = jobSpec.findNode(self.taskAttrs['Name'])
self.jobSpecLoaded = True
return
def configurationDict(self):
"""
_configurationDict_
Return the RunResDB for this task name as a dictionary
"""
try:
result = self._RunResDB.toDictionary()[self.taskName()]
except StandardError, ex:
result = {}
return result
class TaskState:
"""
_TaskState_
API object for extracting information from a CMSSW Task from the
components of that task including the RunResDB and FrameworkJobReport
"""
def __init__(self, taskDirectory):
self.dir = taskDirectory
self.jobReport = os.path.join(self.dir, "FrameworkJobReport.xml")
self.runresdb = os.path.join(self.dir, "RunResDB.xml")
self.jobSpecNode = None
self.jobSpec = None
self.taskAttrs = {}
self.taskAttrs.setdefault("Name", None)
self.taskAttrs.setdefault("CfgFile", None)
self.taskAttrs.setdefault("PyCfgFile", None)
self.taskAttrs.setdefault("WorkflowSpecID", None)
self.taskAttrs.setdefault("JobSpecID", None)
self.taskAttrs.setdefault("JobType", None)
self.taskAttrs.setdefault("DoSizeMerge", False)
self.taskAttrs.setdefault("MinMergeFileSize", 2000000000)
self._RunResDB = None
self._JobReport = None
self._CatalogEntries = None
self._SiteConfig = None
self.runresLoaded = False
self.jobReportLoaded = False
self.catalogsLoaded = False
self.siteConfigLoaded = False
self.jobSpecLoaded = False
self.parentsForwarded = []
def taskName(self):
"""
_taskName_
get the task name attribute
"""
return self.taskAttrs['Name']
def loadRunResDB(self):
"""
_loadRunResDB_
If the RunResDB file exists, load it
"""
if not os.path.exists(self.runresdb):
return
improvNode = loadIMProvFile(self.runresdb)
self._RunResDB = RunResComponent()
self._RunResDB.children = improvNode.children
self.runresLoaded = True
dbDict = self._RunResDB.toDictionary()
self.taskAttrs['Name'] = dbDict.keys()[0]
tName = self.taskAttrs['Name']
self.taskAttrs['WorkflowSpecID'] = \
dbDict[tName]['WorkflowSpecID'][0]
self.taskAttrs['JobSpecID'] = \
dbDict[tName]['JobSpecID'][0]
self.taskAttrs['JobType'] = \
dbDict[tName]['JobType'][0]
if not dbDict[tName].has_key('SizeBasedMerge'):
return
doSizeMerge = dbDict[tName]['SizeBasedMerge'].get("DoSizeMerge", [])
if len(doSizeMerge) > 0:
if str(doSizeMerge[0]).lower() == "true":
self.taskAttrs["DoSizeMerge"] = True
mergeSize = dbDict[tName]['SizeBasedMerge'].get("MinMergeFileSize", [])
if len(mergeSize) > 0:
size = int(mergeSize[0])
self.taskAttrs["MinMergeFileSize"] = size
return
def loadJobSpecNode(self):
"""
_loadJobSpecNode_
Load the job spec file referenced by PRODAGENT_JOB_SPEC env var and extract the node
from it with the name provided
"""
if not os.environ.has_key("PRODAGENT_JOBSPEC"):
print " No PRODAGENT_JOBSPEC set"
return
specFile = os.environ['PRODAGENT_JOBSPEC']
if not os.path.exists(specFile):
print "Job Spec File %s does not exist" % specFile
return
jobSpec = JobSpec()
jobSpec.load(specFile)
self.jobSpec = jobSpec
self.jobSpecNode = jobSpec.findNode(self.taskAttrs['Name'])
self.jobSpecLoaded = True
return
def configurationDict(self):
"""
_configurationDict_
Return the RunResDB for this task name as a dictionary
"""
try:
result = self._RunResDB.toDictionary()[self.taskName()]
except StandardError, ex:
result = {}
return result
def __call__(self, taskObject):
"""
_operator()_
Install the RunResDB key with a RunResComponent instance as the
value, add it to the list of XML IMProvNodes to be written out
Create basic structure in RunResComponent common to all tasks
"""
newComponent = RunResComponent()
objName = taskObject['Name']
workflow = taskObject['RequestName']
jobspec = taskObject['JobName']
jobtype = taskObject['JobType']
newComponent.addPath(objName)
newComponent.addData("/%s/WorkflowSpecID" % objName, workflow)
newComponent.addData("/%s/JobSpecID" % objName, jobspec)
newComponent.addData("/%s/JobType" % objName, jobtype)
newComponent.addPath("/%s/Type" % objName)
newComponent.addPath("/%s/Configuration" % objName)
newComponent.addPath("/%s/Application" % objName)
newComponent.addPath("/%s/Input" % objName)
newComponent.addPath("/%s/Output" % objName)
newComponent.addPath("/%s/SizeBasedMerge" % objName)
taskObject['RunResDB'] = newComponent
taskObject['IMProvDocs'].append("RunResDB")
return
