def createJobSpec(self):
"""
_createJobSpec_
Create a tree of JobSpecNodes from this WorkflowSpec
instances tree of payload nodes.
Same tree structure will be used, but nodes will be
JobSpecNode instances containing the details from the
corresponding PayloadNode in this instance.
To be used to create JobSpecNode trees from a Workflow
to represent a job created from the general workflow
"""
self._NodeMap = {}
result = JobSpec()
result.payload = self._CloneTreeNode(self.payload)
result.parameters.update(self.parameters)
return result
def loadJobSpecNode(self):
"""
_loadJobSpecNode_
Load the job spec file referenced by PRODAGENT_JOB_SPEC env var and extract the node
from it with the name provided
"""
if not os.environ.has_key("PRODAGENT_JOBSPEC"):
print " No PRODAGENT_JOBSPEC set"
return
specFile = os.environ['PRODAGENT_JOBSPEC']
if not os.path.exists(specFile):
print "Job Spec File %s does not exist" % specFile
return
jobSpec = JobSpec()
jobSpec.load(specFile)
self.jobSpec = jobSpec
self.jobSpecNode = jobSpec.findNode(self.taskAttrs['Name'])
self.jobSpecLoaded = True
return
def factoriseJobSpec(jobSpecInstance,
jobSpecDir,
njobs=[],
eventCount=0,
**args):
"""
_factoriseJobSpec_
njobs is an array of globally unique run numbers
TODO: <<<<NEEDS PILEUP DETAILS>>>>
"""
generators = GeneratorMaker()
jobSpecInstance.payload.operate(generators)
#AFgenerators(jobSpecInstance.payload)
runNumber = int(
args.get("RunNumber", int(jobSpecInstance.parameters['RunNumber'])))
firstEvent = int(args.get("FirstEvent", 0))
maxRunNumber = args.get("MaxRunNumber", None)
eventsPerJob = int(math.ceil(float(eventCount) / float(len(njobs))))
result = []
workflowName = jobSpecInstance.payload.workflow
template = jobSpecInstance.makeIMProv()
currentRun = runNumber
currentEvent = firstEvent
for run_number in njobs:
#jobName = "%s-%s" % (workflowName, run_number)
jobName = jobSpecInstance.parameters[
'JobName'] + '_jobcut-' + workflowName + '-' + str(run_number)
newSpec = JobSpec()
newSpec.loadFromNode(template)
newSpec.setJobName(jobName)
newSpec.parameters['RunNumber'] = run_number
newSpec.payload.operate(DefaultLFNMaker(newSpec))
maker = CfgMaker(generators,
JobName=jobName,
RunNumber=run_number,
MaxEvents=eventsPerJob,
SkipEvents=currentEvent)
newSpec.payload.operate(maker)
newSpec.payload.operate(maker.generateCmsGenConfig)
newSpec.parameters['FirstEvent'] = currentEvent
newSpec.parameters['RunNumber'] = run_number
newSpec.parameters['EventCount'] = eventsPerJob
jobSpecLocation = jobSpecDir + '/' + newSpec.parameters[
'JobName'] + '.xml'
newSpec.save(jobSpecLocation)
result.append({
'id': newSpec.parameters['JobName'],
'spec': jobSpecLocation,
'events': eventsPerJob
})
currentRun += 1
currentEvent += eventsPerJob
if ((eventsPerJob + currentEvent) > (firstEvent + int(eventCount))):
eventsPerJob = firstEvent + int(eventCount) - currentEvent
if maxRunNumber != None:
if currentRun > maxRunNumber:
break
return result
