def createJobSpec(self, jobDef):
"""
_createJobSpec_
Load the WorkflowSpec object and generate a JobSpec from it
"""
jobSpec = self.workflowSpec.createJobSpec()
jobName = "%s-%s" % (self.workflowSpec.workflowName(), self.count)
logging.debug("Creating Job Spec: %s" % jobName)
self.currentJob = jobName
self.currentJobDef = jobDef
jobSpec.setJobName(jobName)
jobSpec.setJobType("Merge")
jobSpec.parameters['RunNumber'] = self.count
jobSpec.payload.operate(DefaultLFNMaker(jobSpec))
jobSpec.payload.operate(self.generateJobConfig)
specCacheDir = os.path.join(self.specCache,
str(self.count // 1000).zfill(4))
if not os.path.exists(specCacheDir):
os.makedirs(specCacheDir)
jobSpecFile = os.path.join(specCacheDir, "%s-JobSpec.xml" % jobName)
# //
# // Add site pref if set
#//
for site in jobDef['SENames']:
jobSpec.addWhitelistSite(site)
jobSpec.save(jobSpecFile)
return jobSpecFile
def createJobSpec(self, jobDef):
"""
_createJobSpec_
Load the WorkflowSpec object and generate a JobSpec from it
"""
jobSpec = self.workflowSpec.createJobSpec()
jobName = "%s-%s" % (
self.workflowSpec.workflowName(),
self.count,
)
logging.debug("Creating Job Spec: %s" % jobName)
self.currentJob = jobName
self.currentJobDef = jobDef
jobSpec.setJobName(jobName)
jobSpec.setJobType("Processing")
jobSpec.parameters['RunNumber'] = self.count
jobSpec.payload.operate(DefaultLFNMaker(jobSpec))
jobSpec.payload.operate(self.generateJobConfig)
self.firstNodeCfg = None # reset after job configured
jobSpec.payload.cfgInterface.rawCfg = None
jobSpec.payload.cfgInterface.originalCfg = None
specCacheDir = os.path.join(
self.specCache, str(self.count // 1000).zfill(4))
if not os.path.exists(specCacheDir):
os.makedirs(specCacheDir)
jobSpecFile = os.path.join(specCacheDir,
"%s-JobSpec.xml" % jobName)
# //
# // Add site pref if set
#//
if self.pileupSites:
whitelist = self.pileupSites
else:
whitelist = jobDef['SENames']
for site in whitelist:
jobSpec.addWhitelistSite(site)
self.pileupSites = [] # Resetting for next jobSpec
jobSpec.save(jobSpecFile)
return jobSpecFile
def createJobSpec(jobSpecId,
workflowSpecFile,
filename,
runNumber,
eventCount,
firstEvent=None,
saveString=False,
loadString=True):
# //
# // Load workflow
#//
workflowSpec = WorkflowSpec()
if loadString:
workflowSpec.loadString(workflowSpecFile)
else:
workflowSpec.load(workflowSpecFile)
# //
# // Create JobSpec
#//
jobSpec = workflowSpec.createJobSpec()
jobName = "%s-%s" % (workflowSpec.workflowName(), runNumber)
#jobSpec.setJobName(jobName)
jobSpec.setJobName(jobSpecId)
jobSpec.setJobType("Processing")
jobSpec.parameters['RunNumber'] = runNumber
jobSpec.parameters['EventCount'] = eventCount
jobSpec.payload.operate(DefaultLFNMaker(jobSpec))
if firstEvent != None:
jobSpec.parameters['FirstEvent'] = firstEvent
cfgMaker = ConfigGenerator(jobSpec)
jobSpec.payload.operate(cfgMaker)
if saveString:
return jobSpec.saveString()
jobSpec.save(filename)
return
def factoriseJobSpec(jobSpecInstance,
jobSpecDir,
njobs=[],
eventCount=0,
**args):
"""
_factoriseJobSpec_
njobs is an array of globally unique run numbers
TODO: <<<<NEEDS PILEUP DETAILS>>>>
"""
generators = GeneratorMaker()
jobSpecInstance.payload.operate(generators)
#AFgenerators(jobSpecInstance.payload)
runNumber = int(
args.get("RunNumber", int(jobSpecInstance.parameters['RunNumber'])))
firstEvent = int(args.get("FirstEvent", 0))
maxRunNumber = args.get("MaxRunNumber", None)
eventsPerJob = int(math.ceil(float(eventCount) / float(len(njobs))))
result = []
workflowName = jobSpecInstance.payload.workflow
template = jobSpecInstance.makeIMProv()
currentRun = runNumber
currentEvent = firstEvent
for run_number in njobs:
#jobName = "%s-%s" % (workflowName, run_number)
jobName = jobSpecInstance.parameters[
'JobName'] + '_jobcut-' + workflowName + '-' + str(run_number)
newSpec = JobSpec()
newSpec.loadFromNode(template)
newSpec.setJobName(jobName)
newSpec.parameters['RunNumber'] = run_number
newSpec.payload.operate(DefaultLFNMaker(newSpec))
maker = CfgMaker(generators,
JobName=jobName,
RunNumber=run_number,
MaxEvents=eventsPerJob,
SkipEvents=currentEvent)
newSpec.payload.operate(maker)
newSpec.payload.operate(maker.generateCmsGenConfig)
newSpec.parameters['FirstEvent'] = currentEvent
newSpec.parameters['RunNumber'] = run_number
newSpec.parameters['EventCount'] = eventsPerJob
jobSpecLocation = jobSpecDir + '/' + newSpec.parameters[
'JobName'] + '.xml'
newSpec.save(jobSpecLocation)
result.append({
'id': newSpec.parameters['JobName'],
'spec': jobSpecLocation,
'events': eventsPerJob
})
currentRun += 1
currentEvent += eventsPerJob
if ((eventsPerJob + currentEvent) > (firstEvent + int(eventCount))):
eventsPerJob = firstEvent + int(eventCount) - currentEvent
if maxRunNumber != None:
if currentRun > maxRunNumber:
break
return result