def createJobSpec(self): """ _createJobSpec_ Create a tree of JobSpecNodes from this WorkflowSpec instances tree of payload nodes. Same tree structure will be used, but nodes will be JobSpecNode instances containing the details from the corresponding PayloadNode in this instance. To be used to create JobSpecNode trees from a Workflow to represent a job created from the general workflow """ self._NodeMap = {} result = JobSpec() result.payload = self._CloneTreeNode(self.payload) result.parameters.update(self.parameters) return result
def loadJobSpecNode(self): """ _loadJobSpecNode_ Load the job spec file referenced by PRODAGENT_JOB_SPEC env var and extract the node from it with the name provided """ if not os.environ.has_key("PRODAGENT_JOBSPEC"): print " No PRODAGENT_JOBSPEC set" return specFile = os.environ['PRODAGENT_JOBSPEC'] if not os.path.exists(specFile): print "Job Spec File %s does not exist" % specFile return jobSpec = JobSpec() jobSpec.load(specFile) self.jobSpec = jobSpec self.jobSpecNode = jobSpec.findNode(self.taskAttrs['Name']) self.jobSpecLoaded = True return
def factoriseJobSpec(jobSpecInstance, jobSpecDir, njobs=[], eventCount=0, **args): """ _factoriseJobSpec_ njobs is an array of globally unique run numbers TODO: <<<<NEEDS PILEUP DETAILS>>>> """ generators = GeneratorMaker() jobSpecInstance.payload.operate(generators) #AFgenerators(jobSpecInstance.payload) runNumber = int( args.get("RunNumber", int(jobSpecInstance.parameters['RunNumber']))) firstEvent = int(args.get("FirstEvent", 0)) maxRunNumber = args.get("MaxRunNumber", None) eventsPerJob = int(math.ceil(float(eventCount) / float(len(njobs)))) result = [] workflowName = jobSpecInstance.payload.workflow template = jobSpecInstance.makeIMProv() currentRun = runNumber currentEvent = firstEvent for run_number in njobs: #jobName = "%s-%s" % (workflowName, run_number) jobName = jobSpecInstance.parameters[ 'JobName'] + '_jobcut-' + workflowName + '-' + str(run_number) newSpec = JobSpec() newSpec.loadFromNode(template) newSpec.setJobName(jobName) newSpec.parameters['RunNumber'] = run_number newSpec.payload.operate(DefaultLFNMaker(newSpec)) maker = CfgMaker(generators, JobName=jobName, RunNumber=run_number, MaxEvents=eventsPerJob, SkipEvents=currentEvent) newSpec.payload.operate(maker) newSpec.payload.operate(maker.generateCmsGenConfig) newSpec.parameters['FirstEvent'] = currentEvent newSpec.parameters['RunNumber'] = run_number newSpec.parameters['EventCount'] = eventsPerJob jobSpecLocation = jobSpecDir + '/' + newSpec.parameters[ 'JobName'] + '.xml' newSpec.save(jobSpecLocation) result.append({ 'id': newSpec.parameters['JobName'], 'spec': jobSpecLocation, 'events': eventsPerJob }) currentRun += 1 currentEvent += eventsPerJob if ((eventsPerJob + currentEvent) > (firstEvent + int(eventCount))): eventsPerJob = firstEvent + int(eventCount) - currentEvent if maxRunNumber != None: if currentRun > maxRunNumber: break return result