Ejemplo n.º 1
0
    def createJobSpec(self, jobDef):
        """
        _createJobSpec_

        Load the WorkflowSpec object and generate a JobSpec from it

        """
        jobSpec = self.workflowSpec.createJobSpec()
        jobName = "%s-%s" % (self.workflowSpec.workflowName(), self.count)

        logging.debug("Creating Job Spec: %s" % jobName)
        self.currentJob = jobName
        self.currentJobDef = jobDef
        jobSpec.setJobName(jobName)
        jobSpec.setJobType("Merge")
        jobSpec.parameters['RunNumber'] = self.count

        jobSpec.payload.operate(DefaultLFNMaker(jobSpec))
        jobSpec.payload.operate(self.generateJobConfig)

        specCacheDir = os.path.join(self.specCache,
                                    str(self.count // 1000).zfill(4))
        if not os.path.exists(specCacheDir):
            os.makedirs(specCacheDir)
        jobSpecFile = os.path.join(specCacheDir, "%s-JobSpec.xml" % jobName)

        #  //
        # // Add site pref if set
        #//
        for site in jobDef['SENames']:
            jobSpec.addWhitelistSite(site)

        jobSpec.save(jobSpecFile)

        return jobSpecFile
Ejemplo n.º 2
0
    def createJobSpec(self, jobDef):
        """
        _createJobSpec_

        Load the WorkflowSpec object and generate a JobSpec from it

        """
        jobSpec = self.workflowSpec.createJobSpec()
        jobName = "%s-%s" % (
            self.workflowSpec.workflowName(),
            self.count,
            )

        logging.debug("Creating Job Spec: %s" % jobName)
        self.currentJob = jobName
        self.currentJobDef = jobDef
        jobSpec.setJobName(jobName)
        jobSpec.setJobType("Processing")
        jobSpec.parameters['RunNumber'] = self.count


        jobSpec.payload.operate(DefaultLFNMaker(jobSpec))
        jobSpec.payload.operate(self.generateJobConfig)
        self.firstNodeCfg = None # reset after job configured

        jobSpec.payload.cfgInterface.rawCfg = None
        jobSpec.payload.cfgInterface.originalCfg = None

        specCacheDir =  os.path.join(
            self.specCache, str(self.count // 1000).zfill(4))
        if not os.path.exists(specCacheDir):
            os.makedirs(specCacheDir)
        jobSpecFile = os.path.join(specCacheDir,
                                   "%s-JobSpec.xml" % jobName)
        
        

        #  //
        # // Add site pref if set
        #//
        if self.pileupSites:
            whitelist = self.pileupSites
        else:
            whitelist = jobDef['SENames']
        for site in whitelist:
            jobSpec.addWhitelistSite(site)
        self.pileupSites = [] # Resetting for next jobSpec
            
        
        jobSpec.save(jobSpecFile)
        
        return jobSpecFile
Ejemplo n.º 3
0
def createJobSpec(jobSpecId,
                  workflowSpecFile,
                  filename,
                  runNumber,
                  eventCount,
                  firstEvent=None,
                  saveString=False,
                  loadString=True):

    #  //
    # // Load workflow
    #//
    workflowSpec = WorkflowSpec()
    if loadString:
        workflowSpec.loadString(workflowSpecFile)
    else:
        workflowSpec.load(workflowSpecFile)

    #  //
    # // Create JobSpec
    #//
    jobSpec = workflowSpec.createJobSpec()
    jobName = "%s-%s" % (workflowSpec.workflowName(), runNumber)

    #jobSpec.setJobName(jobName)
    jobSpec.setJobName(jobSpecId)
    jobSpec.setJobType("Processing")
    jobSpec.parameters['RunNumber'] = runNumber
    jobSpec.parameters['EventCount'] = eventCount

    jobSpec.payload.operate(DefaultLFNMaker(jobSpec))

    if firstEvent != None:
        jobSpec.parameters['FirstEvent'] = firstEvent

    cfgMaker = ConfigGenerator(jobSpec)
    jobSpec.payload.operate(cfgMaker)

    if saveString:
        return jobSpec.saveString()
    jobSpec.save(filename)
    return
Ejemplo n.º 4
0
def factoriseJobSpec(jobSpecInstance,
                     jobSpecDir,
                     njobs=[],
                     eventCount=0,
                     **args):
    """
    _factoriseJobSpec_

    njobs is an array of globally unique run numbers


    TODO: <<<<NEEDS PILEUP DETAILS>>>>
    
    """
    generators = GeneratorMaker()
    jobSpecInstance.payload.operate(generators)
    #AFgenerators(jobSpecInstance.payload)

    runNumber = int(
        args.get("RunNumber", int(jobSpecInstance.parameters['RunNumber'])))
    firstEvent = int(args.get("FirstEvent", 0))
    maxRunNumber = args.get("MaxRunNumber", None)

    eventsPerJob = int(math.ceil(float(eventCount) / float(len(njobs))))

    result = []

    workflowName = jobSpecInstance.payload.workflow

    template = jobSpecInstance.makeIMProv()

    currentRun = runNumber
    currentEvent = firstEvent

    for run_number in njobs:
        #jobName = "%s-%s" % (workflowName, run_number)
        jobName = jobSpecInstance.parameters[
            'JobName'] + '_jobcut-' + workflowName + '-' + str(run_number)
        newSpec = JobSpec()
        newSpec.loadFromNode(template)
        newSpec.setJobName(jobName)
        newSpec.parameters['RunNumber'] = run_number

        newSpec.payload.operate(DefaultLFNMaker(newSpec))
        maker = CfgMaker(generators,
                         JobName=jobName,
                         RunNumber=run_number,
                         MaxEvents=eventsPerJob,
                         SkipEvents=currentEvent)
        newSpec.payload.operate(maker)
        newSpec.payload.operate(maker.generateCmsGenConfig)

        newSpec.parameters['FirstEvent'] = currentEvent
        newSpec.parameters['RunNumber'] = run_number
        newSpec.parameters['EventCount'] = eventsPerJob

        jobSpecLocation = jobSpecDir + '/' + newSpec.parameters[
            'JobName'] + '.xml'
        newSpec.save(jobSpecLocation)

        result.append({
            'id': newSpec.parameters['JobName'],
            'spec': jobSpecLocation,
            'events': eventsPerJob
        })

        currentRun += 1
        currentEvent += eventsPerJob
        if ((eventsPerJob + currentEvent) > (firstEvent + int(eventCount))):
            eventsPerJob = firstEvent + int(eventCount) - currentEvent
        if maxRunNumber != None:
            if currentRun > maxRunNumber:
                break

    return result