origcmsswsearch=os.environ.get("CMSSW_SEARCH_PATH", None) if not origcmsswsearch: msg = "CMSSW_SEARCH_PATH not set....you need CMSSW environment " raise RuntimeError, msg cmsswsearch="/:%s"%origcmsswsearch os.environ["CMSSW_SEARCH_PATH"]=cmsswsearch # // # // Instantiate a WorkflowMaker #// maker = WorkflowMaker(requestId, channel, label ) maker.setPhysicsGroup(physicsGroup) maker.changeCategory(category) if selectionEfficiency != None: maker.addSelectionEfficiency(selectionEfficiency) # loop over cfg's provided and add to workflow # first cmsRun node created implicitly by WorkflowMaker nodeNumber = 0 for cfgFile in cfgFiles: if cfgTypes[nodeNumber] == "cfg": from FWCore.ParameterSet.Config import include cmsCfg = include(cfgFile) else: import imp modRef = imp.load_source( os.path.basename(cfgFile).replace(".py", ""), cfgFile) cmsCfg = modRef.process cfgWrapper = CMSSWConfig()
def makeWorkflow(self, testInstance): """ _processTest_ Process a test, create a WorkflowSpec for it, generate job specs and add the, to the test instance """ loader = CMSSWAPILoader(testInstance['CMSSWArchitecture'], testInstance['CMSSWVersion'], testInstance['CMSPath']) loader.load() cfgWrapper = CMSSWConfig() process = pickle.load(file(testInstance['PickleFile'])) cfgInt = cfgWrapper.loadConfiguration(process) cfgInt.validateForProduction() cfgAsString = process.dumpPython() # // # // Get release validation PSet from process #// relValPSet = getattr(process, "ReleaseValidation", None) if relValPSet == None: msg = "Unable to extract ReleaseValidation PSet from pickled cfg for \n" msg += "%s\n" % testInstance['PickleFile'] logging.error(msg) return testName = getattr(relValPSet, "primaryDatasetName", None) testInstance['Name'] = testName.value() if testName == None: msg = "No primaryDatasetName parameter in ReleaseValidation PSet\n" msg += "%s\n" % testInstance['PickleFile'] logging.error(msg) return totalEvents = getattr(relValPSet, "totalNumberOfEvents", None) if totalEvents == None: msg = "No totalNumberOfEvents parameter in ReleaseValidation PSet\n" msg += "%s\n" % testInstance['PickleFile'] logging.error(msg) return testInstance['TotalEvents'] = totalEvents.value() eventsPerJob = getattr(relValPSet, "eventsPerJob", None) speedCat = getattr(relValPSet, "speedCategory", None) if (eventsPerJob == None) and (speedCat == None): msg = "ReleaseValidation PSet must contain one of either eventsPerJob or speedCategory\n" msg += "%s\n" % testInstance['PickleFile'] logging.error(msg) return if eventsPerJob != None: testInstance['EventsPerJob'] = eventsPerJob.value() else: testInstance['SpeedCategory'] = speedCat.value() if not self.args.has_key(testInstance['SpeedCategory']): msg = "Unknown Speed Category: %s\n" % testInstance['SpeedCategory'] msg += "In file: %s\n" % testInstance['PickleFile'] logging.error(msg) return testInstance['EventsPerJob'] = self.args[testInstance['SpeedCategory']] inputDataset = getattr(relValPSet, "inputDatasetPath", None) pileupDataset = getattr(relValPSet, "pileupDatasetPath", None) if pileupDataset != None: testInstance['PileupDataset'] = pileupDataset.value() if inputDataset != None: testInstance['InputDataset'] = inputDataset.value() msg = "Processing : %s\n" % testInstance['Name'] msg += "From Pickle: %s\n" % testInstance['PickleFile'] msg += "TotalEvents: %s\n" % testInstance['TotalEvents'] msg += "EventsPerJob: %s\n" % testInstance['EventsPerJob'] msg += "SpeedCategory: %s\n" % testInstance['SpeedCategory'] logging.info(msg) if self.workflows.has_key(testInstance['Name']): testInstance['WorkflowSpecId'] = self.workflows[testInstance['Name']] testInstance['WorkflowSpecFile'] = self.workflowFiles[testInstance['Name']] testInstance['WorkingDir'] = self.workingDirs[testInstance['Name']] loader.unload() return self.jobCounts[testInstance['Name']] = 1 workingDir = os.path.join(self.args['ComponentDir'], testInstance['CMSSWVersion'], testInstance['Name']) if not os.path.exists(workingDir): os.makedirs(workingDir) loader.unload() maker = WorkflowMaker(str(self.timestamp), testInstance['Name'], 'RelVal') maker.setCMSSWVersion(testInstance['CMSSWVersion']) maker.setPhysicsGroup("RelVal") maker.setConfiguration(cfgWrapper, Type = "instance") maker.setOriginalCfg(cfgAsString) psetHash = "NO_PSET_HASH" if cfgWrapper.configMetadata.has_key('PSetHash'): psetHash = cfgWrapper.configMetadata['PSetHash'] maker.setPSetHash(psetHash) maker.changeCategory("relval") if testInstance['SelectionEfficiency'] != None: selEff = float(testInstance['SelectionEfficiency'] ) maker.addSelectionEfficiency(selEff) if testInstance['PileupDataset'] != None: maker.addPileupDataset(testInstance['PileupDataset'], 100) if testInstance['InputDataset'] != None: maker.addInputDataset(testInstance['InputDataset']) maker.inputDataset["SplitType"] = "events" maker.inputDataset["SplitSize"] = testInstance['EventsPerJob'] spec = maker.makeWorkflow() spec.parameters['OnlySites'] = testInstance['Site'] spec.parameters['DBSURL'] = self.dbsUrl specFile = "/%s/%s-Workflow.xml" % (workingDir, maker.workflowName) spec.save(specFile) self.workflows[testInstance['Name']] = str(maker.workflowName) self.workflowFiles[testInstance['Name']] = specFile self.workingDirs[testInstance['Name']] = workingDir testInstance['WorkflowSpecId'] = str(maker.workflowName) testInstance['WorkflowSpecFile'] = specFile testInstance['WorkingDir'] = workingDir msg = "Workflow created for test: %s" % testInstance['Name'] logging.info(msg) msg = "Registering Workflow Entity: %s" % maker.workflowName logging.debug(msg) WEWorkflow.register( maker.workflowName, {"owner" : "RelValInjector", "workflow_spec_file" : specFile, }) msg = "Publishing NewWorkflow/NewDataset for \n" msg += " %s\n "% specFile logging.debug(msg) self.ms.publish("NewWorkflow", specFile) self.ms.publish("NewDataset", specFile) self.ms.commit() return