def checkcreateWorkflow(cfgFile):
"""
"""
try:
# //
# // Set CMSSW_SEARCH_PATH
#//
pwd = getenv ("PWD")
oldCmsswSearchPath = getenv ("CMSSW_SEARCH_PATH")
if not oldCmsswSearchPath:
msg = "CMSSW_SEARCH_PATH not set....you need to set CMSSW environment "
raise RuntimeError, msg
#os.environ["CMSSW_SEARCH_PATH"] = "%s:%s" % (pwd, oldCmsswSearchPath)
os.environ["CMSSW_SEARCH_PATH"] = "/:%s" % (oldCmsswSearchPath)
# //
# // convert cfg
#//
# for the time being we support only cfg file. We might have to support multiple types soon
print ">>> Checking cfg %s"%cfgFile
if cfgType == "cfg":
from FWCore.ParameterSet.Config import include
cmsCfg = include(cfgFile)
else:
modRef = imp.find_module( os.path.basename(cfgFile).replace(".py", ""), os.path.dirname(cfgFile))
cmsCfg = modRef.process
cfgWrapper = CMSSWConfig()
cfgWrapper.originalCfg = file(cfgFile).read()
cfgInt = cfgWrapper.loadConfiguration(cmsCfg)
cfgInt.validateForProduction()
except Exception, e:
print "Unable to create request: %s" % e
# loop over cfg's provided and add to workflow
# first cmsRun node created implicitly by WorkflowMaker
nodeNumber = 0
for cmsRunCfg in cmsRunCfgs:
if cfgTypes[nodeNumber] == "cfg":
from FWCore.ParameterSet.Config import include
cmsCfg = include(cmsRunCfg)
else:
import imp
modRef = imp.load_source( os.path.basename(cmsRunCfg).replace(".py", ""), cmsRunCfg)
cmsCfg = modRef.process
cfgWrapper = CMSSWConfig()
#cfgWrapper.originalCfg = file(cmsRunCfg).read()
cfgInt = cfgWrapper.loadConfiguration(cmsCfg)
cfgInt.validateForProduction()
if nodeNumber:
try:
inputModules = chainedInputs[nodeNumber-1]
except IndexError:
inputModules = []
maker.chainCmsRunNode(stageoutOutputs[nodeNumber-1], *inputModules)
maker.setConfiguration(cfgWrapper, Type = "instance")
maker.setCMSSWVersion(versions[nodeNumber])
maker.setOriginalCfg(file(cmsRunCfg).read())
maker.setPSetHash(WorkflowTools.createPSetHash(cmsRunCfg))
nodeNumber += 1
def makeWorkflow(self, testInstance):
"""
_processTest_
Process a test, create a WorkflowSpec for it, generate job specs
and add the, to the test instance
"""
loader = CMSSWAPILoader(testInstance['CMSSWArchitecture'],
testInstance['CMSSWVersion'],
testInstance['CMSPath'])
loader.load()
cfgWrapper = CMSSWConfig()
process = pickle.load(file(testInstance['PickleFile']))
cfgInt = cfgWrapper.loadConfiguration(process)
cfgInt.validateForProduction()
cfgAsString = process.dumpPython()
# //
# // Get release validation PSet from process
#//
relValPSet = getattr(process, "ReleaseValidation", None)
if relValPSet == None:
msg = "Unable to extract ReleaseValidation PSet from pickled cfg for \n"
msg += "%s\n" % testInstance['PickleFile']
logging.error(msg)
return
testName = getattr(relValPSet, "primaryDatasetName", None)
testInstance['Name'] = testName.value()
if testName == None:
msg = "No primaryDatasetName parameter in ReleaseValidation PSet\n"
msg += "%s\n" % testInstance['PickleFile']
logging.error(msg)
return
totalEvents = getattr(relValPSet, "totalNumberOfEvents", None)
if totalEvents == None:
msg = "No totalNumberOfEvents parameter in ReleaseValidation PSet\n"
msg += "%s\n" % testInstance['PickleFile']
logging.error(msg)
return
testInstance['TotalEvents'] = totalEvents.value()
eventsPerJob = getattr(relValPSet, "eventsPerJob", None)
speedCat = getattr(relValPSet, "speedCategory", None)
if (eventsPerJob == None) and (speedCat == None):
msg = "ReleaseValidation PSet must contain one of either eventsPerJob or speedCategory\n"
msg += "%s\n" % testInstance['PickleFile']
logging.error(msg)
return
if eventsPerJob != None:
testInstance['EventsPerJob'] = eventsPerJob.value()
else:
testInstance['SpeedCategory'] = speedCat.value()
if not self.args.has_key(testInstance['SpeedCategory']):
msg = "Unknown Speed Category: %s\n" % testInstance['SpeedCategory']
msg += "In file: %s\n" % testInstance['PickleFile']
logging.error(msg)
return
testInstance['EventsPerJob'] = self.args[testInstance['SpeedCategory']]
inputDataset = getattr(relValPSet, "inputDatasetPath", None)
pileupDataset = getattr(relValPSet, "pileupDatasetPath", None)
if pileupDataset != None:
testInstance['PileupDataset'] = pileupDataset.value()
if inputDataset != None:
testInstance['InputDataset'] = inputDataset.value()
msg = "Processing : %s\n" % testInstance['Name']
msg += "From Pickle: %s\n" % testInstance['PickleFile']
msg += "TotalEvents: %s\n" % testInstance['TotalEvents']
msg += "EventsPerJob: %s\n" % testInstance['EventsPerJob']
msg += "SpeedCategory: %s\n" % testInstance['SpeedCategory']
logging.info(msg)
if self.workflows.has_key(testInstance['Name']):
testInstance['WorkflowSpecId'] = self.workflows[testInstance['Name']]
testInstance['WorkflowSpecFile'] = self.workflowFiles[testInstance['Name']]
testInstance['WorkingDir'] = self.workingDirs[testInstance['Name']]
loader.unload()
return
self.jobCounts[testInstance['Name']] = 1
workingDir = os.path.join(self.args['ComponentDir'],
testInstance['CMSSWVersion'],
testInstance['Name'])
if not os.path.exists(workingDir):
os.makedirs(workingDir)
loader.unload()
maker = WorkflowMaker(str(self.timestamp),
testInstance['Name'],
'RelVal')
maker.setCMSSWVersion(testInstance['CMSSWVersion'])
maker.setPhysicsGroup("RelVal")
maker.setConfiguration(cfgWrapper, Type = "instance")
maker.setOriginalCfg(cfgAsString)
psetHash = "NO_PSET_HASH"
if cfgWrapper.configMetadata.has_key('PSetHash'):
psetHash = cfgWrapper.configMetadata['PSetHash']
maker.setPSetHash(psetHash)
maker.changeCategory("relval")
if testInstance['SelectionEfficiency'] != None:
selEff = float(testInstance['SelectionEfficiency'] )
maker.addSelectionEfficiency(selEff)
if testInstance['PileupDataset'] != None:
maker.addPileupDataset(testInstance['PileupDataset'], 100)
if testInstance['InputDataset'] != None:
maker.addInputDataset(testInstance['InputDataset'])
maker.inputDataset["SplitType"] = "events"
maker.inputDataset["SplitSize"] = testInstance['EventsPerJob']
spec = maker.makeWorkflow()
spec.parameters['OnlySites'] = testInstance['Site']
spec.parameters['DBSURL'] = self.dbsUrl
specFile = "/%s/%s-Workflow.xml" % (workingDir, maker.workflowName)
spec.save(specFile)
self.workflows[testInstance['Name']] = str(maker.workflowName)
self.workflowFiles[testInstance['Name']] = specFile
self.workingDirs[testInstance['Name']] = workingDir
testInstance['WorkflowSpecId'] = str(maker.workflowName)
testInstance['WorkflowSpecFile'] = specFile
testInstance['WorkingDir'] = workingDir
msg = "Workflow created for test: %s" % testInstance['Name']
logging.info(msg)
msg = "Registering Workflow Entity: %s" % maker.workflowName
logging.debug(msg)
WEWorkflow.register(
maker.workflowName,
{"owner" : "RelValInjector",
"workflow_spec_file" : specFile,
})
msg = "Publishing NewWorkflow/NewDataset for \n"
msg += " %s\n "% specFile
logging.debug(msg)
self.ms.publish("NewWorkflow", specFile)
self.ms.publish("NewDataset", specFile)
self.ms.commit()
return
