def testGetSwVersionAndScramArchMulti(self):
"""
_testGetSwVersionAndScramArchMulti_
Test whether we can fetch the CMSSW release and ScramArch
being used in a task
"""
testTask = makeWMTask("MultiTask")
taskCmssw = testTask.makeStep("cmsRun1")
taskCmssw.setStepType("CMSSW")
taskCmsswStageOut = taskCmssw.addStep("stageOut1")
taskCmsswStageOut.setStepType("StageOut")
taskCmsswLogArch = taskCmsswStageOut.addStep("logArch1")
taskCmsswLogArch.setStepType("LogArchive")
testTask.applyTemplates()
taskCmsswHelper = taskCmssw.getTypeHelper()
taskCmsswHelper.cmsswSetup("CMSSW_1_2_3",
softwareEnvironment="",
scramArch="slc7_amd64_gcc123")
# setup step2/cmsRun2
step1Cmssw = testTask.getStep("cmsRun1")
step2Cmssw = step1Cmssw.addTopStep("cmsRun2")
step2Cmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(step2Cmssw.data)
step2CmsswHelper = step2Cmssw.getTypeHelper()
step2CmsswHelper.setupChainedProcessing("cmsRun1", "RAWSIMoutput")
step2CmsswHelper.cmsswSetup("CMSSW_2_2_3",
softwareEnvironment="",
scramArch="slc7_amd64_gcc223")
# setup step3/cmsRun3 --> duplicate CMSSW and ScramArch
step3Cmssw = step2Cmssw.addTopStep("cmsRun3")
step3Cmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(step3Cmssw.data)
step3CmsswHelper = step3Cmssw.getTypeHelper()
step3CmsswHelper.setupChainedProcessing("cmsRun2", "AODoutput")
step3CmsswHelper.cmsswSetup("CMSSW_1_2_3",
softwareEnvironment="",
scramArch="slc7_amd64_gcc123")
self.assertEqual(testTask.getSwVersion(), "CMSSW_1_2_3")
self.assertEqual(testTask.getSwVersion(allSteps=True),
["CMSSW_1_2_3", "CMSSW_2_2_3", "CMSSW_1_2_3"])
self.assertEqual(testTask.getScramArch(), "slc7_amd64_gcc123")
self.assertEqual(
testTask.getScramArch(allSteps=True),
["slc7_amd64_gcc123", "slc7_amd64_gcc223", "slc7_amd64_gcc123"])
return
def testGetSwVersionAndScramArchMulti(self):
"""
_testGetSwVersionAndScramArchMulti_
Test whether we can fetch the CMSSW release and ScramArch
being used in a task
"""
testTask = makeWMTask("MultiTask")
taskCmssw = testTask.makeStep("cmsRun1")
taskCmssw.setStepType("CMSSW")
taskCmsswStageOut = taskCmssw.addStep("stageOut1")
taskCmsswStageOut.setStepType("StageOut")
taskCmsswLogArch = taskCmsswStageOut.addStep("logArch1")
taskCmsswLogArch.setStepType("LogArchive")
testTask.applyTemplates()
taskCmsswHelper = taskCmssw.getTypeHelper()
taskCmsswHelper.cmsswSetup("CMSSW_1_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc123")
# setup step2/cmsRun2
step1Cmssw = testTask.getStep("cmsRun1")
step2Cmssw = step1Cmssw.addTopStep("cmsRun2")
step2Cmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(step2Cmssw.data)
step2CmsswHelper = step2Cmssw.getTypeHelper()
step2CmsswHelper.setupChainedProcessing("cmsRun1", "RAWSIMoutput")
step2CmsswHelper.cmsswSetup("CMSSW_2_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc223")
# setup step3/cmsRun3 --> duplicate CMSSW and ScramArch
step3Cmssw = step2Cmssw.addTopStep("cmsRun3")
step3Cmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(step3Cmssw.data)
step3CmsswHelper = step3Cmssw.getTypeHelper()
step3CmsswHelper.setupChainedProcessing("cmsRun2", "AODoutput")
step3CmsswHelper.cmsswSetup("CMSSW_1_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc123")
self.assertEqual(testTask.getSwVersion(), "CMSSW_1_2_3")
self.assertEqual(testTask.getSwVersion(allSteps=True), ["CMSSW_1_2_3", "CMSSW_2_2_3", "CMSSW_1_2_3"])
self.assertEqual(testTask.getScramArch(), "slc7_amd64_gcc123")
self.assertEqual(testTask.getScramArch(allSteps=True), ["slc7_amd64_gcc123", "slc7_amd64_gcc223", "slc7_amd64_gcc123"])
return
def setupThreeStepChainedProcessing(self, stepOneTask):
"""
_setupThreeStepChainedProcessing_
Modify the step one task to include two more CMSSW steps and chain the
output between all three steps.
"""
parentCmsswStep = stepOneTask.getStep("cmsRun1")
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
parentCmsswStepHelper.keepOutput(False)
stepTwoCmssw = parentCmsswStep.addTopStep("cmsRun2")
stepTwoCmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(stepTwoCmssw.data)
stepTwoCmsswHelper = stepTwoCmssw.getTypeHelper()
stepTwoCmsswHelper.setGlobalTag(self.globalTag)
stepTwoCmsswHelper.setupChainedProcessing("cmsRun1", self.stepOneOutputModuleName)
stepTwoCmsswHelper.cmsswSetup(self.frameworkVersion, softwareEnvironment = "",
scramArch = self.scramArch)
stepTwoCmsswHelper.setConfigCache(self.couchURL, self.stepTwoConfigCacheID,
self.couchDBName)
stepTwoCmsswHelper.keepOutput(False)
stepThreeCmssw = stepTwoCmssw.addTopStep("cmsRun3")
stepThreeCmssw.setStepType("CMSSW")
template(stepThreeCmssw.data)
stepThreeCmsswHelper = stepThreeCmssw.getTypeHelper()
stepThreeCmsswHelper.setGlobalTag(self.globalTag)
stepThreeCmsswHelper.setupChainedProcessing("cmsRun2", self.stepTwoOutputModuleName)
stepThreeCmsswHelper.cmsswSetup(self.frameworkVersion, softwareEnvironment = "",
scramArch = self.scramArch)
stepThreeCmsswHelper.setConfigCache(self.couchURL, self.stepThreeConfigCacheID,
self.couchDBName)
configOutput = self.determineOutputModules(None, None, self.stepTwoConfigCacheID,
self.couchURL, self.couchDBName)
for outputModuleName in configOutput.keys():
outputModule = self.addOutputModule(stepOneTask,
outputModuleName,
self.inputPrimaryDataset,
configOutput[outputModuleName]["dataTier"],
configOutput[outputModuleName]["filterName"],
stepName = "cmsRun2")
configOutput = self.determineOutputModules(None, None, self.stepThreeConfigCacheID,
self.couchURL, self.couchDBName)
outputMods = {}
for outputModuleName in configOutput.keys():
outputModule = self.addOutputModule(stepOneTask,
outputModuleName,
self.inputPrimaryDataset,
configOutput[outputModuleName]["dataTier"],
configOutput[outputModuleName]["filterName"],
stepName = "cmsRun3")
outputMods[outputModuleName] = outputModule
self.addMergeTasks(stepOneTask, "cmsRun3", outputMods)
return
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
stepHelper = step = self.task.makeStep("DeleteTest")
self.step = stepHelper.data
self.actualStep = stepHelper
template = DeleteTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.insert(0, self.testDir)
sys.path.insert(0, self.sandboxDir)
self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def testD_ExecuteNoOutput(self):
"""
_ExecuteNoOutput_
Test what happens when no output is produced,
the proper error should be included.
"""
self.step.application.command.executable = "cmsRun.py"
shutil.copy(
os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml"),
os.path.join(self.step.builder.workingDir,
"FrameworkJobReport.xml"))
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
executor.execute()
executor.post()
self.assertEqual(60450, executor.report.getExitCode())
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testD_ExecuteNoOutput(self):
"""
_ExecuteNoOutput_
Test what happens when no output is produced,
the proper error should be included.
"""
self.step.application.command.executable = "cmsRun.py"
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
executor.execute()
executor.post()
self.assertEqual(60450, executor.report.getExitCode())
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testC_ExecuteSegfault(self):
"""
_ExecuteSegfault_
Test the execution of a script
which raises a ABRT signal which
is the normal CMSSW response
to a SEGFAULT.
"""
self.step.application.command.executable = "test.sh"
# CMSSW leaves an empty FWJR when a SEGFAULT is present
open(os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"), "w").close()
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
try:
executor.execute()
self.fail("An exception should have been raised")
except WMExecutionFailure as ex:
executor.diagnostic(ex.code, executor, ExceptionInstance=ex)
self.assertEqual(50115, executor.report.getExitCode())
report = Report()
report.load("Report.pkl")
self.assertEqual(50115, report.getExitCode())
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testGPUSettings(self):
"""
Test GPU settings and the 'getGPURequired' and 'getGPURequirements' methods
"""
# create a standard step object - without the CMSSW template applied
wmStep = makeWMStep("step1")
self.assertIsNone(wmStep.stepType())
self.assertFalse(hasattr(wmStep.data, "gpu"))
self.assertIsNone(wmStep.getGPURequired())
# now apply the CMSSW template
wmStep.setStepType("CMSSW")
self.assertEqual(wmStep.stepType(), "CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(wmStep.data)
wmStepHelper = wmStep.getTypeHelper()
self.assertEqual(wmStepHelper.getGPURequired(), "forbidden")
self.assertIsNone(wmStepHelper.getGPURequirements())
gpuParams = {
"GPUMemoryMB": 1234,
"CUDARuntime": "11.2.3",
"CUDACapabilities": ["7.5", "8.0"]
}
wmStepHelper.setGPUSettings("required", gpuParams)
self.assertEqual(wmStepHelper.getGPURequired(), "required")
self.assertItemsEqual(wmStepHelper.getGPURequirements(), gpuParams)
return
def testB_ExecuteNonZeroExit(self):
"""
_ExecuteNonZeroExit_
Test the execution of a script
which exits with non-zero code.
"""
self.step.application.command.executable = "brokenCmsRun.py"
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWFailReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
try:
executor.execute()
self.fail("An exception should have been raised")
except WMExecutionFailure as ex:
executor.diagnostic(ex.code, executor, ExceptionInstance=ex)
self.assertEqual(8001, executor.report.getExitCode())
report = Report()
report.load("Report.pkl")
self.assertEqual(8001, report.getExitCode())
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testC_ExecuteSegfault(self):
"""
_ExecuteSegfault_
Test the execution of a script
which raises a ABRT signal which
is the normal CMSSW response
to a SEGFAULT.
"""
self.step.application.command.executable = "test.sh"
# CMSSW leaves an empty FWJR when a SEGFAULT is present
open(os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"), "w").close()
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
try:
executor.execute()
self.fail("An exception should have been raised")
except WMExecutionFailure, ex:
executor.diagnostic(ex.code, executor, ExceptionInstance = ex)
self.assertEqual(134, executor.report.getExitCode())
report = Report()
report.load("Report.pkl")
self.assertEqual(134, report.getExitCode())
except Exception, ex:
self.fail("Failure encountered, %s" % str(ex))
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
stepHelper = step = self.task.makeStep("DeleteTest")
self.step = stepHelper.data
self.actualStep = stepHelper
template = DeleteTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.insert(0, self.testDir)
sys.path.insert(0, self.sandboxDir)
self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def testB_ExecuteNonZeroExit(self):
"""
_ExecuteNonZeroExit_
Test the execution of a script
which exits with non-zero code.
"""
self.step.application.command.executable = "brokenCmsRun.py"
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWFailReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
try:
executor.execute()
self.fail("An exception should have been raised")
except WMExecutionFailure, ex:
executor.diagnostic(ex.code, executor, ExceptionInstance = ex)
self.assertEqual(8001, executor.report.getExitCode())
report = Report()
report.load("Report.pkl")
self.assertEqual(8001, report.getExitCode())
except Exception, ex:
self.fail("Failure encountered, %s" % str(ex))
def testC(self):
"""emulator"""
try:
cmssw = StepFactory.getStepEmulator("CMSSW")
except Exception, ex:
msg = "Error loading Step Emulator of Type CMSSW\n"
msg += str(ex)
self.fail(msg)
def testBuild(self):
""" build a directory and verify it exists"""
mytemplate = StepFactory.getStepTemplate("CMSSW")
mystep = WMStep.makeWMStep("DummyStagingStep")
mytemplate(mystep.data)
self.testBuilder(mystep.data, "testTask", self.tempDir)
self.assertTrue(os.path.exists(self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py" % self.tempDir))
def testEmulator(self):
"""emulator"""
try:
cmssw = StepFactory.getStepEmulator("CMSSW")
except Exception as ex:
msg = "Error loading Step Emulator of Type CMSSW\n"
msg += str(ex)
self.fail(msg)
def testA(self):
"""template"""
try:
cmssw = StepFactory.getStepTemplate("CMSSW")
except Exception, ex:
msg = "Error loading Step Template of Type CMSSW\n"
msg += str(ex)
self.fail(msg)
def testA(self):
"""template"""
try:
cmssw = StepFactory.getStepTemplate("CMSSW")
except Exception as ex:
msg = "Error loading Step Template of Type CMSSW\n"
msg += str(ex)
self.fail(msg)
def testC(self):
"""builder"""
try:
cmssw = StepFactory.getStepBuilder("CMSSW")
except Exception as ex:
msg = "Error loading Step Builder of Type CMSSW\n"
msg += str(ex)
self.fail(msg)
def setupNextSteps(self, task, origArgs):
"""
_setupNextSteps_
Modify the step one task to include N more CMSSW steps and
chain the output between all three steps.
"""
configCacheUrl = self.configCacheUrl or self.couchURL
stepMapping = {}
stepMapping.setdefault(origArgs['Step1']['StepName'], ('Step1', 'cmsRun1'))
for i in range(2, self.stepChain + 1):
currentStepNumber = "Step%d" % i
currentCmsRun = "cmsRun%d" % i
stepMapping.setdefault(origArgs[currentStepNumber]['StepName'], (currentStepNumber, currentCmsRun))
taskConf = {}
for k, v in origArgs[currentStepNumber].iteritems():
taskConf[k] = v
parentStepNumber = stepMapping.get(taskConf['InputStep'])[0]
parentCmsRun = stepMapping.get(taskConf['InputStep'])[1]
parentCmsswStep = task.getStep(parentCmsRun)
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
# Set default values for the task parameters
self.modifyTaskConfiguration(taskConf, False, 'InputDataset' not in taskConf)
globalTag = taskConf.get("GlobalTag", self.globalTag)
frameworkVersion = taskConf.get("CMSSWVersion", self.frameworkVersion)
scramArch = taskConf.get("ScramArch", self.scramArch)
childCmssw = parentCmsswStep.addTopStep(currentCmsRun)
childCmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(childCmssw.data)
childCmsswStepHelper = childCmssw.getTypeHelper()
childCmsswStepHelper.setGlobalTag(globalTag)
childCmsswStepHelper.setupChainedProcessing(parentCmsRun, taskConf['InputFromOutputModule'])
childCmsswStepHelper.cmsswSetup(frameworkVersion, softwareEnvironment="", scramArch=scramArch)
childCmsswStepHelper.setConfigCache(configCacheUrl, taskConf['ConfigCacheID'], self.couchDBName)
# Pileup check
taskConf["PileupConfig"] = parsePileupConfig(taskConf["MCPileup"], taskConf["DataPileup"])
if taskConf["PileupConfig"]:
self.setupPileup(task, taskConf['PileupConfig'])
# Handling the output modules
parentKeepOutput = strToBool(origArgs[parentStepNumber].get('KeepOutput', True))
parentCmsswStepHelper.keepOutput(parentKeepOutput)
childKeepOutput = strToBool(taskConf.get('KeepOutput', True))
childCmsswStepHelper.keepOutput(childKeepOutput)
self.setupOutputModules(task, taskConf["ConfigCacheID"], currentCmsRun, childKeepOutput,
taskConf['StepName'])
# Closing out the task configuration. The last step output must be saved/merged
childCmsswStepHelper.keepOutput(True)
return
def setupChainedProcessing(self, stepOneTask):
"""
_setupChainedProcessing_
Modify the step one task to include a second chained CMSSW step to
do RECO on the RAW.
"""
parentCmsswStep = stepOneTask.getStep("cmsRun1")
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
parentCmsswStepHelper.keepOutput(False)
stepTwoCmssw = parentCmsswStep.addTopStep("cmsRun2")
stepTwoCmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(stepTwoCmssw.data)
stepTwoCmsswHelper = stepTwoCmssw.getTypeHelper()
stepTwoCmsswHelper.setGlobalTag(self.globalTag)
stepTwoCmsswHelper.setupChainedProcessing("cmsRun1",
self.stepOneOutputModuleName)
stepTwoCmsswHelper.cmsswSetup(self.frameworkVersion,
softwareEnvironment="",
scramArch=self.scramArch)
stepTwoCmsswHelper.setConfigCache(self.configCacheUrl,
self.stepTwoConfigCacheID,
self.couchDBName)
configOutput = self.determineOutputModules(None, None,
self.stepTwoConfigCacheID,
self.couchDBName,
self.configCacheUrl)
outputMods = {}
for outputModuleName in configOutput.keys():
outputModule = self.addOutputModule(
stepOneTask,
outputModuleName,
self.inputPrimaryDataset,
configOutput[outputModuleName]["dataTier"],
configOutput[outputModuleName]["filterName"],
stepName="cmsRun2")
outputMods[outputModuleName] = outputModule
mergeTasks = self.addMergeTasks(stepOneTask, "cmsRun2", outputMods)
stepTwoCmsswHelper.setNumberOfCores(self.multicore, self.eventStreams)
if self.stepThreeConfigCacheID is None:
return
mergeTask = mergeTasks[self.stepTwoOutputModuleName]
self.addDependentProcTask("StepThreeProc",
mergeTask,
self.stepThreeConfigCacheID,
timePerEvent=self.stepThreeTimePerEvent,
sizePerEvent=self.stepThreeSizePerEvent,
memoryReq=self.stepThreeMemory)
return
def testBuild(self):
''' build a directory and verify it exists'''
mytemplate = StepFactory.getStepTemplate("StageOut")
mystep = WMStep.makeWMStep("DummyStagingStep")
mytemplate(mystep.data)
self.testBuilder(mystep.data, "testTask", self.tempDir)
self.assertTrue(os.path.exists(self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py"
% self.tempDir))
def setUp(self):
'''create the builder object and the temporary directory'''
self.tempDir = tempfile.mkdtemp()
try:
self.testBuilder = StepFactory.getStepBuilder("StageOut")
except Exception, ex:
msg = "Failed to instantiate Builder:\n"
msg += str(ex)
self.fail(msg)
def setUp(self):
"""create the builder object and the temporary directory"""
self.tempDir = tempfile.mkdtemp()
try:
self.testBuilder = StepFactory.getStepBuilder("CMSSW")
except Exception as ex:
msg = "Failed to instantiate Builder:\n"
msg += str(ex)
self.fail(msg)
def setUp(self):
'''create the builder object and the temporary directory'''
self.tempDir = tempfile.mkdtemp()
try:
self.testBuilder = StepFactory.getStepBuilder("StageOut")
except Exception as ex:
msg = "Failed to instantiate Builder:\n"
msg += str(ex)
self.fail(msg)
def testCustomBuild(self):
""" add in a custom directory and verify it gets created"""
mytemplate = StepFactory.getStepTemplate("CMSSW")
mystep = WMStep.makeWMStep("DummyStagingStep")
mytemplate(mystep.data)
helper = TemplateNS.Template.CoreHelper(mystep.data)
helper.addDirectory("testdirectory1")
helper.addDirectory("testdirectory2/testsubdir")
self.testBuilder(mystep.data, "testTask", self.tempDir)
self.assertTrue(os.path.exists(self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py" % self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/testdirectory1" % self.tempDir))
self.assertTrue(os.path.exists("%s/%s/testdirectory2/testsubdir" % (self.tempDir, "DummyStagingStep")))
def addEmulator(self, nodeName, emulatorName):
"""
_addEmulator_
Add an Emulator for the node provided, emulatorName is the name
of the emulator class to be loaded by the Emulator factory
TODO: Exception handling
"""
emuInstance = StepFactory.getStepEmulator(emulatorName)
self.emulators[nodeName] = emuInstance
return
def applyTemplates(self):
"""
_applyTemplates_
For each step, load the appropriate template and install the default structure
TODO: Exception handling
"""
for step in self.steps().nodeIterator():
stepType = step.stepType
template = StepFactory.getStepTemplate(stepType)
template(step)
def getStepHelper(self, stepName):
"""
_getStepHelper_
Get the named step, look up its type specific helper and retrieve
the step wrapped in the type based helper.
"""
step = self.getStep(stepName)
stepType = step.stepType()
template = StepFactory.getStepTemplate(stepType)
helper = template.helper(step.data)
return helper
def applyTemplates(self):
"""
_applyTemplates_
For each step, load the appropriate template and install the default structure
TODO: Exception handling
"""
for step in self.steps().nodeIterator():
stepType = step.stepType
template = StepFactory.getStepTemplate(stepType)
template(step)
def getStepHelper(self, stepName):
"""
_getStepHelper_
Get the named step, look up its type specific helper and retrieve
the step wrapped in the type based helper.
"""
step = self.getStep(stepName)
stepType = step.stepType()
template = StepFactory.getStepTemplate(stepType)
helper = template.helper(step.data)
return helper
def addEmulator(self, nodeName, emulatorName):
"""
_addEmulator_
Add an Emulator for the node provided, emulatorName is the name
of the emulator class to be loaded by the Emulator factory
TODO: Exception handling
"""
emuInstance = StepFactory.getStepEmulator(emulatorName)
self.emulators[nodeName] = emuInstance
return
def setupChainedProcessing(self, stepOneTask):
"""
_setupChainedProcessing_
Modify the step one task to include a second chained CMSSW step to
do RECO on the RAW.
"""
configCacheUrl = self.configCacheUrl or self.couchURL
parentCmsswStep = stepOneTask.getStep("cmsRun1")
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
parentCmsswStepHelper.keepOutput(False)
stepTwoCmssw = parentCmsswStep.addTopStep("cmsRun2")
stepTwoCmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(stepTwoCmssw.data)
stepTwoCmsswHelper = stepTwoCmssw.getTypeHelper()
stepTwoCmsswHelper.setGlobalTag(self.globalTag)
stepTwoCmsswHelper.setupChainedProcessing("cmsRun1", self.stepOneOutputModuleName)
stepTwoCmsswHelper.cmsswSetup(self.frameworkVersion, softwareEnvironment = "",
scramArch = self.scramArch)
stepTwoCmsswHelper.setConfigCache(configCacheUrl, self.stepTwoConfigCacheID,
self.couchDBName)
configOutput = self.determineOutputModules(None, None, self.stepTwoConfigCacheID,
configCacheUrl, self.couchDBName)
outputMods = {}
for outputModuleName in configOutput.keys():
outputModule = self.addOutputModule(stepOneTask,
outputModuleName,
self.inputPrimaryDataset,
configOutput[outputModuleName]["dataTier"],
configOutput[outputModuleName]["filterName"],
stepName = "cmsRun2")
outputMods[outputModuleName] = outputModule
mergeTasks = self.addMergeTasks(stepOneTask, "cmsRun2", outputMods)
stepTwoCmsswHelper.setNumberOfCores(self.multicore, self.eventStreams)
if self.stepThreeConfigCacheID is None:
return
mergeTask = mergeTasks[self.stepTwoOutputModuleName]
self.addDependentProcTask("StepThreeProc", mergeTask,
self.stepThreeConfigCacheID,
timePerEvent = self.stepThreeTimePerEvent,
sizePerEvent = self.stepThreeSizePerEvent,
memoryReq = self.stepThreeMemory)
return
def createTestStep(self):
"""
_createTestStep_
Create a test step that can be passed to the setup script.
"""
newStep = WMStep("cmsRun1")
newStepHelper = CMSSWStepHelper(newStep)
newStepHelper.setStepType("CMSSW")
newStepHelper.setGlobalTag("SomeGlobalTag")
stepTemplate = StepFactory.getStepTemplate("CMSSW")
stepTemplate(newStep)
newStep.application.command.configuration = "PSet.py"
return newStepHelper
def __call__(self, task):
"""
_operator(task)_
Invoke the builder on the task provided
TODO: Exception handling
"""
taskName = task.getPathName()
self.taskSpace = initialiseWMTaskSpace(self.workDir, taskName)
for step in task.steps().nodeIterator():
stepType = step.stepType
builder = StepFactory.getStepBuilder(stepType)
builder(step, task.name(), self.taskSpace)
def __call__(self, task):
"""
_operator(task)_
Invoke the builder on the task provided
TODO: Exception handling
"""
taskName = task.getPathName()
self.taskSpace = initialiseWMTaskSpace(self.workDir, taskName)
for step in task.steps().nodeIterator():
stepType = step.stepType
builder = StepFactory.getStepBuilder(stepType)
builder(step, task.name(), self.taskSpace)
def testCustomBuild(self):
''' add in a custom directory and verify it gets created'''
mytemplate = StepFactory.getStepTemplate("StageOut")
mystep = WMStep.makeWMStep("DummyStagingStep")
mytemplate(mystep.data)
helper = TemplateNS.Template.CoreHelper(mystep.data)
helper.addDirectory( 'testdirectory1' )
helper.addDirectory( 'testdirectory2/testsubdir' )
self.testBuilder(mystep.data, "testTask", self.tempDir)
self.assertTrue(os.path.exists(self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py"
% self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/testdirectory1"
% self.tempDir))
self.assertTrue(os.path.exists("%s/%s/testdirectory2/testsubdir"
% (self.tempDir, 'DummyStagingStep')))
def createTestStep(self):
"""
_createTestStep_
Create a test step that can be passed to the setup script.
"""
newStep = WMStep("cmsRun1")
stepTemplate = StepFactory.getStepTemplate("CMSSW")
stepTemplate.install(newStep)
newStepHelper = stepTemplate.helper(newStep)
newStepHelper.setStepType("CMSSW")
newStepHelper.setGlobalTag("SomeGlobalTag")
newStepHelper.data.application.section_("setup")
newStepHelper.cmsswSetup("CMSSW_11_0_2", scramArch=['slc7_amd64_gcc820'])
return newStepHelper
def __call__(self, task, wmbsJob):
"""
_operator(task)_
Load and run executors for all steps in Task, if an emulator is
available for that step, use it instead.
"""
myThread = threading.currentThread
try:
myThread.watchdogMonitor.setupMonitors(task, wmbsJob)
myThread.watchdogMonitor.notifyJobStart(task)
except WMException:
self.toTaskDirectory()
raise
except Exception as ex:
msg = "Encountered unhandled exception while starting monitors:\n"
msg += str(ex) + '\n'
msg += str(traceback.format_exc()) + '\n'
logging.error(msg)
self.toTaskDirectory()
raise WMExecutionFailure(msg)
skipToStep = None
for step in task.steps().nodeIterator():
try:
helper = WMStepHelper(step)
stepType = helper.stepType()
stepName = helper.name()
if skipToStep and skipToStep != stepName:
# Then we continue until we get to the required step
continue
skipToStep = None # Reset this when we get to the right step
executor = StepFactory.getStepExecutor(stepType)
result = self.doExecution(executor, step, wmbsJob)
if result is not None:
skipToStep = result
except WMException as ex:
msg = "Encountered error while running ExecuteMaster:\n"
msg += str(ex) + "\n"
logging.error(msg)
self.toTaskDirectory()
break
except Exception as ex:
msg = "Encountered error while running ExecuteMaster:\n"
msg += str(ex) + "\n"
msg += str(traceback.format_exc()) + "\n"
self.toTaskDirectory()
logging.error(msg)
break
try:
myThread.watchdogMonitor.notifyJobEnd(task)
except WMException:
self.toTaskDirectory()
except Exception as ex:
msg = "Encountered unhandled exception while ending the job:\n"
msg += str(ex) + '\n'
msg += str(traceback.format_exc()) + '\n'
logging.error(msg)
self.toTaskDirectory()
return
def setupNextSteps(self, task, origArgs):
"""
_setupNextSteps_
Modify the step one task to include N more CMSSW steps and
chain the output between all three steps.
"""
configCacheUrl = self.configCacheUrl or self.couchURL
for i in range(2, self.stepChain + 1):
inputStepName = "cmsRun%d" % (i-1)
parentCmsswStep = task.getStep(inputStepName)
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
parentCmsswStepHelper.keepOutput(False)
currentStepName = "cmsRun%d" % i
taskConf = {}
for k, v in origArgs["Step%d" % i].iteritems():
taskConf[k] = v
# Set default values to task parameters
self.modifyTaskConfiguration(taskConf, False, 'InputDataset' not in taskConf)
globalTag = taskConf.get("GlobalTag", self.globalTag)
childCmssw = parentCmsswStep.addTopStep(currentStepName)
childCmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(childCmssw.data)
childCmsswHelper = childCmssw.getTypeHelper()
childCmsswHelper.setGlobalTag(globalTag)
childCmsswHelper.setupChainedProcessing(inputStepName, taskConf['InputFromOutputModule'])
# Assuming we cannot change the CMSSW version inside the same job
childCmsswHelper.cmsswSetup(self.frameworkVersion, softwareEnvironment="",
scramArch=self.scramArch)
childCmsswHelper.setConfigCache(configCacheUrl, taskConf['ConfigCacheID'],
self.couchDBName)
childCmsswHelper.keepOutput(False)
# Pileup check
taskConf["PileupConfig"] = parsePileupConfig(taskConf["MCPileup"], taskConf["DataPileup"])
if taskConf["PileupConfig"]:
self.setupPileup(task, taskConf['PileupConfig'])
# Handling the output modules
outputMods = {}
configOutput = self.determineOutputModules(configDoc=taskConf['ConfigCacheID'],
couchURL=configCacheUrl,
couchDBName=self.couchDBName)
for outputModuleName in configOutput.keys():
outputModule = self.addOutputModule(task, outputModuleName,
self.inputPrimaryDataset,
configOutput[outputModuleName]["dataTier"],
configOutput[outputModuleName]["filterName"],
stepName=currentStepName)
outputMods[outputModuleName] = outputModule
# Closing out the task configuration
# Only the last step output is important :-)
childCmsswHelper.keepOutput(True)
self.addMergeTasks(task, currentStepName, outputMods)
# Override task parameters by the workload ones in case of their absence
self.updateCommonParams(task, taskConf)
return
def setupNextSteps(self, task, origArgs):
"""
_setupNextSteps_
Modify the step one task to include N more CMSSW steps and
chain the output between all three steps.
"""
self.stepParentageMapping.setdefault(origArgs['Step1']['StepName'], {})
for i in range(2, self.stepChain + 1):
currentStepNumber = "Step%d" % i
currentCmsRun = "cmsRun%d" % i
taskConf = {}
for k, v in viewitems(origArgs[currentStepNumber]):
taskConf[k] = v
parentStepNumber = self.stepMapping.get(taskConf['InputStep'])[0]
parentCmsRun = self.stepMapping.get(taskConf['InputStep'])[1]
parentCmsswStep = task.getStep(parentCmsRun)
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
# Set default values for the task parameters
self.modifyTaskConfiguration(taskConf, False, 'InputDataset'
not in taskConf)
globalTag = self.getStepValue('GlobalTag', taskConf,
self.globalTag)
frameworkVersion = self.getStepValue('CMSSWVersion', taskConf,
self.frameworkVersion)
scramArch = self.getStepValue('ScramArch', taskConf,
self.scramArch)
prepId = self.getStepValue('PrepID', taskConf, self.prepID)
currentCmssw = parentCmsswStep.addTopStep(currentCmsRun)
currentCmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(currentCmssw.data)
currentCmsswStepHelper = currentCmssw.getTypeHelper()
currentCmsswStepHelper.setPrepId(prepId)
currentCmsswStepHelper.setGlobalTag(globalTag)
currentCmsswStepHelper.setupChainedProcessing(
parentCmsRun, taskConf['InputFromOutputModule'])
currentCmsswStepHelper.cmsswSetup(frameworkVersion,
softwareEnvironment="",
scramArch=scramArch)
currentCmsswStepHelper.setConfigCache(self.configCacheUrl,
taskConf['ConfigCacheID'],
self.couchDBName)
# multicore settings
multicore = self.multicore
eventStreams = self.eventStreams
if taskConf['Multicore'] > 0:
multicore = taskConf['Multicore']
if taskConf.get("EventStreams"
) is not None and taskConf['EventStreams'] >= 0:
eventStreams = taskConf['EventStreams']
currentCmsswStepHelper.setNumberOfCores(multicore, eventStreams)
# Pileup check
taskConf["PileupConfig"] = parsePileupConfig(
taskConf["MCPileup"], taskConf["DataPileup"])
if taskConf["PileupConfig"]:
self.setupPileup(task,
taskConf['PileupConfig'],
stepName=currentCmsRun)
# Handling the output modules in order to decide whether we should
# stage them out and report them in the Report.pkl file
parentKeepOutput = strToBool(origArgs[parentStepNumber].get(
'KeepOutput', True))
parentCmsswStepHelper.keepOutput(parentKeepOutput)
childKeepOutput = strToBool(taskConf.get('KeepOutput', True))
currentCmsswStepHelper.keepOutput(childKeepOutput)
self.setupOutputModules(task, taskConf, currentCmsRun,
childKeepOutput)
# Closing out the task configuration. The last step output must be saved/merged
currentCmsswStepHelper.keepOutput(True)
return
def setupNextSteps(self, task, origArgs):
"""
_setupNextSteps_
Modify the step one task to include N more CMSSW steps and
chain the output between all three steps.
"""
configCacheUrl = self.configCacheUrl or self.couchURL
for i in range(2, self.stepChain + 1):
inputStepName = "cmsRun%d" % (i - 1)
parentCmsswStep = task.getStep(inputStepName)
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
parentCmsswStepHelper.keepOutput(False)
currentStepName = "cmsRun%d" % i
taskConf = {}
for k, v in origArgs["Step%d" % i].iteritems():
taskConf[k] = v
# Set default values to task parameters
self.modifyTaskConfiguration(taskConf, False, 'InputDataset'
not in taskConf)
globalTag = taskConf.get("GlobalTag", self.globalTag)
frameworkVersion = taskConf.get("CMSSWVersion",
self.frameworkVersion)
scramArch = taskConf.get("ScramArch", self.scramArch)
childCmssw = parentCmsswStep.addTopStep(currentStepName)
childCmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(childCmssw.data)
childCmsswHelper = childCmssw.getTypeHelper()
childCmsswHelper.setGlobalTag(globalTag)
childCmsswHelper.setupChainedProcessing(
inputStepName, taskConf['InputFromOutputModule'])
# Assuming we cannot change the CMSSW version inside the same job
childCmsswHelper.cmsswSetup(frameworkVersion,
softwareEnvironment="",
scramArch=scramArch)
childCmsswHelper.setConfigCache(configCacheUrl,
taskConf['ConfigCacheID'],
self.couchDBName)
childCmsswHelper.keepOutput(False)
# Pileup check
taskConf["PileupConfig"] = parsePileupConfig(
taskConf["MCPileup"], taskConf["DataPileup"])
if taskConf["PileupConfig"]:
self.setupPileup(task, taskConf['PileupConfig'])
# Handling the output modules
outputMods = {}
configOutput = self.determineOutputModules(
configDoc=taskConf['ConfigCacheID'],
couchURL=configCacheUrl,
couchDBName=self.couchDBName)
for outputModuleName in configOutput.keys():
outputModule = self.addOutputModule(
task,
outputModuleName,
self.inputPrimaryDataset,
configOutput[outputModuleName]["dataTier"],
configOutput[outputModuleName]["filterName"],
stepName=currentStepName)
outputMods[outputModuleName] = outputModule
# Closing out the task configuration
# Only the last step output is important :-)
childCmsswHelper.keepOutput(True)
self.addMergeTasks(task, currentStepName, outputMods)
return
def buildWorkload(self):
"""
_buildWorkload_
Build the workload given all of the input parameters.
Not that there will be LogCollect tasks created for each processing
task and Cleanup tasks created for each merge task.
"""
workload = self.createWorkload()
workload.setDashboardActivity("tier0")
self.reportWorkflowToDashboard(workload.getDashboardActivity())
cmsswStepType = "CMSSW"
taskType = "Processing"
# complete output configuration
for output in self.outputs:
output['moduleLabel'] = "write_%s_%s" % (output['primaryDataset'],
output['dataTier'])
# finalize splitting parameters
mySplitArgs = self.expressSplitArgs.copy()
mySplitArgs['algo_package'] = "T0.JobSplitting"
expressTask = workload.newTask("Express")
#
# need to split this up into two separate code paths
# one is direct reco from the streamer files
# the other is conversion and then reco
#
if self.recoFrameworkVersion == None or self.recoFrameworkVersion == self.frameworkVersion:
expressRecoStepName = "cmsRun1"
scenarioArgs = { 'globalTag' : self.globalTag,
'globalTagTransaction' : self.globalTagTransaction,
'skims' : self.alcaSkims,
'dqmSeq' : self.dqmSequences,
'outputs' : self.outputs,
'inputSource' : "DAT" }
if self.globalTagConnect:
scenarioArgs['globalTagConnect'] = self.globalTagConnect
expressOutMods = self.setupProcessingTask(expressTask, taskType,
scenarioName = self.procScenario,
scenarioFunc = "expressProcessing",
scenarioArgs = scenarioArgs,
splitAlgo = "Express",
splitArgs = mySplitArgs,
stepType = cmsswStepType,
forceUnmerged = True)
else:
expressRecoStepName = "cmsRun2"
conversionOutMods = self.setupProcessingTask(expressTask, taskType,
scenarioName = self.procScenario,
scenarioFunc = "repack",
scenarioArgs = { 'outputs' : [ { 'dataTier' : "RAW",
'eventContent' : "ALL",
'primaryDataset' : self.specialDataset,
'moduleLabel' : "write_RAW" } ] },
splitAlgo = "Express",
splitArgs = mySplitArgs,
stepType = cmsswStepType,
forceUnmerged = True)
# there is only one
conversionOutLabel = conversionOutMods.keys()[0]
# everything coming after should use the reco CMSSW version and Scram Arch
self.frameworkVersion = self.recoFrameworkVersion
self.scramArch = self.recoScramArch
# add a second step doing the reconstruction
parentCmsswStep = expressTask.getStep("cmsRun1")
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
parentCmsswStepHelper.keepOutput(False)
stepTwoCmssw = parentCmsswStep.addTopStep("cmsRun2")
stepTwoCmssw.setStepType(cmsswStepType)
template = StepFactory.getStepTemplate(cmsswStepType)
template(stepTwoCmssw.data)
stepTwoCmsswHelper = stepTwoCmssw.getTypeHelper()
if self.multicore:
# if multicore, poke in the number of cores setting
stepTwoCmsswHelper.setNumberOfCores(self.multicoreNCores)
stepTwoCmsswHelper.setGlobalTag(self.globalTag)
stepTwoCmsswHelper.setupChainedProcessing("cmsRun1", conversionOutLabel)
stepTwoCmsswHelper.cmsswSetup(self.frameworkVersion, softwareEnvironment = "",
scramArch = self.scramArch)
scenarioFunc = "expressProcessing"
scenarioArgs = { 'globalTag' : self.globalTag,
'globalTagTransaction' : self.globalTagTransaction,
'skims' : self.alcaSkims,
'dqmSeq' : self.dqmSequences,
'outputs' : self.outputs,
'inputSource' : "EDM" }
if self.globalTagConnect:
scenarioArgs['globalTagConnect'] = self.globalTagConnect
configOutput = self.determineOutputModules(scenarioFunc, scenarioArgs)
expressOutMods = {}
for outputModuleName in configOutput.keys():
outputModule = self.addOutputModule(expressTask,
outputModuleName,
configOutput[outputModuleName]['primaryDataset'],
configOutput[outputModuleName]['dataTier'],
configOutput[outputModuleName].get('filterName', None),
stepName = "cmsRun2", forceUnmerged = True)
expressOutMods[outputModuleName] = outputModule
if 'outputs' in scenarioArgs:
for output in scenarioArgs['outputs']:
if 'primaryDataset' in output:
del output['primaryDataset']
if 'primaryDataset' in scenarioArgs:
del scenarioArgs['primaryDataset']
stepTwoCmsswHelper.setDataProcessingConfig(self.procScenario,
scenarioFunc,
**scenarioArgs)
expressTask.setTaskType("Express")
self.addLogCollectTask(expressTask, taskName = "ExpressLogCollect")
for expressOutLabel, expressOutInfo in expressOutMods.items():
if expressOutInfo['dataTier'] == "ALCARECO":
# finalize splitting parameters
mySplitArgs = self.expressMergeSplitArgs.copy()
mySplitArgs['algo_package'] = "T0.JobSplitting"
alcaSkimTask = expressTask.addTask("%sAlcaSkim%s" % (expressTask.name(), expressOutLabel))
alcaSkimTask.setInputReference(expressTask.getStep(expressRecoStepName),
outputModule = expressOutLabel)
scenarioArgs = { 'globalTag' : self.globalTag,
'globalTagTransaction' : self.globalTagTransaction,
'skims' : self.alcaSkims,
'primaryDataset' : self.specialDataset }
if self.globalTagConnect:
scenarioArgs['globalTagConnect'] = self.globalTagConnect
alcaSkimOutMods = self.setupProcessingTask(alcaSkimTask, taskType,
scenarioName = self.procScenario,
scenarioFunc = "alcaSkim",
scenarioArgs = scenarioArgs,
splitAlgo = "ExpressMerge",
splitArgs = mySplitArgs,
stepType = cmsswStepType,
forceMerged = True,
useMulticore = False)
alcaSkimTask.setTaskType("Express")
self.addLogCollectTask(alcaSkimTask, taskName = "AlcaSkimLogCollect")
self.addCleanupTask(expressTask, expressOutLabel)
for alcaSkimOutLabel, alcaSkimOutInfo in alcaSkimOutMods.items():
if alcaSkimOutInfo['dataTier'] == "ALCAPROMPT" and self.alcaHarvestDir != None:
harvestTask = self.addAlcaHarvestTask(alcaSkimTask, alcaSkimOutLabel,
alcapromptdataset = alcaSkimOutInfo['filterName'],
condOutLabel = self.alcaHarvestOutLabel,
condUploadDir = self.alcaHarvestDir,
uploadProxy = self.dqmUploadProxy,
doLogCollect = True)
self.addConditionTask(harvestTask, self.alcaHarvestOutLabel)
else:
mergeTask = self.addExpressMergeTask(expressTask, expressRecoStepName, expressOutLabel)
if expressOutInfo['dataTier'] in [ "DQM", "DQMIO" ]:
self.addDQMHarvestTask(mergeTask, "Merged",
uploadProxy = self.dqmUploadProxy,
periodic_harvest_interval = self.periodicHarvestInterval,
doLogCollect = True)
workload.setBlockCloseSettings(self.blockCloseDelay,
workload.getBlockCloseMaxFiles(),
workload.getBlockCloseMaxEvents(),
workload.getBlockCloseMaxSize())
# setting the parameters which need to be set for all the tasks
# sets acquisitionEra, processingVersion, processingString
workload.setTaskPropertiesFromWorkload()
# set the LFN bases (normally done by request manager)
# also pass run number to add run based directories
workload.setLFNBase(self.mergedLFNBase, self.unmergedLFNBase,
runNumber = self.runNumber)
return workload
def testSubscriptionInformation(self):
"""
_testSubscriptionInformation_
Check the three methods related to the subscription information in a task
Make sure that we can set the subscription information for all datasets produced by this task
and we can select only some primaryDatasets/DataTiers.
Since subscriptions are defined during request assignment, there is no more need to update
them, they are set once only.
"""
testTask = makeWMTask("TestTask")
cmsswStep = testTask.makeStep("cmsRun1")
cmsswStep.setStepType("CMSSW")
testTask.applyTemplates()
cmsswHelper = cmsswStep.getTypeHelper()
cmsswHelper.addOutputModule("outputRECO", primaryDataset="OneParticle",
processedDataset="DawnOfAnEra-v1", dataTier="RECO")
cmsswHelper.addOutputModule("outputDQM", primaryDataset="TwoParticles",
processedDataset="DawnOfAnEra-v1", dataTier="DQM")
cmsswHelper.addOutputModule("outputAOD", primaryDataset="OneParticle",
processedDataset="DawnOfAnEra-v1", dataTier="AOD")
childStep = cmsswHelper.addTopStep("cmsRun2")
childStep.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(childStep.data)
childStep = childStep.getTypeHelper()
childStep.addOutputModule("outputAOD", primaryDataset="ThreeParticles",
processedDataset="DawnOfAnEra-v1", dataTier="MINIAOD")
self.assertEqual(testTask.getSubscriptionInformation(), {}, "There should not be any subscription info")
testTask.setSubscriptionInformation(custodialSites=["mercury"],
nonCustodialSites=["mars", "earth"],
autoApproveSites=["earth"],
custodialSubType="Replica",
nonCustodialSubType="Move",
custodialGroup="DataOps",
nonCustodialGroup="AnalysisOps",
priority="Normal",
deleteFromSource=True,
primaryDataset="OneParticle")
subInfo = testTask.getSubscriptionInformation()
outputRecoSubInfo = {"CustodialSites": ["mercury"],
"NonCustodialSites": ["mars", "earth"],
"AutoApproveSites": ["earth"],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Move",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "AnalysisOps",
"Priority": "Normal",
"DeleteFromSource": True}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertFalse("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertFalse("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
testTask.setSubscriptionInformation(custodialSites=["jupiter"],
primaryDataset="TwoParticles")
subInfo = testTask.getSubscriptionInformation()
outputDQMSubInfo = {"CustodialSites": ["jupiter"],
"NonCustodialSites": [],
"AutoApproveSites": [],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Replica",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "DataOps",
"Priority": "Low",
"DeleteFromSource": False}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertEqual(subInfo["/TwoParticles/DawnOfAnEra-v1/DQM"], outputDQMSubInfo,
"The DQM subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertFalse("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
testTask.setSubscriptionInformation(nonCustodialSites=["jupiter"],
primaryDataset="ThreeParticles")
subInfo = testTask.getSubscriptionInformation()
outputAODSubInfo = {"CustodialSites": [],
"NonCustodialSites": ["jupiter"],
"AutoApproveSites": [],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Replica",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "DataOps",
"Priority": "Low",
"DeleteFromSource": False}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertEqual(subInfo["/TwoParticles/DawnOfAnEra-v1/DQM"], outputDQMSubInfo,
"The DQM subscription information is wrong")
self.assertEqual(subInfo["/ThreeParticles/DawnOfAnEra-v1/MINIAOD"], outputAODSubInfo,
"The AOD subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertTrue("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
def setUp(self):
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# shut up SiteLocalConfig
os.environ['CMS_PATH'] = os.getcwd()
workload = copy.deepcopy(testWorkloads.workload)
task = workload.getTask("Production")
step = task.getStep("stageOut1")
# want to get the cmsstep so I can make the Report
cmsstep = task.getStep('cmsRun1')
self.cmsstepdir = os.path.join(self.testDir, 'cmsRun1')
os.mkdir(self.cmsstepdir)
open(os.path.join(self.cmsstepdir, '__init__.py'), 'w').close()
open(os.path.join(self.cmsstepdir, 'Report.pkl'), 'w').close()
cmsbuilder = CMSSWBuilder.CMSSW()
cmsbuilder(cmsstep.data, 'Production', self.cmsstepdir)
realstep = StageOutTemplate.StageOutStepHelper(step.data)
realstep.disableRetries()
self.realstep = realstep
self.stepDir = os.path.join(self.testDir, 'stepdir')
os.mkdir(self.stepDir)
builder = StageOutBuilder.StageOut()
builder(step.data, 'Production', self.stepDir)
# stolen from CMSSWExecutor_t. thanks, dave
# first, delete all the sandboxen and taskspaces
# because of caching, this leaks from other tests in other files
# this sucks because the other tests are using sandboxen that
# are deleted after the test is over, which causes theses tests
# to break
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except Exception:
pass
del sys.modules[modname]
self.oldpath = sys.path[:]
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.job = Job(name="/UnitTests/DeleterTask/DeleteTest-test-job")
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
cmsswHelper = self.task.makeStep("cmsRun1")
cmsswHelper.setStepType('CMSSW')
stepHelper = cmsswHelper.addStep("DeleteTest")
stepHelper.setStepType('StageOut')
self.cmsswstep = cmsswHelper.data
self.cmsswHelper = cmsswHelper
self.stepdata = stepHelper.data
self.stephelp = StageOutTemplate.StageOutStepHelper(stepHelper.data)
self.task.applyTemplates()
self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.task.build(os.path.join(self.testDir, 'UnitTests'))
sys.path.insert(0, self.testDir)
sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))
# binDir = inspect.getsourcefile(ModuleLocator)
# binDir = binDir.replace("__init__.py", "bin")
#
# if not binDir in os.environ['PATH']:
# os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
open(os.path.join(self.testDir, 'UnitTests', '__init__.py'),
'w').close()
shutil.copyfile(
os.path.join(os.path.dirname(__file__), 'MergeSuccess.pkl'),
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
class ExecuteMaster:
"""
_ExecuteMaster_
Traverse the given task and invoke the execute framework
If an emulator is provided, then invoke the appropriate emulator
instead of the executor
"""
def __init__(self):
pass
def __call__(self, task, wmbsJob):
"""
_operator(task)_
Load and run executors for all steps in Task, if an emulator is
available for that step, use it instead.
"""
myThread = threading.currentThread
try:
myThread.watchdogMonitor.setupMonitors(task, wmbsJob)
myThread.watchdogMonitor.notifyJobStart(task)
except WMException:
self.toTaskDirectory()
raise
except Exception, ex:
msg = "Encountered unhandled exception while starting monitors:\n"
msg += str(ex) + '\n'
msg += str(traceback.format_exc()) + '\n'
logging.error(msg)
self.toTaskDirectory()
raise WMExecutionFailure(msg)
skipToStep = None
for step in task.steps().nodeIterator():
try:
helper = WMStepHelper(step)
stepType = helper.stepType()
stepName = helper.name()
if skipToStep and skipToStep != stepName:
# Then we continue until we get to the required step
continue
skipToStep = None # Reset this when we get to the right step
executor = StepFactory.getStepExecutor(stepType)
result = self.doExecution(executor, step, wmbsJob)
if not result == None:
skipToStep = result
except WMException, ex:
self.toTaskDirectory()
break
except Exception, ex:
msg = "Encountered error while running ExecuteMaster:\n"
msg += str(ex) + "\n"
msg += str(traceback.format_exc()) + "\n"
self.toTaskDirectory()
logging.error(msg)
break
def setupThreeStepChainedProcessing(self, stepOneTask):
"""
_setupThreeStepChainedProcessing_
Modify the step one task to include two more CMSSW steps and chain the
output between all three steps.
"""
configCacheUrl = self.configCacheUrl or self.couchURL
parentCmsswStep = stepOneTask.getStep("cmsRun1")
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
parentCmsswStepHelper.keepOutput(False)
stepTwoCmssw = parentCmsswStep.addTopStep("cmsRun2")
stepTwoCmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(stepTwoCmssw.data)
stepTwoCmsswHelper = stepTwoCmssw.getTypeHelper()
stepTwoCmsswHelper.setGlobalTag(self.globalTag)
stepTwoCmsswHelper.setupChainedProcessing("cmsRun1", self.stepOneOutputModuleName)
stepTwoCmsswHelper.cmsswSetup(self.frameworkVersion, softwareEnvironment = "",
scramArch = self.scramArch)
stepTwoCmsswHelper.setConfigCache(configCacheUrl, self.stepTwoConfigCacheID,
self.couchDBName)
stepTwoCmsswHelper.keepOutput(False)
stepThreeCmssw = stepTwoCmssw.addTopStep("cmsRun3")
stepThreeCmssw.setStepType("CMSSW")
template(stepThreeCmssw.data)
stepThreeCmsswHelper = stepThreeCmssw.getTypeHelper()
stepThreeCmsswHelper.setGlobalTag(self.globalTag)
stepThreeCmsswHelper.setupChainedProcessing("cmsRun2", self.stepTwoOutputModuleName)
stepThreeCmsswHelper.cmsswSetup(self.frameworkVersion, softwareEnvironment = "",
scramArch = self.scramArch)
stepThreeCmsswHelper.setConfigCache(configCacheUrl, self.stepThreeConfigCacheID,
self.couchDBName)
configOutput = self.determineOutputModules(None, None, self.stepTwoConfigCacheID,
configCacheUrl, self.couchDBName)
for outputModuleName in configOutput.keys():
outputModule = self.addOutputModule(stepOneTask,
outputModuleName,
self.inputPrimaryDataset,
configOutput[outputModuleName]["dataTier"],
configOutput[outputModuleName]["filterName"],
stepName = "cmsRun2")
configOutput = self.determineOutputModules(None, None, self.stepThreeConfigCacheID,
configCacheUrl, self.couchDBName)
outputMods = {}
for outputModuleName in configOutput.keys():
outputModule = self.addOutputModule(stepOneTask,
outputModuleName,
self.inputPrimaryDataset,
configOutput[outputModuleName]["dataTier"],
configOutput[outputModuleName]["filterName"],
stepName = "cmsRun3")
outputMods[outputModuleName] = outputModule
self.addMergeTasks(stepOneTask, "cmsRun3", outputMods)
return
def setupNextSteps(self, task, origArgs):
"""
_setupNextSteps_
Modify the step one task to include N more CMSSW steps and
chain the output between all three steps.
"""
configCacheUrl = self.configCacheUrl or self.couchURL
stepMapping = {}
stepMapping.setdefault(origArgs['Step1']['StepName'],
('Step1', 'cmsRun1'))
for i in range(2, self.stepChain + 1):
currentStepNumber = "Step%d" % i
currentCmsRun = "cmsRun%d" % i
stepMapping.setdefault(origArgs[currentStepNumber]['StepName'],
(currentStepNumber, currentCmsRun))
taskConf = {}
for k, v in origArgs[currentStepNumber].iteritems():
taskConf[k] = v
parentStepNumber = stepMapping.get(taskConf['InputStep'])[0]
parentCmsRun = stepMapping.get(taskConf['InputStep'])[1]
parentCmsswStep = task.getStep(parentCmsRun)
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
# Set default values for the task parameters
self.modifyTaskConfiguration(taskConf, False, 'InputDataset'
not in taskConf)
globalTag = taskConf.get("GlobalTag", self.globalTag)
frameworkVersion = taskConf.get("CMSSWVersion",
self.frameworkVersion)
scramArch = taskConf.get("ScramArch", self.scramArch)
childCmssw = parentCmsswStep.addTopStep(currentCmsRun)
childCmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(childCmssw.data)
childCmsswStepHelper = childCmssw.getTypeHelper()
childCmsswStepHelper.setGlobalTag(globalTag)
childCmsswStepHelper.setupChainedProcessing(
parentCmsRun, taskConf['InputFromOutputModule'])
childCmsswStepHelper.cmsswSetup(frameworkVersion,
softwareEnvironment="",
scramArch=scramArch)
childCmsswStepHelper.setConfigCache(configCacheUrl,
taskConf['ConfigCacheID'],
self.couchDBName)
# Pileup check
taskConf["PileupConfig"] = parsePileupConfig(
taskConf["MCPileup"], taskConf["DataPileup"])
if taskConf["PileupConfig"]:
self.setupPileup(task, taskConf['PileupConfig'])
# Handling the output modules
parentKeepOutput = strToBool(origArgs[parentStepNumber].get(
'KeepOutput', True))
parentCmsswStepHelper.keepOutput(parentKeepOutput)
childKeepOutput = strToBool(taskConf.get('KeepOutput', True))
childCmsswStepHelper.keepOutput(childKeepOutput)
self.setupOutputModules(task, taskConf["ConfigCacheID"],
currentCmsRun, childKeepOutput,
taskConf['StepName'])
# Closing out the task configuration. The last step output must be saved/merged
childCmsswStepHelper.keepOutput(True)
return
def __call__(self, task, wmbsJob):
"""
_operator(task)_
Load and run executors for all steps in Task, if an emulator is
available for that step, use it instead.
"""
myThread = threading.currentThread
try:
myThread.watchdogMonitor.setupMonitors(task, wmbsJob)
myThread.watchdogMonitor.notifyJobStart(task)
except WMException:
self.toTaskDirectory()
raise
except Exception as ex:
msg = "Encountered unhandled exception while starting monitors:\n"
msg += str(ex) + '\n'
msg += str(traceback.format_exc()) + '\n'
logging.error(msg)
self.toTaskDirectory()
raise WMExecutionFailure(msg)
failureUpstream = False
for step in task.steps().nodeIterator():
try:
helper = WMStepHelper(step)
stepType = helper.stepType()
stepName = helper.name()
if failureUpstream:
# for chained steps, don't execute further steps if a
# failure has already happened
helper.addOverride("previousCmsRunFailure", True)
executor = StepFactory.getStepExecutor(stepType)
result = self.doExecution(executor, step, wmbsJob)
logging.info("StepName: %s, StepType: %s, with result: %r",
stepName, stepType, result)
if result: # can be either None, or the step exit code
failureUpstream = True
except WMException as ex:
msg = "Encountered error while running ExecuteMaster:\n"
msg += str(ex) + "\n"
logging.error(msg)
self.toTaskDirectory()
break
except Exception as ex:
msg = "Encountered error while running ExecuteMaster:\n"
msg += str(ex) + "\n"
msg += str(traceback.format_exc()) + "\n"
self.toTaskDirectory()
logging.error(msg)
break
try:
myThread.watchdogMonitor.notifyJobEnd(task)
except WMException:
self.toTaskDirectory()
except Exception as ex:
msg = "Encountered unhandled exception while ending the job:\n"
msg += str(ex) + '\n'
msg += str(traceback.format_exc()) + '\n'
logging.error(msg)
self.toTaskDirectory()
return
def testSubscriptionInformation(self):
"""
_testSubscriptionInformation_
Check the three methods related to the subscription information in a task
Make sure that we can set the subscription information for all datasets produced by this task
and we can select only some primaryDatasets/DataTiers.
Since subscriptions are defined during request assignment, there is no more need to update
them, they are set once only.
"""
testTask = makeWMTask("TestTask")
cmsswStep = testTask.makeStep("cmsRun1")
cmsswStep.setStepType("CMSSW")
testTask.applyTemplates()
cmsswHelper = cmsswStep.getTypeHelper()
cmsswHelper.addOutputModule("outputRECO", primaryDataset="OneParticle",
processedDataset="DawnOfAnEra-v1", dataTier="RECO")
cmsswHelper.addOutputModule("outputDQM", primaryDataset="TwoParticles",
processedDataset="DawnOfAnEra-v1", dataTier="DQM")
cmsswHelper.addOutputModule("outputAOD", primaryDataset="OneParticle",
processedDataset="DawnOfAnEra-v1", dataTier="AOD")
childStep = cmsswHelper.addTopStep("cmsRun2")
childStep.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(childStep.data)
childStep = childStep.getTypeHelper()
childStep.addOutputModule("outputAOD", primaryDataset="ThreeParticles",
processedDataset="DawnOfAnEra-v1", dataTier="MINIAOD")
self.assertEqual(testTask.getSubscriptionInformation(), {}, "There should not be any subscription info")
testTask.setSubscriptionInformation(custodialSites=["mercury"],
nonCustodialSites=["mars", "earth"],
autoApproveSites=["earth"],
custodialSubType="Replica",
nonCustodialSubType="Move",
custodialGroup="DataOps",
nonCustodialGroup="AnalysisOps",
priority="Normal",
deleteFromSource=True,
primaryDataset="OneParticle")
subInfo = testTask.getSubscriptionInformation()
outputRecoSubInfo = {"CustodialSites": ["mercury"],
"NonCustodialSites": ["mars", "earth"],
"AutoApproveSites": ["earth"],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Move",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "AnalysisOps",
"Priority": "Normal",
"DeleteFromSource": True}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertFalse("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertFalse("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
testTask.setSubscriptionInformation(custodialSites=["jupiter"],
primaryDataset="TwoParticles")
subInfo = testTask.getSubscriptionInformation()
outputDQMSubInfo = {"CustodialSites": ["jupiter"],
"NonCustodialSites": [],
"AutoApproveSites": [],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Replica",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "DataOps",
"Priority": "Low",
"DeleteFromSource": False}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertEqual(subInfo["/TwoParticles/DawnOfAnEra-v1/DQM"], outputDQMSubInfo,
"The DQM subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertFalse("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
testTask.setSubscriptionInformation(nonCustodialSites=["jupiter"],
primaryDataset="ThreeParticles")
subInfo = testTask.getSubscriptionInformation()
outputAODSubInfo = {"CustodialSites": [],
"NonCustodialSites": ["jupiter"],
"AutoApproveSites": [],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Replica",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "DataOps",
"Priority": "Low",
"DeleteFromSource": False}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertEqual(subInfo["/TwoParticles/DawnOfAnEra-v1/DQM"], outputDQMSubInfo,
"The DQM subscription information is wrong")
self.assertEqual(subInfo["/ThreeParticles/DawnOfAnEra-v1/MINIAOD"], outputAODSubInfo,
"The AOD subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertTrue("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
def setUp(self):
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# shut up SiteLocalConfig
os.environ['CMS_PATH'] = os.getcwd()
workload = copy.deepcopy(testWorkloads.workload)
task = workload.getTask("Production")
step = task.getStep("stageOut1")
# want to get the cmsstep so I can make the Report
cmsstep = task.getStep('cmsRun1')
self.cmsstepdir = os.path.join( self.testDir, 'cmsRun1')
os.mkdir( self.cmsstepdir )
open( os.path.join( self.cmsstepdir, '__init__.py'),'w').close()
open( os.path.join( self.cmsstepdir, 'Report.pkl'),'w').close()
cmsbuilder = CMSSWBuilder.CMSSW()
cmsbuilder( cmsstep.data, 'Production', self.cmsstepdir )
realstep = LogArchiveTemplate.LogArchiveStepHelper(step.data)
realstep.disableRetries()
self.realstep = realstep
self.stepDir = os.path.join( self.testDir, 'stepdir')
os.mkdir( self.stepDir )
builder = LogArchiveBuilder.LogArchive()
builder( step.data, 'Production', self.stepDir)
# stolen from CMSSWExecutor_t. thanks, dave
# first, delete all the sandboxen and taskspaces
# because of caching, this leaks from other tests in other files
# this sucks because the other tests are using sandboxen that
# are deleted after the test is over, which causes theses tests
# to break
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except:
pass
del sys.modules[modname]
self.oldpath = sys.path[:]
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.job = Job(name = "/UnitTests/DeleterTask/DeleteTest-test-job")
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
cmsswHelper = self.task.makeStep("cmsRun1")
cmsswHelper.setStepType('CMSSW')
stepHelper = cmsswHelper.addStep("DeleteTest")
stepHelper.setStepType('LogArchive')
self.cmsswstep = cmsswHelper.data
self.cmsswHelper = cmsswHelper
self.stepdata = stepHelper.data
self.stephelp = LogArchiveTemplate.LogArchiveStepHelper(stepHelper.data)
self.task.applyTemplates()
self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.task.build(os.path.join(self.testDir, 'UnitTests'))
sys.path.insert(0, self.testDir)
sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))
# binDir = inspect.getsourcefile(ModuleLocator)
# binDir = binDir.replace("__init__.py", "bin")
#
# if not binDir in os.environ['PATH']:
# os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
open( os.path.join( self.testDir, 'UnitTests', '__init__.py'),'w').close()
shutil.copyfile( os.path.join( os.path.dirname( __file__ ), 'MergeSuccess.pkl'),
os.path.join( self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
def buildWorkload(self):
"""
_buildWorkload_
Build the workload given all of the input parameters.
Not that there will be LogCollect tasks created for each processing
task and Cleanup tasks created for each merge task.
"""
workload = self.createWorkload()
workload.setDashboardActivity("tier0")
self.reportWorkflowToDashboard(workload.getDashboardActivity())
cmsswStepType = "CMSSW"
taskType = "Processing"
# complete output configuration
for output in self.outputs:
output['moduleLabel'] = "write_%s_%s" % (output['primaryDataset'],
output['dataTier'])
# finalize splitting parameters
mySplitArgs = self.expressSplitArgs.copy()
mySplitArgs['algo_package'] = "T0.JobSplitting"
expressTask = workload.newTask("Express")
#
# need to split this up into two separate code paths
# one is direct reco from the streamer files
# the other is conversion and then reco
#
if self.recoFrameworkVersion == None or self.recoFrameworkVersion == self.frameworkVersion:
expressRecoStepName = "cmsRun1"
scenarioArgs = {
'globalTag': self.globalTag,
'globalTagTransaction': self.globalTagTransaction,
'skims': self.alcaSkims,
'dqmSeq': self.dqmSequences,
'outputs': self.outputs,
'inputSource': "DAT"
}
if self.globalTagConnect:
scenarioArgs['globalTagConnect'] = self.globalTagConnect
expressOutMods = self.setupProcessingTask(
expressTask,
taskType,
scenarioName=self.procScenario,
scenarioFunc="expressProcessing",
scenarioArgs=scenarioArgs,
splitAlgo="Express",
splitArgs=mySplitArgs,
stepType=cmsswStepType,
forceUnmerged=True)
else:
expressRecoStepName = "cmsRun2"
conversionOutMods = self.setupProcessingTask(
expressTask,
taskType,
scenarioName=self.procScenario,
scenarioFunc="repack",
scenarioArgs={
'outputs': [{
'dataTier': "RAW",
'eventContent': "ALL",
'primaryDataset': self.specialDataset,
'moduleLabel': "write_RAW"
}]
},
splitAlgo="Express",
splitArgs=mySplitArgs,
stepType=cmsswStepType,
forceUnmerged=True)
# there is only one
conversionOutLabel = conversionOutMods.keys()[0]
# everything coming after should use the reco CMSSW version and Scram Arch
self.frameworkVersion = self.recoFrameworkVersion
self.scramArch = self.recoScramArch
# add a second step doing the reconstruction
parentCmsswStep = expressTask.getStep("cmsRun1")
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
parentCmsswStepHelper.keepOutput(False)
stepTwoCmssw = parentCmsswStep.addTopStep("cmsRun2")
stepTwoCmssw.setStepType(cmsswStepType)
template = StepFactory.getStepTemplate(cmsswStepType)
template(stepTwoCmssw.data)
stepTwoCmsswHelper = stepTwoCmssw.getTypeHelper()
if self.multicore:
# if multicore, poke in the number of cores setting
stepTwoCmsswHelper.setMulticoreCores(self.multicoreNCores)
stepTwoCmsswHelper.setGlobalTag(self.globalTag)
stepTwoCmsswHelper.setupChainedProcessing("cmsRun1",
conversionOutLabel)
stepTwoCmsswHelper.cmsswSetup(self.frameworkVersion,
softwareEnvironment="",
scramArch=self.scramArch)
scenarioFunc = "expressProcessing"
scenarioArgs = {
'globalTag': self.globalTag,
'globalTagTransaction': self.globalTagTransaction,
'skims': self.alcaSkims,
'dqmSeq': self.dqmSequences,
'outputs': self.outputs,
'inputSource': "EDM"
}
if self.globalTagConnect:
scenarioArgs['globalTagConnect'] = self.globalTagConnect
configOutput = self.determineOutputModules(scenarioFunc,
scenarioArgs)
expressOutMods = {}
for outputModuleName in configOutput.keys():
outputModule = self.addOutputModule(
expressTask,
outputModuleName,
configOutput[outputModuleName]['primaryDataset'],
configOutput[outputModuleName]['dataTier'],
configOutput[outputModuleName].get('filterName', None),
stepName="cmsRun2",
forceUnmerged=True)
expressOutMods[outputModuleName] = outputModule
if 'outputs' in scenarioArgs:
for output in scenarioArgs['outputs']:
if 'primaryDataset' in output:
del output['primaryDataset']
if 'primaryDataset' in scenarioArgs:
del scenarioArgs['primaryDataset']
stepTwoCmsswHelper.setDataProcessingConfig(self.procScenario,
scenarioFunc,
**scenarioArgs)
expressTask.setTaskType("Express")
self.addLogCollectTask(expressTask, taskName="ExpressLogCollect")
for expressOutLabel, expressOutInfo in expressOutMods.items():
if expressOutInfo['dataTier'] == "ALCARECO":
# finalize splitting parameters
mySplitArgs = self.expressMergeSplitArgs.copy()
mySplitArgs['algo_package'] = "T0.JobSplitting"
alcaSkimTask = expressTask.addTask(
"%sAlcaSkim%s" % (expressTask.name(), expressOutLabel))
alcaSkimTask.setInputReference(
expressTask.getStep(expressRecoStepName),
outputModule=expressOutLabel)
scenarioArgs = {
'globalTag': self.globalTag,
'globalTagTransaction': self.globalTagTransaction,
'skims': self.alcaSkims,
'primaryDataset': self.specialDataset
}
if self.globalTagConnect:
scenarioArgs['globalTagConnect'] = self.globalTagConnect
alcaSkimOutMods = self.setupProcessingTask(
alcaSkimTask,
taskType,
scenarioName=self.procScenario,
scenarioFunc="alcaSkim",
scenarioArgs=scenarioArgs,
splitAlgo="ExpressMerge",
splitArgs=mySplitArgs,
stepType=cmsswStepType,
forceMerged=True,
useMulticore=False)
alcaSkimTask.setTaskType("Express")
self.addLogCollectTask(alcaSkimTask,
taskName="AlcaSkimLogCollect")
self.addCleanupTask(expressTask, expressOutLabel)
for alcaSkimOutLabel, alcaSkimOutInfo in alcaSkimOutMods.items(
):
if alcaSkimOutInfo[
'dataTier'] == "ALCAPROMPT" and self.alcaHarvestDir != None:
harvestTask = self.addAlcaHarvestTask(
alcaSkimTask,
alcaSkimOutLabel,
alcapromptdataset=alcaSkimOutInfo['filterName'],
condOutLabel=self.alcaHarvestOutLabel,
condUploadDir=self.alcaHarvestDir,
uploadProxy=self.dqmUploadProxy,
doLogCollect=True)
self.addConditionTask(harvestTask,
self.alcaHarvestOutLabel)
else:
mergeTask = self.addExpressMergeTask(expressTask,
expressRecoStepName,
expressOutLabel)
if expressOutInfo['dataTier'] in ["DQM", "DQMIO"]:
self.addDQMHarvestTask(
mergeTask,
"Merged",
uploadProxy=self.dqmUploadProxy,
periodic_harvest_interval=self.periodicHarvestInterval,
doLogCollect=True)
workload.setBlockCloseSettings(self.blockCloseDelay,
workload.getBlockCloseMaxFiles(),
workload.getBlockCloseMaxEvents(),
workload.getBlockCloseMaxSize())
# setting the parameters which need to be set for all the tasks
# sets acquisitionEra, processingVersion, processingString
workload.setTaskPropertiesFromWorkload()
# set the LFN bases (normally done by request manager)
# also pass run number to add run based directories
workload.setLFNBase(self.mergedLFNBase,
self.unmergedLFNBase,
runNumber=self.runNumber)
return workload
def setupNextSteps(self, task, origArgs):
"""
_setupNextSteps_
Modify the step one task to include N more CMSSW steps and
chain the output between all three steps.
"""
self.stepParentageMapping.setdefault(origArgs['Step1']['StepName'], {})
for i in range(2, self.stepChain + 1):
currentStepNumber = "Step%d" % i
currentCmsRun = "cmsRun%d" % i
taskConf = {}
for k, v in origArgs[currentStepNumber].items():
taskConf[k] = v
parentStepNumber = self.stepMapping.get(taskConf['InputStep'])[0]
parentCmsRun = self.stepMapping.get(taskConf['InputStep'])[1]
parentCmsswStep = task.getStep(parentCmsRun)
parentCmsswStepHelper = parentCmsswStep.getTypeHelper()
# Set default values for the task parameters
self.modifyTaskConfiguration(taskConf, False, 'InputDataset' not in taskConf)
globalTag = self.getStepValue('GlobalTag', taskConf, self.globalTag)
frameworkVersion = self.getStepValue('CMSSWVersion', taskConf, self.frameworkVersion)
scramArch = self.getStepValue('ScramArch', taskConf, self.scramArch)
currentCmssw = parentCmsswStep.addTopStep(currentCmsRun)
currentCmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(currentCmssw.data)
currentCmsswStepHelper = currentCmssw.getTypeHelper()
currentCmsswStepHelper.setGlobalTag(globalTag)
currentCmsswStepHelper.setupChainedProcessing(parentCmsRun, taskConf['InputFromOutputModule'])
currentCmsswStepHelper.cmsswSetup(frameworkVersion, softwareEnvironment="", scramArch=scramArch)
currentCmsswStepHelper.setConfigCache(self.configCacheUrl, taskConf['ConfigCacheID'], self.couchDBName)
# multicore settings
multicore = self.multicore
eventStreams = self.eventStreams
if taskConf['Multicore'] > 0:
multicore = taskConf['Multicore']
if taskConf.get('EventStreams') >= 0:
eventStreams = taskConf['EventStreams']
currentCmsswStepHelper.setNumberOfCores(multicore, eventStreams)
# Pileup check
taskConf["PileupConfig"] = parsePileupConfig(taskConf["MCPileup"], taskConf["DataPileup"])
if taskConf["PileupConfig"]:
self.setupPileup(task, taskConf['PileupConfig'], stepName=currentCmsRun)
# Handling the output modules in order to decide whether we should
# stage them out and report them in the Report.pkl file
parentKeepOutput = strToBool(origArgs[parentStepNumber].get('KeepOutput', True))
parentCmsswStepHelper.keepOutput(parentKeepOutput)
childKeepOutput = strToBool(taskConf.get('KeepOutput', True))
currentCmsswStepHelper.keepOutput(childKeepOutput)
self.setupOutputModules(task, taskConf, currentCmsRun, childKeepOutput)
# Closing out the task configuration. The last step output must be saved/merged
currentCmsswStepHelper.keepOutput(True)
return