def testAddingSteps(self):
"""
_testAddingSteps_
Verify that adding steps to a task works correctly.
"""
task1 = makeWMTask("task1")
task2a = task1.addTask("task2a")
task2b = task1.addTask("task2b")
task2c = task1.addTask("task2c")
task3 = task2a.addTask("task3")
step1 = makeWMStep("step1")
step1.addStep("step1a")
step1.addStep("step1b")
step1.addStep("step1c")
step2 = makeWMStep("step2")
step2.addStep("step2a")
step2.addStep("step2b")
step2.addStep("step2c")
step3 = makeWMStep("step3")
step3.addStep("step3a")
step3.addStep("step3b")
step3.addStep("step3c")
step4 = makeWMStep("step4")
step4.addStep("step4a")
step4.addStep("step4b")
step4.addStep("step4c")
task1.setStep(step1)
task2a.setStep(step2)
task2b.setStep(step3)
task2c.setStep(step4)
self.assertEqual(task1.getStep("step1a").name(), "step1a")
self.assertEqual(task2a.getStep("step2b").name(), "step2b")
self.assertEqual(task2b.getStep("step3c").name(), "step3c")
self.assertEqual(task2c.getStep("step4a").name(), "step4a")
self.assertEqual(task1.getStep("step2"), None)
self.assertEqual(task2a.getStep("step4"), None)
self.assertEqual(task2b.getStep("step2"), None)
self.assertEqual(task2c.getStep("step1"), None)
self.assertEqual(task1.listNodes(),
['task1', 'task2a', 'task3', 'task2b', 'task2c'])
return
def testAddingSteps(self):
"""
_testAddingSteps_
Verify that adding steps to a task works correctly.
"""
task1 = makeWMTask("task1")
task2a = task1.addTask("task2a")
task2b = task1.addTask("task2b")
task2c = task1.addTask("task2c")
task3 = task2a.addTask("task3")
step1 = makeWMStep("step1")
step1.addStep("step1a")
step1.addStep("step1b")
step1.addStep("step1c")
step2 = makeWMStep("step2")
step2.addStep("step2a")
step2.addStep("step2b")
step2.addStep("step2c")
step3 = makeWMStep("step3")
step3.addStep("step3a")
step3.addStep("step3b")
step3.addStep("step3c")
step4 = makeWMStep("step4")
step4.addStep("step4a")
step4.addStep("step4b")
step4.addStep("step4c")
task1.setStep(step1)
task2a.setStep(step2)
task2b.setStep(step3)
task2c.setStep(step4)
self.assertEqual(task1.getStep("step1a").name(), "step1a")
self.assertEqual(task2a.getStep("step2b").name(), "step2b")
self.assertEqual(task2b.getStep("step3c").name(), "step3c")
self.assertEqual(task2c.getStep("step4a").name(), "step4a")
self.assertEqual(task1.getStep("step2"), None)
self.assertEqual(task2a.getStep("step4"), None)
self.assertEqual(task2b.getStep("step2"), None)
self.assertEqual(task2c.getStep("step1"), None)
self.assertEqual(task1.listNodes(),
['task1', 'task2a', 'task3', 'task2b', 'task2c'])
return
def testFileProperties(self):
"""
_testFileProperties_
Test some CMSSW step output file properties
"""
step = makeWMStep("cmsRun1")
step.setStepType("CMSSW")
template = CMSSWTemplate()
template(step.data)
helper = step.getTypeHelper()
# default values
self.assertIsNone(helper.getAcqEra(), None)
self.assertIsNone(helper.getProcStr(), None)
self.assertIsNone(helper.getProcVer(), None)
self.assertIsNone(helper.getPrepId(), None)
self.assertEqual(helper.listOutputModules(), [])
# now write something to the step object
helper.setAcqEra("TestAcqEra")
helper.setProcStr("TestProcStr")
helper.setProcVer(111)
helper.setPrepId("TestPrepId")
helper.addOutputModule("Merged", primaryDataset="Primary",
processedDataset="Processed", dataTier="RECO")
self.assertEqual(helper.getAcqEra(), "TestAcqEra")
self.assertEqual(helper.getProcStr(), "TestProcStr")
self.assertEqual(helper.getProcVer(), 111)
self.assertEqual(helper.getPrepId(), "TestPrepId")
self.assertItemsEqual(helper.listOutputModules(), ["Merged"])
def testGPUSettings(self):
"""
Test GPU settings and the 'getGPURequired' and 'getGPURequirements' methods
"""
# create a standard step object - without the CMSSW template applied
wmStep = makeWMStep("step1")
self.assertIsNone(wmStep.stepType())
self.assertFalse(hasattr(wmStep.data, "gpu"))
self.assertIsNone(wmStep.getGPURequired())
# now apply the CMSSW template
wmStep.setStepType("CMSSW")
self.assertEqual(wmStep.stepType(), "CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(wmStep.data)
wmStepHelper = wmStep.getTypeHelper()
self.assertEqual(wmStepHelper.getGPURequired(), "forbidden")
self.assertIsNone(wmStepHelper.getGPURequirements())
gpuParams = {
"GPUMemoryMB": 1234,
"CUDARuntime": "11.2.3",
"CUDACapabilities": ["7.5", "8.0"]
}
wmStepHelper.setGPUSettings("required", gpuParams)
self.assertEqual(wmStepHelper.getGPURequired(), "required")
self.assertItemsEqual(wmStepHelper.getGPURequirements(), gpuParams)
return
def testC_testGetSetOverrides(self):
"""
Test whether we can use the override manipulation tools in StepTypeHelper
"""
wmStep = makeWMStep("step2")
output = wmStep.getOverrides()
# This should be empty since we haven't put anything in it
self.assertEqual(output, {})
wmStep.addOverride(override = 'test', overrideValue = 'nonsense')
self.assertTrue(hasattr(wmStep.data, 'override'))
self.assertTrue(hasattr(wmStep.data.override, 'test'))
self.assertEqual(wmStep.data.override.test, 'nonsense')
output = wmStep.getOverrides()
self.assertEqual(output, {'test': 'nonsense'})
return
def testB(self):
"""tree building"""
wmStep1 = makeWMStep("step1")
wmStep1.setStepType("TYPE1")
wmStep2a = wmStep1.addStep("step2a")
wmStep2a.setStepType("TYPE2")
wmStep2b = wmStep1.addStep("step2b")
wmStep2b.setStepType("TYPE3")
wmStep2c = wmStep1.addStep("step2c")
wmStep2c.setStepType("TYPE4")
wmStep3a = wmStep2a.addStep("step3a")
wmStep3a.setStepType("TYPE5")
wmStep3b = wmStep2a.addStep("step3b")
wmStep3b.setStepType("TYPE6")
wmStep3c = wmStep2a.addStep("step3c")
wmStep3c.setStepType("TYPE7")
wmStep3d = wmStep2b.addStep("step3d")
wmStep3d.setStepType("TYPE8")
wmStep3e = wmStep2b.addStep("step3e")
wmStep3e.setStepType("TYPE9")
wmStep3f = wmStep2b.addStep("step3f")
wmStep3f.setStepType("TYPE10")
wmStep3g = wmStep2c.addStep("step3g")
wmStep3g.setStepType("TYPE11")
wmStep3h = wmStep2c.addStep("step3h")
wmStep3h.setStepType("TYPE12")
wmStep3i = wmStep2c.addStep("step3i")
wmStep3i.setStepType("TYPE13")
nameOrder = [
'step1', 'step2a', 'step3a', 'step3b', 'step3c', 'step2b',
'step3d', 'step3e', 'step3f', 'step2c', 'step3g', 'step3h',
'step3i'
]
typeOrder = [
'TYPE1', 'TYPE2', 'TYPE5', 'TYPE6', 'TYPE7', 'TYPE3', 'TYPE8',
'TYPE9', 'TYPE10', 'TYPE4', 'TYPE11', 'TYPE12', 'TYPE13'
]
checkType = [x.stepType for x in wmStep1.nodeIterator()]
checkOrder = [x._internal_name for x in wmStep1.nodeIterator()]
self.assertEqual(nameOrder, checkOrder)
self.assertEqual(typeOrder, checkType)
def testA(self):
"""instantiation"""
try:
wmStep = WMStep("step1")
except Exception as ex:
msg = "Failed to instantiate WMStep:\n"
msg += str(ex)
self.fail(msg)
try:
wmStep2 = makeWMStep("step2")
except Exception as ex:
msg = "Failed to instantiate WMStep via makeWMStep:\n"
msg += str(ex)
self.fail(msg)
def testA(self):
"""instantiation"""
try:
wmStep = WMStep("step1")
except Exception as ex:
msg = "Failed to instantiate WMStep:\n"
msg += str(ex)
self.fail(msg)
try:
wmStep2 = makeWMStep("step2")
except Exception as ex:
msg = "Failed to instantiate WMStep via makeWMStep:\n"
msg += str(ex)
self.fail(msg)
def testD_getOutputModule(self):
"""
Test our ability to get an output module
"""
wmStep = makeWMStep("step2")
wmStep.data.output.section_('modules')
wmStep.data.output.modules.section_('test')
setattr(wmStep.data.output.modules.test, 'tester', 'nonsense')
testModule = wmStep.getOutputModule(moduleName='test')
self.assertEqual(testModule.tester, 'nonsense')
return
def testD_getOutputModule(self):
"""
Test our ability to get an output module
"""
wmStep = makeWMStep("step2")
wmStep.data.output.section_('modules')
wmStep.data.output.modules.section_('test')
setattr(wmStep.data.output.modules.test, 'tester', 'nonsense')
testModule = wmStep.getOutputModule(moduleName = 'test')
self.assertEqual(testModule.tester, 'nonsense')
return
def testB(self):
"""tree building"""
wmStep1 = makeWMStep("step1")
wmStep1.setStepType("TYPE1")
wmStep2a = wmStep1.addStep("step2a")
wmStep2a.setStepType("TYPE2")
wmStep2b = wmStep1.addStep("step2b")
wmStep2b.setStepType("TYPE3")
wmStep2c = wmStep1.addStep("step2c")
wmStep2c.setStepType("TYPE4")
wmStep3a = wmStep2a.addStep("step3a")
wmStep3a.setStepType("TYPE5")
wmStep3b = wmStep2a.addStep("step3b")
wmStep3b.setStepType("TYPE6")
wmStep3c = wmStep2a.addStep("step3c")
wmStep3c.setStepType("TYPE7")
wmStep3d = wmStep2b.addStep("step3d")
wmStep3d.setStepType("TYPE8")
wmStep3e = wmStep2b.addStep("step3e")
wmStep3e.setStepType("TYPE9")
wmStep3f = wmStep2b.addStep("step3f")
wmStep3f.setStepType("TYPE10")
wmStep3g = wmStep2c.addStep("step3g")
wmStep3g.setStepType("TYPE11")
wmStep3h = wmStep2c.addStep("step3h")
wmStep3h.setStepType("TYPE12")
wmStep3i = wmStep2c.addStep("step3i")
wmStep3i.setStepType("TYPE13")
nameOrder = ['step1', 'step2a', 'step3a', 'step3b', 'step3c',
'step2b', 'step3d', 'step3e', 'step3f', 'step2c',
'step3g', 'step3h', 'step3i']
typeOrder = ['TYPE1', 'TYPE2', 'TYPE5', 'TYPE6', 'TYPE7',
'TYPE3', 'TYPE8', 'TYPE9', 'TYPE10', 'TYPE4',
'TYPE11', 'TYPE12', 'TYPE13']
checkType = [ x.stepType for x in wmStep1.nodeIterator()]
checkOrder = [ x._internal_name for x in wmStep1.nodeIterator()]
self.assertEqual(nameOrder, checkOrder)
self.assertEqual(typeOrder, checkType)
class WMStepTest(unittest.TestCase):
"""
TestCase for WMStep class
"""
def testA(self):
"""instantiation"""
try:
wmStep = WMStep("step1")
except Exception, ex:
msg = "Failed to instantiate WMStep:\n"
msg += str(ex)
self.fail(msg)
try:
wmStep2 = makeWMStep("step2")
except Exception, ex:
msg = "Failed to instantiate WMStep via makeWMStep:\n"
msg += str(ex)
self.fail(msg)
def testE_Properties(self):
"""
_Properties_
Test the various properties that we have set in the step
"""
wmStep = makeWMStep("step2")
# errorDestinatio
self.assertEqual(wmStep.getErrorDestinationStep(), None)
wmStep.setErrorDestinationStep(stepName='testStep')
self.assertEqual(wmStep.getErrorDestinationStep(), 'testStep')
self.assertEqual(wmStep.getConfigInfo(), (None, None, None))
wmStep.data.application.configuration.configCacheUrl = 'test1'
wmStep.data.application.configuration.cacheName = 'test2'
wmStep.data.application.configuration.configId = 'test3'
self.assertEqual(wmStep.getConfigInfo(), ('test1', 'test2', 'test3'))
return
def testE_Properties(self):
"""
_Properties_
Test the various properties that we have set in the step
"""
wmStep = makeWMStep("step2")
# errorDestinatio
self.assertEqual(wmStep.getErrorDestinationStep(), None)
wmStep.setErrorDestinationStep(stepName = 'testStep')
self.assertEqual(wmStep.getErrorDestinationStep(), 'testStep')
self.assertEqual(wmStep.getConfigInfo(), (None, None, None))
wmStep.data.application.configuration.configCacheUrl = 'test1'
wmStep.data.application.configuration.cacheName = 'test2'
wmStep.data.application.configuration.configId = 'test3'
self.assertEqual(wmStep.getConfigInfo(), ('test1', 'test2', 'test3'))
return
def testInstallWorkingArea(self):
'''make sure installWorkingArea changes the attr in the data object'''
mystep = makeWMStep("demostep")
self.testBuilder.installWorkingArea(mystep.data, "workingdir")
self.assertEqual(mystep.data.builder.workingDir, "workingdir")
def testBuild(self):
'''make sure the build method throws an exception'''
mystep = makeWMStep("demostep")
self.assertRaises(NotImplementedError, self.testBuilder.build, mystep,
'/')
def testInstallWorkingArea(self):
'''make sure installWorkingArea changes the attr in the data object'''
mystep = makeWMStep("demostep")
self.testBuilder.installWorkingArea(mystep.data,"workingdir")
self.assertEqual(mystep.data.builder.workingDir, "workingdir")
def testBuild(self):
'''make sure the build method throws an exception'''
mystep = makeWMStep("demostep")
self.assertRaises(NotImplementedError, self.testBuilder.build, mystep,
'/')