def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset="Bogus",
processedDataset="Test-Era-v1",
dataTier="DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset="Bogus",
processedDataset="Test-Era-v1",
dataTier="DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def testB(self):
"""
try using the helper API to set and get information in the step
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
try:
helper = template.helper(step)
except Exception as ex:
msg = "Failure to create CMSSW Step Helper"
msg += str(ex)
self.fail(msg)
helper.cmsswSetup("CMSSW_X_Y_Z", scramArch="slc5_ia32_gcc443")
helper.addOutputModule(
"outputModule1", primaryDataset="Primary",
processedDataset='Processed',
dataTier='Tier',
lfnBase="/store/unmerged/whatever"
)
def testB(self):
"""
try using the helper API to set and get information in the step
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
try:
helper = template.helper(step)
except Exception as ex:
msg = "Failure to create CMSSW Step Helper"
msg += str(ex)
self.fail(msg)
helper.cmsswSetup("CMSSW_X_Y_Z", scramArch="slc5_ia32_gcc443")
helper.addOutputModule(
"outputModule1", primaryDataset="Primary",
processedDataset='Processed',
dataTier='Tier',
lfnBase="/store/unmerged/whatever"
)
def install(self, step):
"""
_install_
Install normal CMSSW fields plus the multicore information
"""
CMSSW.install(self, step)
step.stepType = "MulticoreCMSSW"
step.application.section_("multicore")
step.application.multicore.numberOfCores = 1
step.application.multicore.inputfilelist = "input.filelist"
step.application.multicore.inputmanifest = "manifest.json"
step.application.multicore.edmFileUtil = "edmFileUtil --JSON -F input.filelist > manifest.json"
def install(self, step):
"""
_install_
Install normal CMSSW fields plus the multicore information
"""
CMSSW.install(self, step)
step.stepType = "MulticoreCMSSW"
step.application.section_("multicore")
step.application.multicore.numberOfCores = 1
step.application.multicore.inputfilelist = "input.filelist"
step.application.multicore.inputmanifest = "manifest.json"
step.application.multicore.edmFileUtil = "edmFileUtil --JSON -F input.filelist > manifest.json"
def testMulticoreSettings(self):
"""
test multicore related methods
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
helper = template.helper(step)
self.assertEqual(helper.getNumberOfCores(), 1)
helper.setNumberOfCores(8)
self.assertEqual(helper.getNumberOfCores(), 8)
def testMulticoreSettings(self):
"""
test multicore related methods
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
helper = template.helper(step)
self.assertEqual(helper.getNumberOfCores(), 1)
helper.setNumberOfCores(8)
self.assertEqual(helper.getNumberOfCores(), 8)
def testB(self):
"""
try using the helper API to set and get information in the step
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
try:
helper = template.helper(step)
except Exception, ex:
msg = "Failure to create CMSSW Step Helper"
msg += str(ex)
self.fail(msg)
def testChainedProcessing(self):
"""
check the chained processing set up is correct
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
step = task.makeStep("TemplateTest")
template = CMSSWTemplate()
helper = template.helper(step.data)
inputStepName = "some_inputStepName"
inputOutputModule = "some_inputOutputModule"
helper.setupChainedProcessing(inputStepName, inputOutputModule)
self.assertEqual(helper.data.input.chainedProcessing, True)
self.assertEqual(helper.data.input.inputStepName, "some_inputStepName")
self.assertEqual(helper.data.input.inputOutputModule, "some_inputOutputModule")
def testChainedProcessing(self):
"""
check the chained processing set up is correct
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
step = task.makeStep("TemplateTest")
template = CMSSWTemplate()
helper = template.helper(step.data)
inputStepName = "some_inputStepName"
inputOutputModule = "some_inputOutputModule"
helper.setupChainedProcessing(inputStepName, inputOutputModule)
self.assertEqual(helper.data.input.chainedProcessing, True)
self.assertEqual(helper.data.input.inputStepName, "some_inputStepName")
self.assertEqual(helper.data.input.inputOutputModule, "some_inputOutputModule")
def testB(self):
"""
try using the helper API to set and get information in the step
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
try:
helper = template.helper(step)
except Exception, ex:
msg = "Failure to create CMSSW Step Helper"
msg += str(ex)
self.fail(msg)
def setUp(self):
"""
build a step for testing purposes
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = step = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def setUp(self):
"""
build a step for testing purposes
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = step = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def testGPUSettings(self):
"""
Test GPU methods at CMSSW template level
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
helper = template.helper(step)
self.assertEqual(helper.getGPURequired(), "forbidden")
self.assertIsNone(helper.getGPURequirements())
helper.setGPUSettings("optional", "test 1 2 3")
self.assertEqual(helper.getGPURequired(), "optional")
self.assertItemsEqual(helper.getGPURequirements(), "test 1 2 3")
helper.setGPUSettings("required", {"key1": "value1", "key2": "value2"})
self.assertEqual(helper.getGPURequired(), "required")
self.assertItemsEqual(helper.getGPURequirements(), {
"key1": "value1",
"key2": "value2"
})
