def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def testB(self):
"""
try using the helper API to set and get information in the step
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
try:
helper = template.helper(step)
except Exception as ex:
msg = "Failure to create CMSSW Step Helper"
msg += str(ex)
self.fail(msg)
helper.cmsswSetup("CMSSW_X_Y_Z", scramArch="slc5_ia32_gcc443")
helper.addOutputModule(
"outputModule1", primaryDataset="Primary",
processedDataset='Processed',
dataTier='Tier',
lfnBase="/store/unmerged/whatever"
)
def testA(self):
"""
instantiate & apply template
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
# first up, create the template
try:
template = CMSSWTemplate()
except Exception as ex:
msg = "Failed to instantiate the CMSSW Step Template"
msg += str(ex)
self.fail(msg)
# now apply it to the step
try:
template(step)
except Exception as ex:
msg = "Failed to apply template to step"
msg += str(ex)
self.fail(msg)
# TODO: Check the step has the appropriate attributes expected for CMSSW
self.assertTrue(
hasattr(step, "application"))
self.assertTrue(
hasattr(step.application, "setup"))
def testFileProperties(self):
"""
_testFileProperties_
Test some CMSSW step output file properties
"""
step = makeWMStep("cmsRun1")
step.setStepType("CMSSW")
template = CMSSWTemplate()
template(step.data)
helper = step.getTypeHelper()
# default values
self.assertIsNone(helper.getAcqEra(), None)
self.assertIsNone(helper.getProcStr(), None)
self.assertIsNone(helper.getProcVer(), None)
self.assertIsNone(helper.getPrepId(), None)
self.assertEqual(helper.listOutputModules(), [])
# now write something to the step object
helper.setAcqEra("TestAcqEra")
helper.setProcStr("TestProcStr")
helper.setProcVer(111)
helper.setPrepId("TestPrepId")
helper.addOutputModule("Merged", primaryDataset="Primary",
processedDataset="Processed", dataTier="RECO")
self.assertEqual(helper.getAcqEra(), "TestAcqEra")
self.assertEqual(helper.getProcStr(), "TestProcStr")
self.assertEqual(helper.getProcVer(), 111)
self.assertEqual(helper.getPrepId(), "TestPrepId")
self.assertItemsEqual(helper.listOutputModules(), ["Merged"])
def testA(self):
"""
instantiate & apply template
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
# first up, create the template
try:
template = CMSSWTemplate()
except Exception, ex:
msg = "Failed to instantiate the CMSSW Step Template"
msg += str(ex)
self.fail(msg)
def testMulticoreSettings(self):
"""
test multicore related methods
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
helper = template.helper(step)
self.assertEqual(helper.getNumberOfCores(), 1)
helper.setNumberOfCores(8)
self.assertEqual(helper.getNumberOfCores(), 8)
def testB(self):
"""
try using the helper API to set and get information in the step
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
try:
helper = template.helper(step)
except Exception, ex:
msg = "Failure to create CMSSW Step Helper"
msg += str(ex)
self.fail(msg)
def testChainedProcessing(self):
"""
check the chained processing set up is correct
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
step = task.makeStep("TemplateTest")
template = CMSSWTemplate()
helper = template.helper(step.data)
inputStepName = "some_inputStepName"
inputOutputModule = "some_inputOutputModule"
helper.setupChainedProcessing(inputStepName, inputOutputModule)
self.assertEqual(helper.data.input.chainedProcessing, True)
self.assertEqual(helper.data.input.inputStepName, "some_inputStepName")
self.assertEqual(helper.data.input.inputOutputModule, "some_inputOutputModule")
def setUp(self):
"""
build a step for testing purposes
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = step = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def testGPUSettings(self):
"""
Test GPU methods at CMSSW template level
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
helper = template.helper(step)
self.assertEqual(helper.getGPURequired(), "forbidden")
self.assertIsNone(helper.getGPURequirements())
helper.setGPUSettings("optional", "test 1 2 3")
self.assertEqual(helper.getGPURequired(), "optional")
self.assertItemsEqual(helper.getGPURequirements(), "test 1 2 3")
helper.setGPUSettings("required", {"key1": "value1", "key2": "value2"})
self.assertEqual(helper.getGPURequired(), "required")
self.assertItemsEqual(helper.getGPURequirements(), {
"key1": "value1",
"key2": "value2"
})
def executeCMSSWStack(opts, scram):
def getOutputModules():
pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\
"config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\
"tweakJson = makeTweak(config.process).jsondictionary();"+\
"print tweakJson[\"process\"][\"outputModules_\"]"
with tempSetLogLevel(logger=logging.getLogger(), level=logging.ERROR):
ret = scram("python -c '%s'" % pythonScript,
runtimeDir=os.getcwd())
if ret > 0:
msg = 'Error getting output modules from the pset.\n\tScram Diagnostic %s' % scram.diagnostic(
)
handleException("FAILED", EC_CMSRunWrapper, msg)
mintime()
sys.exit(EC_CMSRunWrapper)
output = literal_eval(scram.getStdout())
return output
cmssw = CMSSW()
cmssw.stepName = "cmsRun"
cmssw.step = WMStep(cmssw.stepName)
CMSSWTemplate().install(cmssw.step)
cmssw.task = makeWMTask(cmssw.stepName)
cmssw.workload = newWorkload(cmssw.stepName)
cmssw.step.application.setup.softwareEnvironment = ''
cmssw.step.application.setup.scramArch = opts.scramArch
cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion
cmssw.step.application.configuration.section_("arguments")
cmssw.step.application.configuration.arguments.globalTag = ""
for output in getOutputModules():
cmssw.step.output.modules.section_(output)
getattr(cmssw.step.output.modules, output).primaryDataset = ''
getattr(cmssw.step.output.modules, output).processedDataset = ''
getattr(cmssw.step.output.modules, output).dataTier = ''
#cmssw.step.application.command.arguments = '' #TODO
cmssw.step.user.inputSandboxes = [opts.archiveJob]
cmssw.step.user.userFiles = opts.userFiles or ''
#Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles.
#Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime..
cmssw.job = {'counter': 0, 'workflow': 'unused'}
cmssw.step.user.lfnBase = '/store/temp/user/'
cmssw.step.section_("builder")
cmssw.step.builder.workingDir = os.getcwd()
cmssw.step.runtime.invokeCommand = 'python'
cmssw.step.runtime.scramPreDir = os.getcwd()
cmssw.step.runtime.preScripts = []
cmssw.step.runtime.scramPreScripts = [
('%s/TweakPSet.py --location=%s ' + '--inputFile=\'%s\' ' +
'--runAndLumis=\'%s\' ' + '--firstEvent=%s ' + '--lastEvent=%s ' +
'--firstLumi=%s ' + '--firstRun=%s ' + '--seeding=%s ' +
'--lheInputFiles=%s ' + '--oneEventMode=%s ' + '--eventsPerLumi=%s ' +
'--maxRuntime=%s') %
(os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis,
opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun,
opts.seeding, opts.lheInputFiles, opts.oneEventMode,
opts.eventsPerLumi, opts.maxRuntime)
]
cmssw.step.section_("execution") #exitStatus of cmsRun is set here
cmssw.report = Report("cmsRun") #report is loaded and put here
cmssw.execute()
return cmssw