def setUp(self): """ _setUp_ Build a testing environment similar to a WN """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # Build a workload/task/step with the basic required information self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" self.step.output.jobReport = "FrameworkJobReport.xml" self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus", processedDataset = "Test-Era-v1", dataTier = "DATA") self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus", processedDataset = "Test-Era-v1", dataTier = "DATA") self.helper.setGlobalTag("Bogus") taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() # Build the TaskSpace/StepSpace self.sandboxDir = os.path.join(self.testDir, "UnitTests") self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) # Copy the files that cmsRun would have generated in the step space open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close() open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close() shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) # Create a job self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job") self.job["id"] = 1 # Set the PATH binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) self.oldCwd = os.getcwd()
def setUp(self): """ _setUp_ Build a testing environment similar to a WN """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # Build a workload/task/step with the basic required information self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" self.step.output.jobReport = "FrameworkJobReport.xml" self.helper.addOutputModule("outputRECORECO", primaryDataset="Bogus", processedDataset="Test-Era-v1", dataTier="DATA") self.helper.addOutputModule("outputALCARECORECO", primaryDataset="Bogus", processedDataset="Test-Era-v1", dataTier="DATA") self.helper.setGlobalTag("Bogus") taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() # Build the TaskSpace/StepSpace self.sandboxDir = os.path.join(self.testDir, "UnitTests") self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) # Copy the files that cmsRun would have generated in the step space open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close() open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close() shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) # Create a job self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job") self.job["id"] = 1 # Set the PATH binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) self.oldCwd = os.getcwd()
def testB(self): """ try using the helper API to set and get information in the step """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") stepHelper = step = task.makeStep("TemplateTest") step = stepHelper.data template = CMSSWTemplate() template(step) try: helper = template.helper(step) except Exception as ex: msg = "Failure to create CMSSW Step Helper" msg += str(ex) self.fail(msg) helper.cmsswSetup("CMSSW_X_Y_Z", scramArch="slc5_ia32_gcc443") helper.addOutputModule( "outputModule1", primaryDataset="Primary", processedDataset='Processed', dataTier='Tier', lfnBase="/store/unmerged/whatever" )
def install(self, step): """ _install_ Install normal CMSSW fields plus the multicore information """ CMSSW.install(self, step) step.stepType = "MulticoreCMSSW" step.application.section_("multicore") step.application.multicore.numberOfCores = 1 step.application.multicore.inputfilelist = "input.filelist" step.application.multicore.inputmanifest = "manifest.json" step.application.multicore.edmFileUtil = "edmFileUtil --JSON -F input.filelist > manifest.json"
def testMulticoreSettings(self): """ test multicore related methods """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") stepHelper = step = task.makeStep("TemplateTest") step = stepHelper.data template = CMSSWTemplate() template(step) helper = template.helper(step) self.assertEqual(helper.getNumberOfCores(), 1) helper.setNumberOfCores(8) self.assertEqual(helper.getNumberOfCores(), 8)
def testB(self): """ try using the helper API to set and get information in the step """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") stepHelper = step = task.makeStep("TemplateTest") step = stepHelper.data template = CMSSWTemplate() template(step) try: helper = template.helper(step) except Exception, ex: msg = "Failure to create CMSSW Step Helper" msg += str(ex) self.fail(msg)
def testChainedProcessing(self): """ check the chained processing set up is correct """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") step = task.makeStep("TemplateTest") template = CMSSWTemplate() helper = template.helper(step.data) inputStepName = "some_inputStepName" inputOutputModule = "some_inputOutputModule" helper.setupChainedProcessing(inputStepName, inputOutputModule) self.assertEqual(helper.data.input.chainedProcessing, True) self.assertEqual(helper.data.input.inputStepName, "some_inputStepName") self.assertEqual(helper.data.input.inputOutputModule, "some_inputOutputModule")
def setUp(self): """ build a step for testing purposes """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = step = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() self.sandboxDir = "%s/UnitTests" % self.testDir self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job") binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def setUp(self): """ build a step for testing purposes """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = step = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() self.sandboxDir = "%s/UnitTests" % self.testDir self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job") binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def testGPUSettings(self): """ Test GPU methods at CMSSW template level """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") stepHelper = task.makeStep("TemplateTest") step = stepHelper.data template = CMSSWTemplate() template(step) helper = template.helper(step) self.assertEqual(helper.getGPURequired(), "forbidden") self.assertIsNone(helper.getGPURequirements()) helper.setGPUSettings("optional", "test 1 2 3") self.assertEqual(helper.getGPURequired(), "optional") self.assertItemsEqual(helper.getGPURequirements(), "test 1 2 3") helper.setGPUSettings("required", {"key1": "value1", "key2": "value2"}) self.assertEqual(helper.getGPURequired(), "required") self.assertItemsEqual(helper.getGPURequirements(), { "key1": "value1", "key2": "value2" })