def testC(self): """environment setting manipulators""" step = WMStep("CoreHelperTest") template = Template() template.coreInstall(step) helper = CoreHelper(step) try: helper.addEnvironmentVariable("Variable1", "Value1") except Exception as ex: msg = "Failed to call CoreHelper.addEnvironmentVariable:\n" msg += str(ex) self.fail(msg) env = helper.environment() self.failUnless(getattr(env.variables, "Variable1", None) != None) self.assertEqual(getattr(env.variables, "Variable1"), "Value1") helper.addEnvironmentVariable("Variable2", "Value2") self.failUnless(getattr(env.variables, "Variable2", None) != None) self.assertEqual(getattr(env.variables, "Variable2"), "Value2") try: helper.addEnvironmentPath("Path1", "Entry1") helper.addEnvironmentPath("Path1", "Entry2") helper.addEnvironmentPath("Path1", "Entry3") except Exception as ex: msg = "Failed to call CoreHelper.addEnvironmentPath:\n" msg += str(ex) self.fail(msg) path1 = getattr(env.paths, "Path1", None) self.failUnless(path1 != None) self.assertEqual(path1, ['Entry1', 'Entry2', 'Entry3'])
def testC(self): """exceptions""" template = Template() step = WMStep("TestStep") self.assertRaises(NotImplementedError, template.install, step) self.assertRaises(NotImplementedError, template.helper, step)
def testA(self): """instantiation""" try: helper = CoreHelper(WMStep("TestHelper")) except Exception as ex: msg = "Failed to instantiate CoreHelper object:\n" msg += str(ex) self.fail(msg)
def testA(self): """instantiation""" try: wmStep = WMStep("step1") except Exception, ex: msg = "Failed to instantiate WMStep:\n" msg += str(ex) self.fail(msg)
def testB(self): """test stepName""" stepName = "StepNameGoesHere" helper = CoreHelper(WMStep(stepName)) try: name = helper.stepName() except Exception, ex: msg = "Failed to call CoreHelper.stepName:\n" msg += str(ex) self.fail(msg)
def makeStep(self, stepName): """ _makeStep_ create a new WMStep instance, install it as the top step and return the reference to the new step wrapped in a StepHelper """ newStep = WMStep(stepName) self.setStep(newStep) return WMStepHelper(newStep)
def testB(self): """test coreInstall method""" step = WMStep("TestStep") template = Template() try: template.coreInstall(step) except Exception, ex: msg = "Error calling Template.coreInstall(step)\n" msg += str(ex) self.fail(msg)
def testC(self): """environment setting manipulators""" step = WMStep("CoreHelperTest") template = Template() template.coreInstall(step) helper = CoreHelper(step) try: helper.addEnvironmentVariable("Variable1", "Value1") except Exception, ex: msg = "Failed to call CoreHelper.addEnvironmentVariable:\n" msg += str(ex) self.fail(msg)
def createTestStep(self): """ _createTestStep_ Create a test step that can be passed to the setup script. """ newStep = WMStep("cmsRun1") newStepHelper = CMSSWStepHelper(newStep) newStepHelper.setStepType("CMSSW") newStepHelper.setGlobalTag("SomeGlobalTag") stepTemplate = StepFactory.getStepTemplate("CMSSW") stepTemplate(newStep) newStep.application.command.configuration = "PSet.py" return newStepHelper
def testD(self): """build/dir/file structure manipulators""" step = WMStep("CoreHelperTest") template = Template() template.coreInstall(step) helper = CoreHelper(step) try: helper.addDirectory("dir1") helper.addDirectory("dir1/dir2") helper.addDirectory("dir1/dir3") helper.addDirectory("dir1/dir4") helper.addDirectory("/dir1/dir5") except Exception as ex: msg = "Error calling CoreHelper.addDirectory\n" msg += str(ex) self.fail(msg) dirs = helper.directoryStructure() self.assertTrue(hasattr(dirs, helper.stepName())) stepDir = getattr(dirs, helper.stepName()) self.assertTrue(hasattr(stepDir, "dir1")) dir1 = getattr(stepDir, "dir1") dir2 = getattr(dir1, "dir2") self.assertTrue(hasattr(dir1, "dir2")) self.assertTrue(hasattr(dir1, "dir3")) self.assertTrue(hasattr(dir1, "dir4")) self.assertTrue(hasattr(dir1, "dir5")) helper.addFile("file1") helper.addFile("file2", "dir1/dir2/file2") self.assertTrue(hasattr(stepDir, "file1")) self.assertTrue(hasattr(dir2, "file2")) file1 = getattr(stepDir, "file1") file2 = getattr(dir2, "file2") self.assertEqual(file1['Source'], "file1") self.assertEqual(file1['Target'], "file1") self.assertEqual(file2['Source'], "file2") self.assertEqual(file2['Target'], "file2")
def testD(self): """build/dir/file structure manipulators""" step = WMStep("CoreHelperTest") template = Template() template.coreInstall(step) helper = CoreHelper(step) try: helper.addDirectory("dir1") helper.addDirectory("dir1/dir2") helper.addDirectory("dir1/dir3") helper.addDirectory("dir1/dir4") helper.addDirectory("/dir1/dir5") except Exception, ex: msg = "Error calling CoreHelper.addDirectory\n" msg += str(ex) self.fail(msg)
def createTestStep(self): """ _createTestStep_ Create a test step that can be passed to the setup script. """ newStep = WMStep("cmsRun1") stepTemplate = StepFactory.getStepTemplate("CMSSW") stepTemplate.install(newStep) newStepHelper = stepTemplate.helper(newStep) newStepHelper.setStepType("CMSSW") newStepHelper.setGlobalTag("SomeGlobalTag") newStepHelper.data.application.section_("setup") newStepHelper.cmsswSetup("CMSSW_11_0_2", scramArch=['slc7_amd64_gcc820']) return newStepHelper
def testB(self): """test coreInstall method""" step = WMStep("TestStep") template = Template() try: template.coreInstall(step) except Exception as ex: msg = "Error calling Template.coreInstall(step)\n" msg += str(ex) self.fail(msg) # check environment installation self.failUnless(getattr(step, "environment", None) != None) env = getattr(step, "environment") self.failUnless(getattr(env, "paths", None) != None) self.failUnless(getattr(env, "variables", None) != None) # check build installation self.failUnless(getattr(step, "build", None) != None) build = getattr(step, "build") self.failUnless(getattr(build, "directories", None) != None)
def executeCMSSWStack(opts, scram): def getOutputModules(): pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\ "config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\ "tweakJson = makeTweak(config.process).jsondictionary();"+\ "print tweakJson[\"process\"][\"outputModules_\"]" with tempSetLogLevel(logger=logging.getLogger(), level=logging.ERROR): ret = scram("python -c '%s'" % pythonScript, runtimeDir=os.getcwd()) if ret > 0: msg = 'Error getting output modules from the pset.\n\tScram Diagnostic %s' % scram.diagnostic( ) handleException("FAILED", EC_CMSRunWrapper, msg) mintime() sys.exit(EC_CMSRunWrapper) output = literal_eval(scram.getStdout()) return output cmssw = CMSSW() cmssw.stepName = "cmsRun" cmssw.step = WMStep(cmssw.stepName) CMSSWTemplate().install(cmssw.step) cmssw.task = makeWMTask(cmssw.stepName) cmssw.workload = newWorkload(cmssw.stepName) cmssw.step.application.setup.softwareEnvironment = '' cmssw.step.application.setup.scramArch = opts.scramArch cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion cmssw.step.application.configuration.section_("arguments") cmssw.step.application.configuration.arguments.globalTag = "" for output in getOutputModules(): cmssw.step.output.modules.section_(output) getattr(cmssw.step.output.modules, output).primaryDataset = '' getattr(cmssw.step.output.modules, output).processedDataset = '' getattr(cmssw.step.output.modules, output).dataTier = '' #cmssw.step.application.command.arguments = '' #TODO cmssw.step.user.inputSandboxes = [opts.archiveJob] cmssw.step.user.userFiles = opts.userFiles or '' #Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles. #Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime.. cmssw.job = {'counter': 0, 'workflow': 'unused'} cmssw.step.user.lfnBase = '/store/temp/user/' cmssw.step.section_("builder") cmssw.step.builder.workingDir = os.getcwd() cmssw.step.runtime.invokeCommand = 'python' cmssw.step.runtime.scramPreDir = os.getcwd() cmssw.step.runtime.preScripts = [] cmssw.step.runtime.scramPreScripts = [ ('%s/TweakPSet.py --location=%s ' + '--inputFile=\'%s\' ' + '--runAndLumis=\'%s\' ' + '--firstEvent=%s ' + '--lastEvent=%s ' + '--firstLumi=%s ' + '--firstRun=%s ' + '--seeding=%s ' + '--lheInputFiles=%s ' + '--oneEventMode=%s ' + '--eventsPerLumi=%s ' + '--maxRuntime=%s') % (os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis, opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun, opts.seeding, opts.lheInputFiles, opts.oneEventMode, opts.eventsPerLumi, opts.maxRuntime) ] cmssw.step.section_("execution") #exitStatus of cmsRun is set here cmssw.report = Report("cmsRun") #report is loaded and put here cmssw.execute() return cmssw