def testC(self):
"""environment setting manipulators"""
step = WMStep("CoreHelperTest")
template = Template()
template.coreInstall(step)
helper = CoreHelper(step)
try:
helper.addEnvironmentVariable("Variable1", "Value1")
except Exception as ex:
msg = "Failed to call CoreHelper.addEnvironmentVariable:\n"
msg += str(ex)
self.fail(msg)
env = helper.environment()
self.failUnless(getattr(env.variables, "Variable1", None) != None)
self.assertEqual(getattr(env.variables, "Variable1"), "Value1")
helper.addEnvironmentVariable("Variable2", "Value2")
self.failUnless(getattr(env.variables, "Variable2", None) != None)
self.assertEqual(getattr(env.variables, "Variable2"), "Value2")
try:
helper.addEnvironmentPath("Path1", "Entry1")
helper.addEnvironmentPath("Path1", "Entry2")
helper.addEnvironmentPath("Path1", "Entry3")
except Exception as ex:
msg = "Failed to call CoreHelper.addEnvironmentPath:\n"
msg += str(ex)
self.fail(msg)
path1 = getattr(env.paths, "Path1", None)
self.failUnless(path1 != None)
self.assertEqual(path1, ['Entry1', 'Entry2', 'Entry3'])
def testC(self):
"""exceptions"""
template = Template()
step = WMStep("TestStep")
self.assertRaises(NotImplementedError, template.install, step)
self.assertRaises(NotImplementedError, template.helper, step)
def testA(self):
"""instantiation"""
try:
helper = CoreHelper(WMStep("TestHelper"))
except Exception as ex:
msg = "Failed to instantiate CoreHelper object:\n"
msg += str(ex)
self.fail(msg)
def testA(self):
"""instantiation"""
try:
wmStep = WMStep("step1")
except Exception, ex:
msg = "Failed to instantiate WMStep:\n"
msg += str(ex)
self.fail(msg)
def testB(self):
"""test stepName"""
stepName = "StepNameGoesHere"
helper = CoreHelper(WMStep(stepName))
try:
name = helper.stepName()
except Exception, ex:
msg = "Failed to call CoreHelper.stepName:\n"
msg += str(ex)
self.fail(msg)
def makeStep(self, stepName):
"""
_makeStep_
create a new WMStep instance, install it as the top step and
return the reference to the new step wrapped in a StepHelper
"""
newStep = WMStep(stepName)
self.setStep(newStep)
return WMStepHelper(newStep)
def testB(self):
"""test coreInstall method"""
step = WMStep("TestStep")
template = Template()
try:
template.coreInstall(step)
except Exception, ex:
msg = "Error calling Template.coreInstall(step)\n"
msg += str(ex)
self.fail(msg)
def testC(self):
"""environment setting manipulators"""
step = WMStep("CoreHelperTest")
template = Template()
template.coreInstall(step)
helper = CoreHelper(step)
try:
helper.addEnvironmentVariable("Variable1", "Value1")
except Exception, ex:
msg = "Failed to call CoreHelper.addEnvironmentVariable:\n"
msg += str(ex)
self.fail(msg)
def createTestStep(self):
"""
_createTestStep_
Create a test step that can be passed to the setup script.
"""
newStep = WMStep("cmsRun1")
newStepHelper = CMSSWStepHelper(newStep)
newStepHelper.setStepType("CMSSW")
newStepHelper.setGlobalTag("SomeGlobalTag")
stepTemplate = StepFactory.getStepTemplate("CMSSW")
stepTemplate(newStep)
newStep.application.command.configuration = "PSet.py"
return newStepHelper
def testD(self):
"""build/dir/file structure manipulators"""
step = WMStep("CoreHelperTest")
template = Template()
template.coreInstall(step)
helper = CoreHelper(step)
try:
helper.addDirectory("dir1")
helper.addDirectory("dir1/dir2")
helper.addDirectory("dir1/dir3")
helper.addDirectory("dir1/dir4")
helper.addDirectory("/dir1/dir5")
except Exception as ex:
msg = "Error calling CoreHelper.addDirectory\n"
msg += str(ex)
self.fail(msg)
dirs = helper.directoryStructure()
self.assertTrue(hasattr(dirs, helper.stepName()))
stepDir = getattr(dirs, helper.stepName())
self.assertTrue(hasattr(stepDir, "dir1"))
dir1 = getattr(stepDir, "dir1")
dir2 = getattr(dir1, "dir2")
self.assertTrue(hasattr(dir1, "dir2"))
self.assertTrue(hasattr(dir1, "dir3"))
self.assertTrue(hasattr(dir1, "dir4"))
self.assertTrue(hasattr(dir1, "dir5"))
helper.addFile("file1")
helper.addFile("file2", "dir1/dir2/file2")
self.assertTrue(hasattr(stepDir, "file1"))
self.assertTrue(hasattr(dir2, "file2"))
file1 = getattr(stepDir, "file1")
file2 = getattr(dir2, "file2")
self.assertEqual(file1['Source'], "file1")
self.assertEqual(file1['Target'], "file1")
self.assertEqual(file2['Source'], "file2")
self.assertEqual(file2['Target'], "file2")
def testD(self):
"""build/dir/file structure manipulators"""
step = WMStep("CoreHelperTest")
template = Template()
template.coreInstall(step)
helper = CoreHelper(step)
try:
helper.addDirectory("dir1")
helper.addDirectory("dir1/dir2")
helper.addDirectory("dir1/dir3")
helper.addDirectory("dir1/dir4")
helper.addDirectory("/dir1/dir5")
except Exception, ex:
msg = "Error calling CoreHelper.addDirectory\n"
msg += str(ex)
self.fail(msg)
def createTestStep(self):
"""
_createTestStep_
Create a test step that can be passed to the setup script.
"""
newStep = WMStep("cmsRun1")
stepTemplate = StepFactory.getStepTemplate("CMSSW")
stepTemplate.install(newStep)
newStepHelper = stepTemplate.helper(newStep)
newStepHelper.setStepType("CMSSW")
newStepHelper.setGlobalTag("SomeGlobalTag")
newStepHelper.data.application.section_("setup")
newStepHelper.cmsswSetup("CMSSW_11_0_2", scramArch=['slc7_amd64_gcc820'])
return newStepHelper
def testB(self):
"""test coreInstall method"""
step = WMStep("TestStep")
template = Template()
try:
template.coreInstall(step)
except Exception as ex:
msg = "Error calling Template.coreInstall(step)\n"
msg += str(ex)
self.fail(msg)
# check environment installation
self.failUnless(getattr(step, "environment", None) != None)
env = getattr(step, "environment")
self.failUnless(getattr(env, "paths", None) != None)
self.failUnless(getattr(env, "variables", None) != None)
# check build installation
self.failUnless(getattr(step, "build", None) != None)
build = getattr(step, "build")
self.failUnless(getattr(build, "directories", None) != None)
def executeCMSSWStack(opts, scram):
def getOutputModules():
pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\
"config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\
"tweakJson = makeTweak(config.process).jsondictionary();"+\
"print tweakJson[\"process\"][\"outputModules_\"]"
with tempSetLogLevel(logger=logging.getLogger(), level=logging.ERROR):
ret = scram("python -c '%s'" % pythonScript,
runtimeDir=os.getcwd())
if ret > 0:
msg = 'Error getting output modules from the pset.\n\tScram Diagnostic %s' % scram.diagnostic(
)
handleException("FAILED", EC_CMSRunWrapper, msg)
mintime()
sys.exit(EC_CMSRunWrapper)
output = literal_eval(scram.getStdout())
return output
cmssw = CMSSW()
cmssw.stepName = "cmsRun"
cmssw.step = WMStep(cmssw.stepName)
CMSSWTemplate().install(cmssw.step)
cmssw.task = makeWMTask(cmssw.stepName)
cmssw.workload = newWorkload(cmssw.stepName)
cmssw.step.application.setup.softwareEnvironment = ''
cmssw.step.application.setup.scramArch = opts.scramArch
cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion
cmssw.step.application.configuration.section_("arguments")
cmssw.step.application.configuration.arguments.globalTag = ""
for output in getOutputModules():
cmssw.step.output.modules.section_(output)
getattr(cmssw.step.output.modules, output).primaryDataset = ''
getattr(cmssw.step.output.modules, output).processedDataset = ''
getattr(cmssw.step.output.modules, output).dataTier = ''
#cmssw.step.application.command.arguments = '' #TODO
cmssw.step.user.inputSandboxes = [opts.archiveJob]
cmssw.step.user.userFiles = opts.userFiles or ''
#Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles.
#Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime..
cmssw.job = {'counter': 0, 'workflow': 'unused'}
cmssw.step.user.lfnBase = '/store/temp/user/'
cmssw.step.section_("builder")
cmssw.step.builder.workingDir = os.getcwd()
cmssw.step.runtime.invokeCommand = 'python'
cmssw.step.runtime.scramPreDir = os.getcwd()
cmssw.step.runtime.preScripts = []
cmssw.step.runtime.scramPreScripts = [
('%s/TweakPSet.py --location=%s ' + '--inputFile=\'%s\' ' +
'--runAndLumis=\'%s\' ' + '--firstEvent=%s ' + '--lastEvent=%s ' +
'--firstLumi=%s ' + '--firstRun=%s ' + '--seeding=%s ' +
'--lheInputFiles=%s ' + '--oneEventMode=%s ' + '--eventsPerLumi=%s ' +
'--maxRuntime=%s') %
(os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis,
opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun,
opts.seeding, opts.lheInputFiles, opts.oneEventMode,
opts.eventsPerLumi, opts.maxRuntime)
]
cmssw.step.section_("execution") #exitStatus of cmsRun is set here
cmssw.report = Report("cmsRun") #report is loaded and put here
cmssw.execute()
return cmssw
