def testA_Execute(self):
"""
_Execute_
Test the full path, including execute.
"""
try:
os.chdir(self.step.builder.workingDir)
executor = CMSSWExecutor()
executor.initialise(self.step, self.job)
executor.pre()
# Remove the scram pre-script as it requires an actual SCRAM environment
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
executor.execute()
executor.post()
# Check that there were no errors
self.assertEqual(0, executor.report.getExitCode())
# Check that we processed the XML
self.assertEqual(2, len(executor.report.getAllFiles()))
# Check that the executable was really executed
self.assertTrue(
os.path.isfile(
os.path.join(self.step.builder.workingDir,
"BogusFile.txt")))
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testA_Execute(self):
"""
_Execute_
Test the full path, including execute.
"""
try:
os.chdir(self.step.builder.workingDir)
executor = CMSSWExecutor()
executor.initialise(self.step, self.job)
executor.pre()
# Remove the scram pre-script as it requires an actual SCRAM environment
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
executor.execute()
executor.post()
# Check that there were no errors
self.assertEqual(0, executor.report.getExitCode())
# Check that we processed the XML
self.assertEqual(2, len(executor.report.getAllFiles()))
# Check that the executable was really executed
self.assertTrue(os.path.isfile(os.path.join(self.step.builder.workingDir, "BogusFile.txt")))
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testB_Execute(self):
"""
_Execute_
This test will only run where the scram setup can be done properly.
To be honest it might have to be updated, but I don't know how.
For now I'm skipping it.
"""
executor = CMSSWExecutor()
executor.initialise(self.step, self.job)
executor.pre()
try:
executor.execute()
except WMExecutionFailure:
# This fails for me now
# If it does so, just return
return
executor.post()
print executor.report
return
def testB_Execute(self):
"""
_Execute_
This test will only run where the scram setup can be done properly.
To be honest it might have to be updated, but I don't know how.
For now I'm skipping it.
"""
executor = CMSSWExecutor()
executor.initialise(self.step, self.job)
executor.pre()
try:
executor.execute()
except WMExecutionFailure:
# This fails for me now
# If it does so, just return
return
executor.post()
print executor.report
return
def testA_PrePost(self):
"""
_PrePost_
This test should run on any machine.
It will simply make sure there are no major syntax errors or
incompatibilities.
It does not test the main functionality.
"""
executor = CMSSWExecutor()
executor.initialise(self.step, self.job)
executor.pre()
#executor.execute()
executor.post()
self.assertEqual(executor.report.data.ExecutorTest.status, 1)
self.assertEqual(executor.report.data.ExecutorTest.analysis.files.fileCount, 0)
return
def testA_PrePost(self):
"""
_PrePost_
This test should run on any machine.
It will simply make sure there are no major syntax errors or
incompatibilities.
It does not test the main functionality.
"""
executor = CMSSWExecutor()
executor.initialise(self.step, self.job)
executor.pre()
#executor.execute()
executor.post()
self.assertEqual(executor.report.data.ExecutorTest.status, 1)
self.assertEqual(
executor.report.data.ExecutorTest.analysis.files.fileCount, 0)
return
def testScramArchParsing(self):
"""
Test the various modes f parsing for the scram arch
"""
try:
os.chdir(self.step.builder.workingDir)
executor = CMSSWExecutor()
with tempfile.NamedTemporaryFile() as tf:
tf.write('GLIDEIN_REQUIRED_OS = "rhel6" \n')
tf.write('Memory = 2048\n')
tf.flush()
with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
self.assertEquals(executor.getSingleScramArch('slc6_blah_blah'), 'slc6_blah_blah')
self.assertEquals(executor.getSingleScramArch('slc5_blah_blah'), 'slc5_blah_blah')
self.assertEquals(executor.getSingleScramArch(['slc6_blah_blah', 'slc7_blah_blah']),
'slc6_blah_blah')
self.assertEquals(executor.getSingleScramArch(['slc6_blah_blah', 'slc5_blah_blah']),
'slc6_blah_blah')
self.assertEquals(executor.getSingleScramArch(['slc7_blah_blah', 'slc8_blah_blah']), None)
with tempfile.NamedTemporaryFile() as tf:
tf.write('GLIDEIN_REQUIRED_OS = "rhel7" \n')
tf.write('Memory = 2048\n')
tf.flush()
with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
self.assertEquals(executor.getSingleScramArch('slc6_blah_blah'), 'slc6_blah_blah')
self.assertEquals(executor.getSingleScramArch('slc7_blah_blah'), 'slc7_blah_blah')
self.assertEquals(executor.getSingleScramArch(['slc6_blah_blah', 'slc7_blah_blah']),
'slc7_blah_blah')
self.assertEquals(executor.getSingleScramArch(['slc6_blah_blah', 'slc5_blah_blah']), None)
self.assertEquals(executor.getSingleScramArch(['slc7_blah_blah', 'slc8_blah_blah']),
'slc7_blah_blah')
except Exception:
raise
finally:
os.chdir(self.oldCwd)
return
def executeCMSSWStack(opts, scram):
def getOutputModules():
pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\
"config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\
"tweakJson = makeTweak(config.process).jsondictionary();"+\
"print tweakJson[\"process\"][\"outputModules_\"]"
ret = scram("python -c '%s'" % pythonScript, logName=subprocess.PIPE, runtimeDir=os.getcwd())
if ret > 0:
msg = scram.diagnostic()
handleException("FAILED", EC_CMSRunWrapper, 'Error getting output modules from the pset.\n\tScram Env %s\n\tCommand:%s' % (msg, pythonScript))
mintime()
sys.exit(EC_CMSRunWrapper)
return literal_eval(scram.stdout)
cmssw = CMSSW()
cmssw.stepName = "cmsRun"
cmssw.step = WMStep(cmssw.stepName)
CMSSWTemplate().install(cmssw.step)
cmssw.task = makeWMTask(cmssw.stepName)
cmssw.workload = newWorkload(cmssw.stepName)
cmssw.step.application.setup.softwareEnvironment = ''
cmssw.step.application.setup.scramArch = opts.scramArch
cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion
cmssw.step.application.configuration.section_("arguments")
cmssw.step.application.configuration.arguments.globalTag = ""
for output in getOutputModules():
cmssw.step.output.modules.section_(output)
getattr(cmssw.step.output.modules, output).primaryDataset = ''
getattr(cmssw.step.output.modules, output).processedDataset = ''
getattr(cmssw.step.output.modules, output).dataTier = ''
#cmssw.step.application.command.arguments = '' #TODO
cmssw.step.user.inputSandboxes = [opts.archiveJob]
cmssw.step.user.userFiles = opts.userFiles or ''
#Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles.
#Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime..
cmssw.job = {'counter' : 0, 'workflow' : 'unused'}
cmssw.step.user.lfnBase = '/store/temp/user/'
cmssw.step.section_("builder")
cmssw.step.builder.workingDir = os.getcwd()
cmssw.step.runtime.invokeCommand = 'python'
cmssw.step.runtime.scramPreDir = os.getcwd()
cmssw.step.runtime.preScripts = []
cmssw.step.runtime.scramPreScripts = [('%s/TweakPSet.py --location=%s '+
'--inputFile=\'%s\' '+
'--runAndLumis=\'%s\' '+
'--firstEvent=%s '+
'--lastEvent=%s '+
'--firstLumi=%s '+
'--firstRun=%s '+
'--seeding=%s '+
'--lheInputFiles=%s '+
'--oneEventMode=%s ' +
'--eventsPerLumi=%s') %
(os.getcwd(), os.getcwd(),
opts.inputFile,
opts.runAndLumis,
opts.firstEvent,
opts.lastEvent,
opts.firstLumi,
opts.firstRun,
opts.seeding,
opts.lheInputFiles,
opts.oneEventMode,
opts.eventsPerLumi)]
cmssw.step.section_("execution") #exitStatus of cmsRun is set here
cmssw.report = Report("cmsRun") #report is loaded and put here
cmssw.execute()
return cmssw
def executeCMSSWStack(opts, scram):
def getOutputModules():
pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\
"config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\
"tweakJson = makeTweak(config.process).jsondictionary();"+\
"print tweakJson[\"process\"][\"outputModules_\"]"
with tempSetLogLevel(logger=logging.getLogger(), level=logging.ERROR):
ret = scram("python -c '%s'" % pythonScript,
runtimeDir=os.getcwd())
if ret > 0:
msg = 'Error getting output modules from the pset.\n\tScram Diagnostic %s' % scram.diagnostic(
)
handleException("FAILED", EC_CMSRunWrapper, msg)
mintime()
sys.exit(EC_CMSRunWrapper)
output = literal_eval(scram.getStdout())
return output
cmssw = CMSSW()
cmssw.stepName = "cmsRun"
cmssw.step = WMStep(cmssw.stepName)
CMSSWTemplate().install(cmssw.step)
cmssw.task = makeWMTask(cmssw.stepName)
cmssw.workload = newWorkload(cmssw.stepName)
cmssw.step.application.setup.softwareEnvironment = ''
cmssw.step.application.setup.scramArch = opts.scramArch
cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion
cmssw.step.application.configuration.section_("arguments")
cmssw.step.application.configuration.arguments.globalTag = ""
for output in getOutputModules():
cmssw.step.output.modules.section_(output)
getattr(cmssw.step.output.modules, output).primaryDataset = ''
getattr(cmssw.step.output.modules, output).processedDataset = ''
getattr(cmssw.step.output.modules, output).dataTier = ''
#cmssw.step.application.command.arguments = '' #TODO
cmssw.step.user.inputSandboxes = [opts.archiveJob]
cmssw.step.user.userFiles = opts.userFiles or ''
#Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles.
#Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime..
cmssw.job = {'counter': 0, 'workflow': 'unused'}
cmssw.step.user.lfnBase = '/store/temp/user/'
cmssw.step.section_("builder")
cmssw.step.builder.workingDir = os.getcwd()
cmssw.step.runtime.invokeCommand = 'python'
cmssw.step.runtime.scramPreDir = os.getcwd()
cmssw.step.runtime.preScripts = []
cmssw.step.runtime.scramPreScripts = [
('%s/TweakPSet.py --location=%s ' + '--inputFile=\'%s\' ' +
'--runAndLumis=\'%s\' ' + '--firstEvent=%s ' + '--lastEvent=%s ' +
'--firstLumi=%s ' + '--firstRun=%s ' + '--seeding=%s ' +
'--lheInputFiles=%s ' + '--oneEventMode=%s ' + '--eventsPerLumi=%s ' +
'--maxRuntime=%s') %
(os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis,
opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun,
opts.seeding, opts.lheInputFiles, opts.oneEventMode,
opts.eventsPerLumi, opts.maxRuntime)
]
cmssw.step.section_("execution") #exitStatus of cmsRun is set here
cmssw.report = Report("cmsRun") #report is loaded and put here
cmssw.execute()
return cmssw