def testPileupFetcherOnMC(self):
pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments()
pileupMcArgs["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO"
pileupMcArgs["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW"
pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", pileupMcArgs)
# now that the workload was created and args validated, we can add this PileupConfig
pileupMcArgs["PileupConfig"] = parsePileupConfig(
pileupMcArgs["MCPileup"], pileupMcArgs["DataPileup"])
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % (self.testDir, testWorkload.name())
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def testPileupFetcherOnMC(self):
pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments()
pileupMcArgs["PileupConfig"] = {
"cosmics": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"minbias": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", pileupMcArgs)
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % (self.testDir, testWorkload.name())
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def testMakeSandbox(self):
creator = SandboxCreator.SandboxCreator()
workload = TestWorkloads.twoTaskTree()
tempdir = tempfile.mkdtemp()
# test that the existing path is deleted else it will crash as in issue #5130
os.makedirs('%s/%s/WMSandbox' % (tempdir, workload.name()))
boxpath = creator.makeSandbox(tempdir, workload)
# extract our sandbox to test it
extractDir = tempfile.mkdtemp()
tarHandle = tarfile.open(boxpath, 'r:bz2')
tarHandle.extractall(extractDir)
self.fileExistsTest(extractDir + "/WMSandbox")
self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl")
self.fileExistsTest(extractDir + "/WMSandbox/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1")
self.fileExistsTest(extractDir +
"/WMSandbox/FirstTask/cmsRun1/__init__.py")
self.fileExistsTest(extractDir +
"/WMSandbox/FirstTask/stageOut1/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2")
self.fileExistsTest(extractDir +
"/WMSandbox/SecondTask/cmsRun2/__init__.py")
self.fileExistsTest(extractDir +
"/WMSandbox/SecondTask/stageOut2/__init__.py")
# make sure the sandbox is there
self.fileExistsTest(extractDir + '/WMCore.zip')
# Test that zipimport works on the dummy module that SandboxCreator inserts
output = subprocess.check_output(
['python', '-m', 'WMCore.ZipImportTestModule'],
env={'PYTHONPATH': os.path.join(extractDir, 'WMCore.zip')})
self.assertIn('ZIPIMPORTTESTOK', output)
# make sure the pickled file is the same
pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl")
pickledWorkload = pickle.load(pickleHandle)
self.assertEqual(workload.data, pickledWorkload)
self.assertEqual(pickledWorkload.sandbox, boxpath)
#TODO:This test will be deprecated when task.data.input.sandbox property is removed
for task in workload.taskIterator():
for t in task.nodeIterator():
t = WMTask.WMTaskHelper(t)
self.assertEqual(t.data.input.sandbox, boxpath)
pickleHandle.close()
pickledWorkload.section_("test_section")
self.assertNotEqual(workload.data, pickledWorkload)
shutil.rmtree(extractDir)
shutil.rmtree(tempdir)
def testPileupFetcherOnRelValMC(self):
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "pileupfetcher_t"
# in this test, try not to define generation task datatier (first output module
# should be automatically picked up)
#defaultArguments["GenDataTier"] = "GEN-SIM-RAW"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments[
"RecoConfigCacheID"] = self.injectReconstructionConfig()
defaultArguments["AlcaRecoConfigCacheID"] = self.injectAlcaRecoConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {
"cosmics": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"minbias": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % ("/tmp", testWorkload.name())
self.dirsToErase.append(pathBase)
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(defaultArguments, task, taskPath)
def createTask(self, fileURL):
"""
_createTask_
Create a test task that includes the
fileURL
"""
task = WMTask.makeWMTask("testTask")
task.makeStep("step1")
task.makeStep("step2")
for t in task.steps().nodeIterator():
t = WMStep.WMStepHelper(t)
os.mkdir('%s/%s' % (self.testDir, t.name()))
t.data.sandbox.section_('file0')
t.data.sandbox.file0.src = fileURL
return task
def createTask(self, fileURL):
"""
_createTask_
Create a test task that includes the
fileURL
"""
task = WMTask.makeWMTask("testTask")
task.makeStep("step1")
task.makeStep("step2")
for t in task.steps().nodeIterator():
t = WMStep.WMStepHelper(t)
os.mkdir('%s/%s' % (self.testDir, t.name()))
t.data.sandbox.section_('file0')
t.data.sandbox.file0.src = fileURL
return task
def createTask(self, configCache):
"""
_createTask_
Create a test task that includes the
fileURL
"""
task = WMTask.makeWMTask("testTask")
task.makeStep("step1")
task.makeStep("step2")
for t in task.steps().nodeIterator():
t = WMStep.WMStepHelper(t)
os.mkdir(os.path.join(self.testDir, t.name()))
t.setStepType("CMSSW")
t.data.application.section_('command')
t.data.application.configuration.configCacheUrl = configCache.dburl
t.data.application.configuration.cacheName = configCache.dbname
t.data.application.configuration.configId = configCache.getCouchID()
t.data.application.command.psetTweak = 'tweak'
t.data.application.command.configuration = 'configCache.file'
return task
def makeSandbox(self, buildItHere, workload):
"""
__makeSandbox__
MakeSandbox creates and archives a sandbox in buildItHere,
returning the path to the archive and putting it in the
task
"""
workloadName = workload.name()
# Create path to sandbox
pileupCachePath = "%s/pileupCache" % buildItHere
path = "%s/%s/WMSandbox" % (buildItHere, workloadName)
workloadFile = os.path.join(path, "WMWorkload.pkl")
archivePath = os.path.join(
buildItHere,
"%s/%s-Sandbox.tar.bz2" % (workloadName, workloadName))
# check if already built
if os.path.exists(archivePath) and os.path.exists(workloadFile):
workload.setSpecUrl(workloadFile) # point to sandbox spec
return archivePath
if os.path.exists(path):
shutil.rmtree(path)
# //
# // Set up Fetcher plugins, use default list for maintaining
#// compatibility
commonFetchers = ["CMSSWFetcher", "URLFetcher", "PileupFetcher"]
# generate the real path and make it
self._makePathonPackage(path)
# Add sandbox path to workload
workload.setSandbox(archivePath)
userSandboxes = []
for topLevelTask in workload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
task = WMTask.WMTaskHelper(taskNode)
fetcherNames = commonFetchers[:]
taskFetchers = getattr(task.data, "fetchers", [])
fetcherNames.extend(taskFetchers)
fetcherInstances = map(getFetcher, fetcherNames)
taskPath = "%s/%s" % (path, task.name())
self._makePathonPackage(taskPath)
#TODO sandbox is property of workload now instead of task
#but someother places uses as task propery (i.e. TaskQueue)
# so backward compatability save as task attribute as well.
setattr(task.data.input, 'sandbox', archivePath)
for s in task.steps().nodeIterator():
s = WMStep.WMStepHelper(s)
stepPath = "%s/%s" % (taskPath, s.name())
self._makePathonPackage(stepPath)
userSandboxes.extend(s.getUserSandboxes())
# //
# // Execute the fetcher plugins
#//
for fetcher in fetcherInstances:
#TODO: when cache directory is set as path, cache is maintained by workflow.
# In that case, cache will be deleted when workflow is done,
# but if different workflow can share the same cache.
# You can set the cache direcoty somewhere else, but need to have cache refresh (delete) policy
fetcher.setCacheDirectory(pileupCachePath)
fetcher.setWorkingDirectory(taskPath)
fetcher(task)
# pickle up the workload for storage in the sandbox
workload.setSpecUrl(workloadFile)
workload.save(workloadFile)
# now, tar everything up and put it somewhere special
#(archiveHandle,archivePath) = tempfile.mkstemp('.tar.bz2','sbox',
# buildItHere)
pythonHandle = open(archivePath, 'w+b')
archive = tarfile.open(None, 'w:bz2', pythonHandle)
archive.add("%s/%s/" % (buildItHere, workloadName),
'/',
exclude=tarballExclusion)
if (self.packageWMCore):
# package up the WMCore distribution in a zip file
# fixes #2943
wmcorePath = os.path.realpath(
os.path.join(os.path.dirname(__file__), '..'))
(zipHandle, zipPath) = tempfile.mkstemp()
os.close(zipHandle)
zipFile = zipfile.ZipFile(zipPath,
mode='w',
compression=zipfile.ZIP_DEFLATED)
for (root, dirnames, filenames) in os.walk(wmcorePath):
for filename in filenames:
if not tarballExclusion(filename):
zipFile.write(
filename=os.path.join(root, filename),
# the name in the archive is the path relative to WMCore/
arcname=os.path.join(
root, filename)[len(wmcorePath) -
len('WMCore/') + 1:])
# Add a dummy module for zipimport testing
(handle, dummyModulePath) = tempfile.mkstemp()
os.write(handle, "#!/usr/bin/env python\n")
os.write(
handle,
"# This file should only appear in zipimports, used for testing\n"
)
os.close(handle)
zipFile.write(filename=dummyModulePath,
arcname='WMCore/ZipImportTestModule.py')
# Add the wmcore zipball to the sandbox
zipFile.close()
archive.add(zipPath, '/WMCore.zip')
os.unlink(zipPath)
os.unlink(dummyModulePath)
psetTweaksPath = PSetTweaks.__path__[0]
archive.add(psetTweaksPath,
'/PSetTweaks',
exclude=tarballExclusion)
utilsPath = Utils.__path__[0]
archive.add(utilsPath, '/Utils', exclude=tarballExclusion)
for sb in userSandboxes:
splitResult = urlparse.urlsplit(sb)
if not splitResult[0]:
archive.add(sb, os.path.basename(sb))
archive.close()
pythonHandle.close()
return archivePath
def testMakeSandbox(self):
creator = SandboxCreator.SandboxCreator()
workload = TestWorkloads.twoTaskTree()
tempdir = tempfile.mkdtemp()
# test that the existing path is deleted else it will crash as in issue #5130
os.makedirs('%s/%s/WMSandbox' % (tempdir, workload.name()))
boxpath = creator.makeSandbox(tempdir, workload)
# extract our sandbox to test it
extractDir = tempfile.mkdtemp()
tarHandle = tarfile.open(boxpath, 'r:bz2')
tarHandle.extractall(extractDir)
self.fileExistsTest(extractDir + "/WMSandbox")
self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl")
self.fileExistsTest(extractDir + "/WMSandbox/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1")
self.fileExistsTest(extractDir +
"/WMSandbox/FirstTask/cmsRun1/__init__.py")
self.fileExistsTest(extractDir +
"/WMSandbox/FirstTask/stageOut1/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2")
self.fileExistsTest(extractDir +
"/WMSandbox/SecondTask/cmsRun2/__init__.py")
self.fileExistsTest(extractDir +
"/WMSandbox/SecondTask/stageOut2/__init__.py")
# make sure the sandbox is there
self.fileExistsTest(extractDir + '/WMCore.zip')
# now test that zipimport works
# This gets replaced in setup/teardown
sys.path.insert(0, os.path.join(extractDir, 'WMCore.zip'))
os.system('ls -lah %s' % extractDir)
# Gotta remove this since python caches subpackage folders in package.__path__
del sys.modules['WMCore']
if 'WMCore.ZipImportTestModule' in sys.modules:
del sys.modules['WMCore.ZipImportTestModule']
import WMCore.ZipImportTestModule
sys.modules = copy.copy(self.modulesBackup)
self.assertTrue('WMCore.zip' in WMCore.ZipImportTestModule.__file__)
# make sure the pickled file is the same
pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl")
pickledWorkload = pickle.load(pickleHandle)
self.assertEqual(workload.data, pickledWorkload)
self.assertEqual(pickledWorkload.sandbox, boxpath)
#TODO:This test will be deprecated when task.data.input.sandbox property is removed
for task in workload.taskIterator():
for t in task.nodeIterator():
t = WMTask.WMTaskHelper(t)
self.assertEqual(t.data.input.sandbox, boxpath)
pickleHandle.close()
pickledWorkload.section_("test_section")
self.assertNotEqual(workload.data, pickledWorkload)
shutil.rmtree(extractDir)
shutil.rmtree(tempdir)