def _queryPileUpConfigFile(self, defaultArguments, task, taskPath):
"""
Query and compare contents of the the pileup JSON
configuration files. Iterate over tasks's steps as
it happens in the PileupFetcher.
"""
for step in task.steps().nodeIterator():
helper = WMStep.WMStepHelper(step)
# returns e.g. instance of CMSSWHelper
if hasattr(helper.data, "pileup"):
decoder = JSONDecoder()
stepPath = "%s/%s" % (taskPath, helper.name())
pileupConfig = "%s/%s" % (stepPath, "pileupconf.json")
try:
f = open(pileupConfig, 'r')
json = f.read()
pileupDict = decoder.decode(json)
f.close()
except IOError:
m = "Could not read pileup JSON configuration file: '%s'" % pileupConfig
self.fail(m)
self._queryAndCompareWithDBS(pileupDict, defaultArguments,
helper.data.dbsUrl)
def __call__(self, wmTask):
"""
Trip through steps, find CMSSW steps, pull in config files,
PSet Tweaks etc
"""
for t in wmTask.steps().nodeIterator():
t = WMStep.WMStepHelper(t)
stepPath = "%s/%s" % (self.workingDirectory(), t.name())
# the CMSSW has a special case with its ConfigCache argument
if not t.stepType() in ("CMSSW", "MulticoreCMSSW"): continue
if getattr(t.data.application.configuration, 'configCacheUrl', None) != None:
# main config file
fileTarget = "%s/%s" % (
stepPath,
t.data.application.command.configuration)
#urllib.urlretrieve(
# t.data.application.configuration.retrieveConfigUrl,
# fileTarget)
# PSet Tweak
cacheUrl = t.data.application.configuration.configCacheUrl
cacheDb = t.data.application.configuration.cacheName
configId = t.data.application.configuration.configId
tweakTarget = t.data.application.command.psetTweak
configCache = ConfigCache(cacheUrl, cacheDb)
configCache.loadByID(configId)
configCache.saveConfigToDisk(targetFile = fileTarget)
tweak = TweakAPI.makeTweakFromJSON(configCache.getPSetTweaks())
if tweak:
tweakFile = "%s/%s" % (stepPath, tweakTarget)
tweak.persist(tweakFile, "json")
def __call__(self, wmTask):
"""
Trip through steps, pull in files specified in the sandbox section
PSet Tweaks etc
"""
for t in wmTask.steps().nodeIterator():
t = WMStep.WMStepHelper(t)
stepPath = "%s/%s" % (self.workingDirectory(), t.name())
for fileInfo in t.data.sandbox:
# fileInfo.src is the source file
# fileInfo.injob is where we stuck it
match = re.search("^.*/(.*?)$", fileInfo.src)
if (match):
fileSuffix = match.group(1)
else:
fileSuffix = "sandboxFile.dat"
fileTarget = "%s/%s" % (stepPath, fileSuffix)
# Now build a Request object, make a request, and write
# the output to a file:
try:
request = Requests.Requests(fileInfo.src)
content = request.get('')[0]
f = open(fileTarget, 'w')
f.write(content)
f.close()
except IOError, ex:
msg = "Could not write to fileTarget %s\n" % fileTarget
msg += str(ex)
logging.error(msg)
logging.debug("FileInfo: %s" % fileInfo)
raise
fileInfo.injob = fileTarget
def __call__(self, wmTask):
"""
Method is called when WorkQueue creates the sandbox for a job.
Need to look at the pileup configuration in the spec and query dbs to
determine the lfns for the files in the datasets and what sites they're
located at (WQ creates the job sandbox).
wmTask is instance of WMTask.WMTaskHelper
"""
siteWhitelist = wmTask.siteWhitelist()
# check whether we need to pretend PU data location
if wmTask.inputLocationFlag():
fakeSites = wmTask.siteWhitelist()
else:
fakeSites = []
for step in wmTask.steps().nodeIterator():
helper = WMStep.WMStepHelper(step)
# returns e.g. instance of CMSSWHelper
# doesn't seem to be necessary ... strangely (some inheritance involved?)
# typeHelper = helper.getTypeHelper()
if hasattr(helper.data, "pileup"):
self._createPileupConfigFile(helper, fakeSites)
def testBuild(self):
""" build a directory and verify it exists"""
mytemplate = StepFactory.getStepTemplate("CMSSW")
mystep = WMStep.makeWMStep("DummyStagingStep")
mytemplate(mystep.data)
self.testBuilder(mystep.data, "testTask", self.tempDir)
self.assertTrue(os.path.exists(self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py" % self.tempDir))
def testBuild(self):
''' build a directory and verify it exists'''
mytemplate = StepFactory.getStepTemplate("StageOut")
mystep = WMStep.makeWMStep("DummyStagingStep")
mytemplate(mystep.data)
self.testBuilder(mystep.data, "testTask", self.tempDir)
self.assertTrue(os.path.exists(self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py"
% self.tempDir))
def testCustomBuild(self):
""" add in a custom directory and verify it gets created"""
mytemplate = StepFactory.getStepTemplate("CMSSW")
mystep = WMStep.makeWMStep("DummyStagingStep")
mytemplate(mystep.data)
helper = TemplateNS.Template.CoreHelper(mystep.data)
helper.addDirectory("testdirectory1")
helper.addDirectory("testdirectory2/testsubdir")
self.testBuilder(mystep.data, "testTask", self.tempDir)
self.assertTrue(os.path.exists(self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py" % self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/testdirectory1" % self.tempDir))
self.assertTrue(os.path.exists("%s/%s/testdirectory2/testsubdir" % (self.tempDir, "DummyStagingStep")))
def testCustomBuild(self):
''' add in a custom directory and verify it gets created'''
mytemplate = StepFactory.getStepTemplate("StageOut")
mystep = WMStep.makeWMStep("DummyStagingStep")
mytemplate(mystep.data)
helper = TemplateNS.Template.CoreHelper(mystep.data)
helper.addDirectory( 'testdirectory1' )
helper.addDirectory( 'testdirectory2/testsubdir' )
self.testBuilder(mystep.data, "testTask", self.tempDir)
self.assertTrue(os.path.exists(self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/__init__.py"
% self.tempDir))
self.assertTrue(os.path.exists("%s/DummyStagingStep/testdirectory1"
% self.tempDir))
self.assertTrue(os.path.exists("%s/%s/testdirectory2/testsubdir"
% (self.tempDir, 'DummyStagingStep')))
def createTask(self, fileURL):
"""
_createTask_
Create a test task that includes the
fileURL
"""
task = WMTask.makeWMTask("testTask")
task.makeStep("step1")
task.makeStep("step2")
for t in task.steps().nodeIterator():
t = WMStep.WMStepHelper(t)
os.mkdir('%s/%s' % (self.testDir, t.name()))
t.data.sandbox.section_('file0')
t.data.sandbox.file0.src = fileURL
return task
def createTask(self, configCache):
"""
_createTask_
Create a test task that includes the
fileURL
"""
task = WMTask.makeWMTask("testTask")
task.makeStep("step1")
task.makeStep("step2")
for t in task.steps().nodeIterator():
t = WMStep.WMStepHelper(t)
os.mkdir(os.path.join(self.testDir, t.name()))
t.setStepType("CMSSW")
t.data.application.section_('command')
t.data.application.configuration.configCacheUrl = configCache.dburl
t.data.application.configuration.cacheName = configCache.dbname
t.data.application.configuration.configId = configCache.getCouchID()
t.data.application.command.psetTweak = 'tweak'
t.data.application.command.configuration = 'configCache.file'
return task
def makeSandbox(self, buildItHere, workload):
"""
__makeSandbox__
MakeSandbox creates and archives a sandbox in buildItHere,
returning the path to the archive and putting it in the
task
"""
workloadName = workload.name()
# Create path to sandbox
pileupCachePath = "%s/pileupCache" % buildItHere
path = "%s/%s/WMSandbox" % (buildItHere, workloadName)
workloadFile = os.path.join(path, "WMWorkload.pkl")
archivePath = os.path.join(
buildItHere,
"%s/%s-Sandbox.tar.bz2" % (workloadName, workloadName))
# check if already built
if os.path.exists(archivePath) and os.path.exists(workloadFile):
workload.setSpecUrl(workloadFile) # point to sandbox spec
return archivePath
if os.path.exists(path):
shutil.rmtree(path)
# //
# // Set up Fetcher plugins, use default list for maintaining
#// compatibility
commonFetchers = ["CMSSWFetcher", "URLFetcher", "PileupFetcher"]
# generate the real path and make it
self._makePathonPackage(path)
# Add sandbox path to workload
workload.setSandbox(archivePath)
userSandboxes = []
for topLevelTask in workload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
task = WMTask.WMTaskHelper(taskNode)
fetcherNames = commonFetchers[:]
taskFetchers = getattr(task.data, "fetchers", [])
fetcherNames.extend(taskFetchers)
fetcherInstances = map(getFetcher, fetcherNames)
taskPath = "%s/%s" % (path, task.name())
self._makePathonPackage(taskPath)
#TODO sandbox is property of workload now instead of task
#but someother places uses as task propery (i.e. TaskQueue)
# so backward compatability save as task attribute as well.
setattr(task.data.input, 'sandbox', archivePath)
for s in task.steps().nodeIterator():
s = WMStep.WMStepHelper(s)
stepPath = "%s/%s" % (taskPath, s.name())
self._makePathonPackage(stepPath)
userSandboxes.extend(s.getUserSandboxes())
# //
# // Execute the fetcher plugins
#//
for fetcher in fetcherInstances:
#TODO: when cache directory is set as path, cache is maintained by workflow.
# In that case, cache will be deleted when workflow is done,
# but if different workflow can share the same cache.
# You can set the cache direcoty somewhere else, but need to have cache refresh (delete) policy
fetcher.setCacheDirectory(pileupCachePath)
fetcher.setWorkingDirectory(taskPath)
fetcher(task)
# pickle up the workload for storage in the sandbox
workload.setSpecUrl(workloadFile)
workload.save(workloadFile)
# now, tar everything up and put it somewhere special
#(archiveHandle,archivePath) = tempfile.mkstemp('.tar.bz2','sbox',
# buildItHere)
pythonHandle = open(archivePath, 'w+b')
archive = tarfile.open(None, 'w:bz2', pythonHandle)
archive.add("%s/%s/" % (buildItHere, workloadName),
'/',
exclude=tarballExclusion)
if (self.packageWMCore):
# package up the WMCore distribution in a zip file
# fixes #2943
wmcorePath = os.path.realpath(
os.path.join(os.path.dirname(__file__), '..'))
(zipHandle, zipPath) = tempfile.mkstemp()
os.close(zipHandle)
zipFile = zipfile.ZipFile(zipPath,
mode='w',
compression=zipfile.ZIP_DEFLATED)
for (root, dirnames, filenames) in os.walk(wmcorePath):
for filename in filenames:
if not tarballExclusion(filename):
zipFile.write(
filename=os.path.join(root, filename),
# the name in the archive is the path relative to WMCore/
arcname=os.path.join(
root, filename)[len(wmcorePath) -
len('WMCore/') + 1:])
# Add a dummy module for zipimport testing
(handle, dummyModulePath) = tempfile.mkstemp()
os.write(handle, "#!/usr/bin/env python\n")
os.write(
handle,
"# This file should only appear in zipimports, used for testing\n"
)
os.close(handle)
zipFile.write(filename=dummyModulePath,
arcname='WMCore/ZipImportTestModule.py')
# Add the wmcore zipball to the sandbox
zipFile.close()
archive.add(zipPath, '/WMCore.zip')
os.unlink(zipPath)
os.unlink(dummyModulePath)
psetTweaksPath = PSetTweaks.__path__[0]
archive.add(psetTweaksPath,
'/PSetTweaks',
exclude=tarballExclusion)
utilsPath = Utils.__path__[0]
archive.add(utilsPath, '/Utils', exclude=tarballExclusion)
for sb in userSandboxes:
splitResult = urlparse.urlsplit(sb)
if not splitResult[0]:
archive.add(sb, os.path.basename(sb))
archive.close()
pythonHandle.close()
return archivePath
