def _queryPileUpConfigFile(self, defaultArguments, task, taskPath): """ Query and compare contents of the the pileup JSON configuration files. Iterate over tasks's steps as it happens in the PileupFetcher. """ for step in task.steps().nodeIterator(): helper = WMStep.WMStepHelper(step) # returns e.g. instance of CMSSWHelper if hasattr(helper.data, "pileup"): decoder = JSONDecoder() stepPath = "%s/%s" % (taskPath, helper.name()) pileupConfig = "%s/%s" % (stepPath, "pileupconf.json") try: f = open(pileupConfig, 'r') json = f.read() pileupDict = decoder.decode(json) f.close() except IOError: m = "Could not read pileup JSON configuration file: '%s'" % pileupConfig self.fail(m) self._queryAndCompareWithDBS(pileupDict, defaultArguments, helper.data.dbsUrl)
def __call__(self, wmTask): """ Trip through steps, find CMSSW steps, pull in config files, PSet Tweaks etc """ for t in wmTask.steps().nodeIterator(): t = WMStep.WMStepHelper(t) stepPath = "%s/%s" % (self.workingDirectory(), t.name()) # the CMSSW has a special case with its ConfigCache argument if not t.stepType() in ("CMSSW", "MulticoreCMSSW"): continue if getattr(t.data.application.configuration, 'configCacheUrl', None) != None: # main config file fileTarget = "%s/%s" % ( stepPath, t.data.application.command.configuration) #urllib.urlretrieve( # t.data.application.configuration.retrieveConfigUrl, # fileTarget) # PSet Tweak cacheUrl = t.data.application.configuration.configCacheUrl cacheDb = t.data.application.configuration.cacheName configId = t.data.application.configuration.configId tweakTarget = t.data.application.command.psetTweak configCache = ConfigCache(cacheUrl, cacheDb) configCache.loadByID(configId) configCache.saveConfigToDisk(targetFile = fileTarget) tweak = TweakAPI.makeTweakFromJSON(configCache.getPSetTweaks()) if tweak: tweakFile = "%s/%s" % (stepPath, tweakTarget) tweak.persist(tweakFile, "json")
def __call__(self, wmTask): """ Trip through steps, pull in files specified in the sandbox section PSet Tweaks etc """ for t in wmTask.steps().nodeIterator(): t = WMStep.WMStepHelper(t) stepPath = "%s/%s" % (self.workingDirectory(), t.name()) for fileInfo in t.data.sandbox: # fileInfo.src is the source file # fileInfo.injob is where we stuck it match = re.search("^.*/(.*?)$", fileInfo.src) if (match): fileSuffix = match.group(1) else: fileSuffix = "sandboxFile.dat" fileTarget = "%s/%s" % (stepPath, fileSuffix) # Now build a Request object, make a request, and write # the output to a file: try: request = Requests.Requests(fileInfo.src) content = request.get('')[0] f = open(fileTarget, 'w') f.write(content) f.close() except IOError, ex: msg = "Could not write to fileTarget %s\n" % fileTarget msg += str(ex) logging.error(msg) logging.debug("FileInfo: %s" % fileInfo) raise fileInfo.injob = fileTarget
def __call__(self, wmTask): """ Method is called when WorkQueue creates the sandbox for a job. Need to look at the pileup configuration in the spec and query dbs to determine the lfns for the files in the datasets and what sites they're located at (WQ creates the job sandbox). wmTask is instance of WMTask.WMTaskHelper """ siteWhitelist = wmTask.siteWhitelist() # check whether we need to pretend PU data location if wmTask.inputLocationFlag(): fakeSites = wmTask.siteWhitelist() else: fakeSites = [] for step in wmTask.steps().nodeIterator(): helper = WMStep.WMStepHelper(step) # returns e.g. instance of CMSSWHelper # doesn't seem to be necessary ... strangely (some inheritance involved?) # typeHelper = helper.getTypeHelper() if hasattr(helper.data, "pileup"): self._createPileupConfigFile(helper, fakeSites)
def createTask(self, fileURL): """ _createTask_ Create a test task that includes the fileURL """ task = WMTask.makeWMTask("testTask") task.makeStep("step1") task.makeStep("step2") for t in task.steps().nodeIterator(): t = WMStep.WMStepHelper(t) os.mkdir('%s/%s' % (self.testDir, t.name())) t.data.sandbox.section_('file0') t.data.sandbox.file0.src = fileURL return task
def createTask(self, configCache): """ _createTask_ Create a test task that includes the fileURL """ task = WMTask.makeWMTask("testTask") task.makeStep("step1") task.makeStep("step2") for t in task.steps().nodeIterator(): t = WMStep.WMStepHelper(t) os.mkdir(os.path.join(self.testDir, t.name())) t.setStepType("CMSSW") t.data.application.section_('command') t.data.application.configuration.configCacheUrl = configCache.dburl t.data.application.configuration.cacheName = configCache.dbname t.data.application.configuration.configId = configCache.getCouchID() t.data.application.command.psetTweak = 'tweak' t.data.application.command.configuration = 'configCache.file' return task
def makeSandbox(self, buildItHere, workload): """ __makeSandbox__ MakeSandbox creates and archives a sandbox in buildItHere, returning the path to the archive and putting it in the task """ workloadName = workload.name() # Create path to sandbox pileupCachePath = "%s/pileupCache" % buildItHere path = "%s/%s/WMSandbox" % (buildItHere, workloadName) workloadFile = os.path.join(path, "WMWorkload.pkl") archivePath = os.path.join( buildItHere, "%s/%s-Sandbox.tar.bz2" % (workloadName, workloadName)) # check if already built if os.path.exists(archivePath) and os.path.exists(workloadFile): workload.setSpecUrl(workloadFile) # point to sandbox spec return archivePath if os.path.exists(path): shutil.rmtree(path) # // # // Set up Fetcher plugins, use default list for maintaining #// compatibility commonFetchers = ["CMSSWFetcher", "URLFetcher", "PileupFetcher"] # generate the real path and make it self._makePathonPackage(path) # Add sandbox path to workload workload.setSandbox(archivePath) userSandboxes = [] for topLevelTask in workload.taskIterator(): for taskNode in topLevelTask.nodeIterator(): task = WMTask.WMTaskHelper(taskNode) fetcherNames = commonFetchers[:] taskFetchers = getattr(task.data, "fetchers", []) fetcherNames.extend(taskFetchers) fetcherInstances = map(getFetcher, fetcherNames) taskPath = "%s/%s" % (path, task.name()) self._makePathonPackage(taskPath) #TODO sandbox is property of workload now instead of task #but someother places uses as task propery (i.e. TaskQueue) # so backward compatability save as task attribute as well. setattr(task.data.input, 'sandbox', archivePath) for s in task.steps().nodeIterator(): s = WMStep.WMStepHelper(s) stepPath = "%s/%s" % (taskPath, s.name()) self._makePathonPackage(stepPath) userSandboxes.extend(s.getUserSandboxes()) # // # // Execute the fetcher plugins #// for fetcher in fetcherInstances: #TODO: when cache directory is set as path, cache is maintained by workflow. # In that case, cache will be deleted when workflow is done, # but if different workflow can share the same cache. # You can set the cache direcoty somewhere else, but need to have cache refresh (delete) policy fetcher.setCacheDirectory(pileupCachePath) fetcher.setWorkingDirectory(taskPath) fetcher(task) # pickle up the workload for storage in the sandbox workload.setSpecUrl(workloadFile) workload.save(workloadFile) # now, tar everything up and put it somewhere special #(archiveHandle,archivePath) = tempfile.mkstemp('.tar.bz2','sbox', # buildItHere) pythonHandle = open(archivePath, 'w+b') archive = tarfile.open(None, 'w:bz2', pythonHandle) archive.add("%s/%s/" % (buildItHere, workloadName), '/', exclude=tarballExclusion) if (self.packageWMCore): # package up the WMCore distribution in a zip file # fixes #2943 wmcorePath = os.path.realpath( os.path.join(os.path.dirname(__file__), '..')) (zipHandle, zipPath) = tempfile.mkstemp() os.close(zipHandle) zipFile = zipfile.ZipFile(zipPath, mode='w', compression=zipfile.ZIP_DEFLATED) for (root, dirnames, filenames) in os.walk(wmcorePath): for filename in filenames: if not tarballExclusion(filename): zipFile.write( filename=os.path.join(root, filename), # the name in the archive is the path relative to WMCore/ arcname=os.path.join( root, filename)[len(wmcorePath) - len('WMCore/') + 1:]) # Add a dummy module for zipimport testing (handle, dummyModulePath) = tempfile.mkstemp() os.write(handle, "#!/usr/bin/env python\n") os.write( handle, "# This file should only appear in zipimports, used for testing\n" ) os.close(handle) zipFile.write(filename=dummyModulePath, arcname='WMCore/ZipImportTestModule.py') # Add the wmcore zipball to the sandbox zipFile.close() archive.add(zipPath, '/WMCore.zip') os.unlink(zipPath) os.unlink(dummyModulePath) psetTweaksPath = PSetTweaks.__path__[0] archive.add(psetTweaksPath, '/PSetTweaks', exclude=tarballExclusion) utilsPath = Utils.__path__[0] archive.add(utilsPath, '/Utils', exclude=tarballExclusion) for sb in userSandboxes: splitResult = urlparse.urlsplit(sb) if not splitResult[0]: archive.add(sb, os.path.basename(sb)) archive.close() pythonHandle.close() return archivePath