def __call__(self, wmTask): """ Trip through steps, find CMSSW steps, pull in config files, PSet Tweaks etc """ for t in wmTask.steps().nodeIterator(): t = WMStep.WMStepHelper(t) stepPath = "%s/%s" % (self.workingDirectory(), t.name()) # the CMSSW has a special case with its ConfigCache argument if not t.stepType() in ("CMSSW", "MulticoreCMSSW"): continue if getattr(t.data.application.configuration, 'configCacheUrl', None) != None: # main config file fileTarget = "%s/%s" % ( stepPath, t.data.application.command.configuration) #urllib.urlretrieve( # t.data.application.configuration.retrieveConfigUrl, # fileTarget) # PSet Tweak cacheUrl = t.data.application.configuration.configCacheUrl cacheDb = t.data.application.configuration.cacheName configId = t.data.application.configuration.configId tweakTarget = t.data.application.command.psetTweak configCache = ConfigCache(cacheUrl, cacheDb) configCache.loadByID(configId) configCache.saveConfigToDisk(targetFile = fileTarget) tweak = TweakAPI.makeTweakFromJSON(configCache.getPSetTweaks()) if tweak: tweakFile = "%s/%s" % (stepPath, tweakTarget) tweak.persist(tweakFile, "json")
def testB_addingConfigsAndTweaks(self): """ _addingConfigsAndTweaks_ Test adding config files and tweak files """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." attach = "Hello, I am an attachment" configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache.createUserGroup(groupname = "testGroup", username = '******') configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.attachments['attach1'] = attach psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt") configCache.addConfig(newConfig = psetPath, psetHash = None) configCache.setLabel("sample-label") configCache.setDescription("describe this config here") configCache.save() configString1 = configCache.getConfig() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test', id = configCache.getCouchID(), rev = configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) configString2 = configCache2.getConfig() self.assertEqual(configString1, configString2) self.assertEqual(configCache2.attachments.get('attach1', None), attach) configCache.delete() return
def testB_addingConfigsAndTweaks(self): """ _addingConfigsAndTweaks_ Test adding config files and tweak files """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." attach = "Hello, I am an attachment" configCache = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test') configCache.createUserGroup(groupname="testGroup", username='******') configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.attachments['attach1'] = attach psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt") configCache.addConfig(newConfig=psetPath, psetHash=None) configCache.setLabel("sample-label") configCache.setDescription("describe this config here") configCache.save() configString1 = configCache.getConfig() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test', id=configCache.getCouchID(), rev=configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) configString2 = configCache2.getConfig() self.assertEqual(configString1, configString2) self.assertEqual(configCache2.attachments.get('attach1', None), attach) configCache.delete() return
def createAlgoFromInfo(info): """ Create an Algo object from basic information """ algo = {'ApplicationName': info.get('ApplicationName'), 'ApplicationFamily': info.get('ApplicationFamily'), 'ApplicationVersion': info.get('ApplicationVersion'), 'PSetHash': info.get('PSetHash'), 'PSetContent': None, 'InDBS': info.get('AlgoInDBS', None) } configString = info.get('PSetContent') if configString: split = configString.split(';;') cacheURL = split[0] cacheDB = split[1] configID = split[2] try: configCache = ConfigCache(cacheURL, cacheDB) configCache.loadByID(configID) algo['PSetContent'] = configCache.getConfig() except Exception, ex: msg = "Exception in getting configCache from DB\n" msg += "Ignoring this exception and continuing without config.\n" msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) logging.debug("URL: %s, DB: %s, ID: %s" % (cacheURL, cacheDB, configID))
def determineOutputModules(self, scenarioFunc = None, scenarioArgs = None, configDoc = None, couchURL = None, couchDBName = None): """ _determineOutputModules_ Determine the output module names and associated metadata for the given config. """ outputModules = {} if configDoc != None and configDoc != "": configCache = ConfigCache(couchURL, couchDBName) configCache.loadByID(configDoc) outputModules = configCache.getOutputModuleInfo() else: if scenarioFunc in [ "promptReco", "expressProcessing" ]: for output in scenarioArgs.get('outputs',[]): dataTier = output['dataTier'] moduleLabel = output['moduleLabel'] filterName = output.get('filterName', None) outputModules[moduleLabel] = {'dataTier' : dataTier, 'filterName' : filterName} elif scenarioFunc == "alcaSkim": for alcaSkim in scenarioArgs.get('skims',[]): dataTier = "ALCARECO" moduleLabel = "ALCARECOStream%s" % alcaSkim filterName = alcaSkim outputModules[moduleLabel] = {'dataTier' : dataTier, 'filterName' : filterName} return outputModules
def showOriginalConfig(self, docId): """ Makes a link to the original text of the config """ configCache = ConfigCache(self.couchUrl, self.configDBName) configCache.loadByID(docId) configString = configCache.getConfig() if configString == None: return "Cannot find document " + str(docId) + " in Couch DB" return '<pre>' + configString + '</pre>'
def showOriginalConfig(self, docId): """ Makes a link to the original text of the config """ configCache = ConfigCache(self.couchUrl, self.configDBName) configCache.loadByID(docId) configString = configCache.getConfig() if configString == None: return "Cannot find document " + str(docId) + " in Couch DB" return '<pre>' + configString + '</pre>'
def determineOutputModules(self, scenarioFunc=None, scenarioArgs=None, configDoc=None, couchURL=None, couchDBName=None, configCacheUrl=None): """ _determineOutputModules_ Determine the output module names and associated metadata for the given config. """ # set default scenarioArgs to empty dictionary if it is None. scenarioArgs = scenarioArgs or {} outputModules = {} if configDoc != None and configDoc != "": url = configCacheUrl or couchURL configCache = ConfigCache(url, couchDBName) configCache.loadByID(configDoc) outputModules = configCache.getOutputModuleInfo() else: if 'outputs' in scenarioArgs and scenarioFunc in [ "promptReco", "expressProcessing", "repack" ]: for output in scenarioArgs.get('outputs', []): moduleLabel = output['moduleLabel'] outputModules[moduleLabel] = { 'dataTier': output['dataTier'] } if output.has_key('primaryDataset'): outputModules[moduleLabel]['primaryDataset'] = output[ 'primaryDataset'] if output.has_key('filterName'): outputModules[moduleLabel]['filterName'] = output[ 'filterName'] elif 'writeTiers' in scenarioArgs and scenarioFunc == "promptReco": for dataTier in scenarioArgs.get('writeTiers'): moduleLabel = "%soutput" % dataTier outputModules[moduleLabel] = {'dataTier': dataTier} elif scenarioFunc == "alcaSkim": for alcaSkim in scenarioArgs.get('skims', []): moduleLabel = "ALCARECOStream%s" % alcaSkim if alcaSkim.startswith("PromptCalibProd"): dataTier = "ALCAPROMPT" else: dataTier = "ALCARECO" outputModules[moduleLabel] = { 'dataTier': dataTier, 'primaryDataset': scenarioArgs.get('primaryDataset'), 'filterName': alcaSkim } return outputModules
def validateConfigCacheExists(self, configID, couchURL, couchDBName, getOutputModules=False): """ _validateConfigCacheExists_ If we have a configCache, we should probably try and load it. """ if configID == '' or configID == ' ': self.raiseValidationException( msg="ConfigCacheID is invalid and cannot be loaded") configCache = ConfigCache(dbURL=couchURL, couchDBName=couchDBName, id=configID) try: configCache.loadByID(configID=configID) except ConfigCacheException: self.raiseValidationException( msg="Failure to load ConfigCache while validating workload") duplicateCheck = {} try: outputModuleInfo = configCache.getOutputModuleInfo() except Exception: # Something's gone wrong with trying to open the configCache msg = "Error in getting output modules from ConfigCache during workload validation. Check ConfigCache formatting!" self.raiseValidationException(msg=msg) for outputModule in outputModuleInfo.values(): dataTier = outputModule.get('dataTier', None) filterName = outputModule.get('filterName', None) if not dataTier: self.raiseValidationException( msg="No DataTier in output module.") # Add dataTier to duplicate dictionary if not dataTier in duplicateCheck.keys(): duplicateCheck[dataTier] = [] if filterName in duplicateCheck[dataTier]: # Then we've seen this combination before self.raiseValidationException( msg="Duplicate dataTier/filterName combination.") else: duplicateCheck[dataTier].append(filterName) if getOutputModules: return outputModuleInfo return
def determineOutputModules(self, scenarioFunc = None, scenarioArgs = None, configDoc = None, couchURL = None, couchDBName = None, configCacheUrl = None): """ _determineOutputModules_ Determine the output module names and associated metadata for the given config. """ # set default scenarioArgs to empty dictionary if it is None. scenarioArgs = scenarioArgs or {} outputModules = {} if configDoc != None and configDoc != "": url = configCacheUrl or couchURL if (url, couchDBName) in self.config_cache: configCache = self.config_cache[(url, couchDBName)] else: configCache = ConfigCache(url, couchDBName, True) self.config_cache[(url, couchDBName)] = configCache #TODO: need to change to DataCache #configCache.loadDocument(configDoc) configCache.loadByID(configDoc) outputModules = configCache.getOutputModuleInfo() else: if 'outputs' in scenarioArgs and scenarioFunc in [ "promptReco", "expressProcessing", "repack" ]: for output in scenarioArgs.get('outputs', []): moduleLabel = output['moduleLabel'] outputModules[moduleLabel] = { 'dataTier' : output['dataTier'] } if 'primaryDataset' in output: outputModules[moduleLabel]['primaryDataset'] = output['primaryDataset'] if 'filterName' in output: outputModules[moduleLabel]['filterName'] = output['filterName'] elif 'writeTiers' in scenarioArgs and scenarioFunc == "promptReco": for dataTier in scenarioArgs.get('writeTiers'): moduleLabel = "%soutput" % dataTier outputModules[moduleLabel] = { 'dataTier' : dataTier } elif scenarioFunc == "alcaSkim": for alcaSkim in scenarioArgs.get('skims',[]): moduleLabel = "ALCARECOStream%s" % alcaSkim if alcaSkim.startswith("PromptCalibProd"): dataTier = "ALCAPROMPT" else: dataTier = "ALCARECO" outputModules[moduleLabel] = { 'dataTier' : dataTier, 'primaryDataset' : scenarioArgs.get('primaryDataset'), 'filterName' : alcaSkim } return outputModules
def validateConfigCacheExists(self, configID, couchURL, couchDBName, getOutputModules = False): """ _validateConfigCacheExists_ If we have a configCache, we should probably try and load it. """ from WMCore.Cache.WMConfigCache import ConfigCache configCache = ConfigCache(dbURL = couchURL, couchDBName = couchDBName, id = configID) try: configCache.loadByID(configID = configID) except ConfigCacheException, ex: self.raiseValidationException(msg = "Failure to load ConfigCache while validating workload")
def createAlgoFromInfo(info): """ Create an Algo object from basic information """ algo = { 'ApplicationName': info['ApplicationName'], 'ApplicationFamily': info['ApplicationFamily'], 'ApplicationVersion': info['ApplicationVersion'], 'PSetHash': info['PSetHash'], 'PSetContent': None, 'InDBS': info['AlgoInDBS'] } configString = info.get('PSetContent') if configString: try: split = configString.split(';;') cacheURL = split[0] cacheDB = split[1] configID = split[2] except IndexError: msg = "configCache not properly formatted\n" msg += "configString\n: %s" % configString msg += "Not attempting to put configCache content in DBS for this algo" msg += "AlgoInfo: %s" % algo logging.error(msg) return algo if cacheURL == "None" or cacheDB == "None" or configID == "None": # No Config for this DB logging.debug("No configCache for this algo") return algo try: configCache = ConfigCache(cacheURL, cacheDB) configCache.loadByID(configID) algo['PSetContent'] = configCache.getConfig() except Exception as ex: msg = "Exception in getting configCache from DB\n" msg += "Ignoring this exception and continuing without config.\n" msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) logging.debug("URL: %s, DB: %s, ID: %s" % (cacheURL, cacheDB, configID)) return algo
def _getConfigCache(self, requestName, processMethod): try: request = Utilities.requestDetails(requestName) except Exception as ex: msg = "Cannot find request %s, check logs." % requestName logging.error("%s, reason: %s" % (msg, ex)) return msg url = request.get("ConfigCacheUrl", None) or self.couchUrl try: configCache = ConfigCache(url, self.configDBName) configDocId = request["ConfigCacheID"] configCache.loadByID(configDocId) except Exception as ex: msg = "Cannot find ConfigCache document %s on %s." % (configDocId, url) logging.error("%s, reason: %s" % (msg, ex)) return msg return getattr(configCache, processMethod)()
def validateConfigCacheExists(self, configID, couchURL, couchDBName, getOutputModules = False): """ _validateConfigCacheExists_ If we have a configCache, we should probably try and load it. """ if configID == '' or configID == ' ': self.raiseValidationException(msg = "ConfigCacheID is invalid and cannot be loaded") configCache = ConfigCache(dbURL = couchURL, couchDBName = couchDBName, id = configID) try: configCache.loadByID(configID = configID) except ConfigCacheException, ex: self.raiseValidationException(msg = "Failure to load ConfigCache while validating workload")
def createAlgoFromInfo(info): """ Create an Algo object from basic information """ algo = { "ApplicationName": info["ApplicationName"], "ApplicationFamily": info["ApplicationFamily"], "ApplicationVersion": info["ApplicationVersion"], "PSetHash": info["PSetHash"], "PSetContent": None, "InDBS": info["AlgoInDBS"], } configString = info.get("PSetContent") if configString: try: split = configString.split(";;") cacheURL = split[0] cacheDB = split[1] configID = split[2] except IndexError: msg = "configCache not properly formatted\n" msg += "configString\n: %s" % configString msg += "Not attempting to put configCache content in DBS for this algo" msg += "AlgoInfo: %s" % algo logging.error(msg) return algo if cacheURL == "None" or cacheDB == "None" or configID == "None": # No Config for this DB logging.debug("No configCache for this algo") return algo try: configCache = ConfigCache(cacheURL, cacheDB) configCache.loadByID(configID) algo["PSetContent"] = configCache.getConfig() except Exception as ex: msg = "Exception in getting configCache from DB\n" msg += "Ignoring this exception and continuing without config.\n" msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) logging.debug("URL: %s, DB: %s, ID: %s" % (cacheURL, cacheDB, configID)) return algo
def validateConfigCacheExists(self, configID, couchURL, couchDBName, getOutputModules = False): """ _validateConfigCacheExists_ If we have a configCache, we should probably try and load it. """ if configID == '' or configID == ' ': self.raiseValidationException(msg = "ConfigCacheID is invalid and cannot be loaded") configCache = ConfigCache(dbURL = couchURL, couchDBName = couchDBName, id = configID) try: configCache.loadByID(configID = configID) except ConfigCacheException: self.raiseValidationException(msg = "Failure to load ConfigCache while validating workload") duplicateCheck = {} try: outputModuleInfo = configCache.getOutputModuleInfo() except Exception: # Something's gone wrong with trying to open the configCache msg = "Error in getting output modules from ConfigCache during workload validation. Check ConfigCache formatting!" self.raiseValidationException(msg = msg) for outputModule in outputModuleInfo.values(): dataTier = outputModule.get('dataTier', None) filterName = outputModule.get('filterName', None) if not dataTier: self.raiseValidationException(msg = "No DataTier in output module.") # Add dataTier to duplicate dictionary if not dataTier in duplicateCheck.keys(): duplicateCheck[dataTier] = [] if filterName in duplicateCheck[dataTier]: # Then we've seen this combination before self.raiseValidationException(msg = "Duplicate dataTier/filterName combination.") else: duplicateCheck[dataTier].append(filterName) if getOutputModules: return outputModuleInfo return
def determineOutputModules(self, scenarioFunc = None, scenarioArgs = None, configDoc = None, couchURL = None, couchDBName = None, configCacheUrl = None): """ _determineOutputModules_ Determine the output module names and associated metadata for the given config. """ outputModules = {} if configDoc != None and configDoc != "": url = configCacheUrl or couchURL configCache = ConfigCache(url, couchDBName) configCache.loadByID(configDoc) outputModules = configCache.getOutputModuleInfo() else: if 'outputs' in scenarioArgs and scenarioFunc in [ "promptReco", "expressProcessing", "repack" ]: for output in scenarioArgs.get('outputs', []): moduleLabel = output['moduleLabel'] outputModules[moduleLabel] = { 'dataTier' : output['dataTier'] } if output.has_key('primaryDataset'): outputModules[moduleLabel]['primaryDataset'] = output['primaryDataset'] if output.has_key('filterName'): outputModules[moduleLabel]['filterName'] = output['filterName'] elif 'writeTiers' in scenarioArgs and scenarioFunc == "promptReco": for dataTier in scenarioArgs.get('writeTiers'): moduleLabel = "%soutput" % dataTier outputModules[moduleLabel] = { 'dataTier' : dataTier } elif scenarioFunc == "alcaSkim": for alcaSkim in scenarioArgs.get('skims',[]): moduleLabel = "ALCARECOStream%s" % alcaSkim if alcaSkim == "PromptCalibProd": dataTier = "ALCAPROMPT" else: dataTier = "ALCARECO" outputModules[moduleLabel] = { 'dataTier' : dataTier, 'primaryDataset' : scenarioArgs.get('primaryDataset'), 'filterName' : alcaSkim } return outputModules
def determineOutputModules(self, scenarioName = None, scenarioArgs = None, configDoc = None, couchURL = None, couchDBName = None): """ _determineOutputModules_ Determine the output module names and associated metadata for the given config. """ outputModules = {} if configDoc != None and configDoc != "": configCache = ConfigCache(couchURL, couchDBName) configCache.loadByID(configDoc) outputModules = configCache.getOutputModuleInfo() else: for dataTier in scenarioArgs.get("writeTiers",[]): outputModuleName = "output%s%s" % (dataTier, dataTier) outputModules[outputModuleName] = {"dataTier": dataTier, "filterName": None} return outputModules
def testA_basicConfig(self): """ _basicConfig_ Basic configCache stuff. """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." configCache = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test') configCache.createUserGroup(groupname="testGroup", username='******') configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.save() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test', id=configCache.getCouchID(), rev=configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) self.assertEqual(configCache2.getPSetTweaks(), PSetTweak) configCache2.delete() configCache3 = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test', id=configCache.getCouchID(), rev=configCache.getCouchRev()) testFlag = False # It should fail to load deleted documents try: configCache3.loadByID(configCache.getCouchID()) except ConfigCacheException: testFlag = True self.assertTrue(testFlag) return
def determineOutputModules( self, scenarioFunc=None, scenarioArgs=None, configDoc=None, couchURL=None, couchDBName=None ): """ _determineOutputModules_ Determine the output module names and associated metadata for the given config. """ outputModules = {} if configDoc != None and configDoc != "": configCache = ConfigCache(couchURL, couchDBName) configCache.loadByID(configDoc) outputModules = configCache.getOutputModuleInfo() else: if "outputs" in scenarioArgs and scenarioFunc in ["promptReco", "expressProcessing", "repack"]: for output in scenarioArgs.get("outputs", []): moduleLabel = output["moduleLabel"] outputModules[moduleLabel] = {"dataTier": output["dataTier"]} if output.has_key("primaryDataset"): outputModules[moduleLabel]["primaryDataset"] = output["primaryDataset"] if output.has_key("filterName"): outputModules[moduleLabel]["filterName"] = output["filterName"] elif "writeTiers" in scenarioArgs and scenarioFunc == "promptReco": for dataTier in scenarioArgs.get("writeTiers"): moduleLabel = "%soutput" % dataTier outputModules[moduleLabel] = {"dataTier": dataTier} elif scenarioFunc == "alcaSkim": for alcaSkim in scenarioArgs.get("skims", []): moduleLabel = "ALCARECOStream%s" % alcaSkim outputModules[moduleLabel] = { "dataTier": "ALCARECO", "primaryDataset": scenarioArgs.get("primaryDataset"), "filterName": alcaSkim, } return outputModules
def validateConfigCacheExists(self, configID, couchURL, couchDBName, getOutputModules=False): """ _validateConfigCacheExists_ If we have a configCache, we should probably try and load it. """ if configID == '' or configID == ' ': self.raiseValidationException( msg="ConfigCacheID is invalid and cannot be loaded") configCache = ConfigCache(dbURL=couchURL, couchDBName=couchDBName, id=configID) try: configCache.loadByID(configID=configID) except ConfigCacheException, ex: self.raiseValidationException( msg="Failure to load ConfigCache while validating workload")
def testA_basicConfig(self): """ _basicConfig_ Basic configCache stuff. """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache.createUserGroup(groupname = "testGroup", username = '******') configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.save() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test', id = configCache.getCouchID(), rev = configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) self.assertEqual(configCache2.getPSetTweaks(), PSetTweak) configCache2.delete() configCache3 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test', id = configCache.getCouchID(), rev = configCache.getCouchRev()) testFlag = False # It should fail to load deleted documents try: configCache3.loadByID(configCache.getCouchID()) except ConfigCacheException: testFlag = True self.assertTrue(testFlag) return
def showTweakFile(self, docId): """ Makes a link to the dump of the tweakfile """ configCache = ConfigCache(self.couchUrl, self.configDBName) configCache.loadByID(docId) return str(configCache.getPSetTweaks()).replace('\n', '<br>')
def showTweakFile(self, docId): """ Makes a link to the dump of the tweakfile """ configCache = ConfigCache(self.couchUrl, self.configDBName) configCache.loadByID(docId) return str(configCache.getPSetTweaks()).replace('\n', '<br>')
messages=request['RequestMessages'], updateDictList=request['RequestUpdates']) def _getConfigCache(self, requestName, processMethod): try: request = Utilities.requestDetails(requestName) except Exception, ex: msg = "Cannot find request %s, check logs." % requestName logging.error("%s, reason: %s" % (msg, ex)) return msg url = request.get("ConfigCacheUrl", None) or self.couchUrl try: configCache = ConfigCache(url, self.configDBName) configDocId = request["ConfigCacheID"] configCache.loadByID(configDocId) except Exception, ex: msg = "Cannot find ConfigCache document %s on %s." % (configDocId, url) logging.error("%s, reason: %s" % (msg, ex)) return msg return getattr(configCache, processMethod)() @cherrypy.expose @cherrypy.tools.secmodv2() def showOriginalConfig(self, requestName): """ Makes a link to the original text of the config document. """ self.validate(requestName)
adminHtml=adminHtml, messages=request['RequestMessages'], updateDictList=request['RequestUpdates']) def _getConfigCache(self, requestName, processMethod): try: request = Utilities.requestDetails(requestName) except Exception, ex: msg = "Cannot find request %s, check logs." % requestName logging.error("%s, reason: %s" % (msg, ex)) return msg url = request.get("ConfigCacheUrl", None) or self.couchUrl try: configCache = ConfigCache(url, self.configDBName) configDocId = request["ConfigCacheID"] configCache.loadByID(configDocId) except Exception, ex: msg = "Cannot find ConfigCache document %s on %s." % (configDocId, url) logging.error("%s, reason: %s" % (msg, ex)) return msg return getattr(configCache, processMethod)() @cherrypy.expose @cherrypy.tools.secmodv2() def showOriginalConfig(self, requestName): """ Makes a link to the original text of the config document. """ self.validate(requestName)
def determineOutputModules( self, scenarioFunc=None, scenarioArgs=None, configDoc=None, couchURL=None, couchDBName=None, configCacheUrl=None ): """ _determineOutputModules_ Determine the output module names and associated metadata for the given config. """ # set default scenarioArgs to empty dictionary if it is None. scenarioArgs = scenarioArgs or {} outputModules = {} if configDoc != None and configDoc != "": url = configCacheUrl or couchURL if (url, couchDBName) in self.config_cache: configCache = self.config_cache[(url, couchDBName)] else: configCache = ConfigCache(url, couchDBName, True) self.config_cache[(url, couchDBName)] = configCache # TODO: need to change to DataCache # configCache.loadDocument(configDoc) configCache.loadByID(configDoc) outputModules = configCache.getOutputModuleInfo() else: if "outputs" in scenarioArgs and scenarioFunc in ["promptReco", "expressProcessing", "repack"]: for output in scenarioArgs.get("outputs", []): moduleLabel = output["moduleLabel"] outputModules[moduleLabel] = {"dataTier": output["dataTier"]} if "primaryDataset" in output: outputModules[moduleLabel]["primaryDataset"] = output["primaryDataset"] if "filterName" in output: outputModules[moduleLabel]["filterName"] = output["filterName"] for physicsSkim in scenarioArgs.get("PhysicsSkims", []): skimToDataTier = { "LogError": "RAW-RECO", "LogErrorMonitor": "USER", "ZElectron": "RAW-RECO", "ZMu": "RAW-RECO", "MuTau": "RAW-RECO", "TopMuEG": "RAW-RECO", "EcalActivity": "RAW-RECO", "CosmicSP": "RAW-RECO", "CosmicTP": "RAW-RECO", "ZMM": "RAW-RECO", "Onia": "RECO", "HighPtJet": "RAW-RECO", "D0Meson": "RECO", "Photon": "AOD", "ZEE": "AOD", "BJet": "AOD", "OniaCentral": "RECO", "OniaPeripheral": "RECO", "SingleTrack": "AOD", "MinBias": "AOD", "OniaUPC": "RAW-RECO", "HighMET": "RECO", "BPHSkim": "USER", } dataTier = skimToDataTier.get(physicsSkim, "USER") moduleLabel = "SKIMStream%s" % physicsSkim outputModules[moduleLabel] = {"dataTier": dataTier, "filterName": physicsSkim} elif scenarioFunc == "alcaSkim": for alcaSkim in scenarioArgs.get("skims", []): moduleLabel = "ALCARECOStream%s" % alcaSkim if alcaSkim.startswith("PromptCalibProd"): dataTier = "ALCAPROMPT" else: dataTier = "ALCARECO" outputModules[moduleLabel] = { "dataTier": dataTier, "primaryDataset": scenarioArgs.get("primaryDataset"), "filterName": alcaSkim, } return outputModules