def __call__(self, wmTask):
"""
Trip through steps, find CMSSW steps, pull in config files,
PSet Tweaks etc
"""
for t in wmTask.steps().nodeIterator():
t = WMStep.WMStepHelper(t)
stepPath = "%s/%s" % (self.workingDirectory(), t.name())
# the CMSSW has a special case with its ConfigCache argument
if not t.stepType() in ("CMSSW", "MulticoreCMSSW"): continue
if getattr(t.data.application.configuration, 'configCacheUrl', None) != None:
# main config file
fileTarget = "%s/%s" % (
stepPath,
t.data.application.command.configuration)
#urllib.urlretrieve(
# t.data.application.configuration.retrieveConfigUrl,
# fileTarget)
# PSet Tweak
cacheUrl = t.data.application.configuration.configCacheUrl
cacheDb = t.data.application.configuration.cacheName
configId = t.data.application.configuration.configId
tweakTarget = t.data.application.command.psetTweak
configCache = ConfigCache(cacheUrl, cacheDb)
configCache.loadByID(configId)
configCache.saveConfigToDisk(targetFile = fileTarget)
tweak = TweakAPI.makeTweakFromJSON(configCache.getPSetTweaks())
if tweak:
tweakFile = "%s/%s" % (stepPath, tweakTarget)
tweak.persist(tweakFile, "json")
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def createAlgoFromInfo(info):
"""
Create an Algo object from basic information
"""
algo = {'ApplicationName': info.get('ApplicationName'),
'ApplicationFamily': info.get('ApplicationFamily'),
'ApplicationVersion': info.get('ApplicationVersion'),
'PSetHash': info.get('PSetHash'),
'PSetContent': None,
'InDBS': info.get('AlgoInDBS', None)
}
configString = info.get('PSetContent')
if configString:
split = configString.split(';;')
cacheURL = split[0]
cacheDB = split[1]
configID = split[2]
try:
configCache = ConfigCache(cacheURL, cacheDB)
configCache.loadByID(configID)
algo['PSetContent'] = configCache.getConfig()
except Exception, ex:
msg = "Exception in getting configCache from DB\n"
msg += "Ignoring this exception and continuing without config.\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
logging.debug("URL: %s, DB: %s, ID: %s" % (cacheURL, cacheDB, configID))
def determineOutputModules(self, scenarioFunc = None, scenarioArgs = None,
configDoc = None, couchURL = None,
couchDBName = None):
"""
_determineOutputModules_
Determine the output module names and associated metadata for the
given config.
"""
outputModules = {}
if configDoc != None and configDoc != "":
configCache = ConfigCache(couchURL, couchDBName)
configCache.loadByID(configDoc)
outputModules = configCache.getOutputModuleInfo()
else:
if scenarioFunc in [ "promptReco", "expressProcessing" ]:
for output in scenarioArgs.get('outputs',[]):
dataTier = output['dataTier']
moduleLabel = output['moduleLabel']
filterName = output.get('filterName', None)
outputModules[moduleLabel] = {'dataTier' : dataTier,
'filterName' : filterName}
elif scenarioFunc == "alcaSkim":
for alcaSkim in scenarioArgs.get('skims',[]):
dataTier = "ALCARECO"
moduleLabel = "ALCARECOStream%s" % alcaSkim
filterName = alcaSkim
outputModules[moduleLabel] = {'dataTier' : dataTier,
'filterName' : filterName}
return outputModules
def showOriginalConfig(self, docId):
""" Makes a link to the original text of the config """
configCache = ConfigCache(self.couchUrl, self.configDBName)
configCache.loadByID(docId)
configString = configCache.getConfig()
if configString == None:
return "Cannot find document " + str(docId) + " in Couch DB"
return '<pre>' + configString + '</pre>'
def showOriginalConfig(self, docId):
""" Makes a link to the original text of the config """
configCache = ConfigCache(self.couchUrl, self.configDBName)
configCache.loadByID(docId)
configString = configCache.getConfig()
if configString == None:
return "Cannot find document " + str(docId) + " in Couch DB"
return '<pre>' + configString + '</pre>'
def determineOutputModules(self,
scenarioFunc=None,
scenarioArgs=None,
configDoc=None,
couchURL=None,
couchDBName=None,
configCacheUrl=None):
"""
_determineOutputModules_
Determine the output module names and associated metadata for the
given config.
"""
# set default scenarioArgs to empty dictionary if it is None.
scenarioArgs = scenarioArgs or {}
outputModules = {}
if configDoc != None and configDoc != "":
url = configCacheUrl or couchURL
configCache = ConfigCache(url, couchDBName)
configCache.loadByID(configDoc)
outputModules = configCache.getOutputModuleInfo()
else:
if 'outputs' in scenarioArgs and scenarioFunc in [
"promptReco", "expressProcessing", "repack"
]:
for output in scenarioArgs.get('outputs', []):
moduleLabel = output['moduleLabel']
outputModules[moduleLabel] = {
'dataTier': output['dataTier']
}
if output.has_key('primaryDataset'):
outputModules[moduleLabel]['primaryDataset'] = output[
'primaryDataset']
if output.has_key('filterName'):
outputModules[moduleLabel]['filterName'] = output[
'filterName']
elif 'writeTiers' in scenarioArgs and scenarioFunc == "promptReco":
for dataTier in scenarioArgs.get('writeTiers'):
moduleLabel = "%soutput" % dataTier
outputModules[moduleLabel] = {'dataTier': dataTier}
elif scenarioFunc == "alcaSkim":
for alcaSkim in scenarioArgs.get('skims', []):
moduleLabel = "ALCARECOStream%s" % alcaSkim
if alcaSkim.startswith("PromptCalibProd"):
dataTier = "ALCAPROMPT"
else:
dataTier = "ALCARECO"
outputModules[moduleLabel] = {
'dataTier': dataTier,
'primaryDataset': scenarioArgs.get('primaryDataset'),
'filterName': alcaSkim
}
return outputModules
def validateConfigCacheExists(self,
configID,
couchURL,
couchDBName,
getOutputModules=False):
"""
_validateConfigCacheExists_
If we have a configCache, we should probably try and load it.
"""
if configID == '' or configID == ' ':
self.raiseValidationException(
msg="ConfigCacheID is invalid and cannot be loaded")
configCache = ConfigCache(dbURL=couchURL,
couchDBName=couchDBName,
id=configID)
try:
configCache.loadByID(configID=configID)
except ConfigCacheException:
self.raiseValidationException(
msg="Failure to load ConfigCache while validating workload")
duplicateCheck = {}
try:
outputModuleInfo = configCache.getOutputModuleInfo()
except Exception:
# Something's gone wrong with trying to open the configCache
msg = "Error in getting output modules from ConfigCache during workload validation. Check ConfigCache formatting!"
self.raiseValidationException(msg=msg)
for outputModule in outputModuleInfo.values():
dataTier = outputModule.get('dataTier', None)
filterName = outputModule.get('filterName', None)
if not dataTier:
self.raiseValidationException(
msg="No DataTier in output module.")
# Add dataTier to duplicate dictionary
if not dataTier in duplicateCheck.keys():
duplicateCheck[dataTier] = []
if filterName in duplicateCheck[dataTier]:
# Then we've seen this combination before
self.raiseValidationException(
msg="Duplicate dataTier/filterName combination.")
else:
duplicateCheck[dataTier].append(filterName)
if getOutputModules:
return outputModuleInfo
return
def determineOutputModules(self, scenarioFunc = None, scenarioArgs = None,
configDoc = None, couchURL = None,
couchDBName = None, configCacheUrl = None):
"""
_determineOutputModules_
Determine the output module names and associated metadata for the
given config.
"""
# set default scenarioArgs to empty dictionary if it is None.
scenarioArgs = scenarioArgs or {}
outputModules = {}
if configDoc != None and configDoc != "":
url = configCacheUrl or couchURL
if (url, couchDBName) in self.config_cache:
configCache = self.config_cache[(url, couchDBName)]
else:
configCache = ConfigCache(url, couchDBName, True)
self.config_cache[(url, couchDBName)] = configCache
#TODO: need to change to DataCache
#configCache.loadDocument(configDoc)
configCache.loadByID(configDoc)
outputModules = configCache.getOutputModuleInfo()
else:
if 'outputs' in scenarioArgs and scenarioFunc in [ "promptReco", "expressProcessing", "repack" ]:
for output in scenarioArgs.get('outputs', []):
moduleLabel = output['moduleLabel']
outputModules[moduleLabel] = { 'dataTier' : output['dataTier'] }
if 'primaryDataset' in output:
outputModules[moduleLabel]['primaryDataset'] = output['primaryDataset']
if 'filterName' in output:
outputModules[moduleLabel]['filterName'] = output['filterName']
elif 'writeTiers' in scenarioArgs and scenarioFunc == "promptReco":
for dataTier in scenarioArgs.get('writeTiers'):
moduleLabel = "%soutput" % dataTier
outputModules[moduleLabel] = { 'dataTier' : dataTier }
elif scenarioFunc == "alcaSkim":
for alcaSkim in scenarioArgs.get('skims',[]):
moduleLabel = "ALCARECOStream%s" % alcaSkim
if alcaSkim.startswith("PromptCalibProd"):
dataTier = "ALCAPROMPT"
else:
dataTier = "ALCARECO"
outputModules[moduleLabel] = { 'dataTier' : dataTier,
'primaryDataset' : scenarioArgs.get('primaryDataset'),
'filterName' : alcaSkim }
return outputModules
def validateConfigCacheExists(self, configID, couchURL, couchDBName,
getOutputModules = False):
"""
_validateConfigCacheExists_
If we have a configCache, we should probably try and load it.
"""
from WMCore.Cache.WMConfigCache import ConfigCache
configCache = ConfigCache(dbURL = couchURL,
couchDBName = couchDBName,
id = configID)
try:
configCache.loadByID(configID = configID)
except ConfigCacheException, ex:
self.raiseValidationException(msg = "Failure to load ConfigCache while validating workload")
def createAlgoFromInfo(info):
"""
Create an Algo object from basic information
"""
algo = {
'ApplicationName': info['ApplicationName'],
'ApplicationFamily': info['ApplicationFamily'],
'ApplicationVersion': info['ApplicationVersion'],
'PSetHash': info['PSetHash'],
'PSetContent': None,
'InDBS': info['AlgoInDBS']
}
configString = info.get('PSetContent')
if configString:
try:
split = configString.split(';;')
cacheURL = split[0]
cacheDB = split[1]
configID = split[2]
except IndexError:
msg = "configCache not properly formatted\n"
msg += "configString\n: %s" % configString
msg += "Not attempting to put configCache content in DBS for this algo"
msg += "AlgoInfo: %s" % algo
logging.error(msg)
return algo
if cacheURL == "None" or cacheDB == "None" or configID == "None":
# No Config for this DB
logging.debug("No configCache for this algo")
return algo
try:
configCache = ConfigCache(cacheURL, cacheDB)
configCache.loadByID(configID)
algo['PSetContent'] = configCache.getConfig()
except Exception as ex:
msg = "Exception in getting configCache from DB\n"
msg += "Ignoring this exception and continuing without config.\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
logging.debug("URL: %s, DB: %s, ID: %s" %
(cacheURL, cacheDB, configID))
return algo
def _getConfigCache(self, requestName, processMethod):
try:
request = Utilities.requestDetails(requestName)
except Exception as ex:
msg = "Cannot find request %s, check logs." % requestName
logging.error("%s, reason: %s" % (msg, ex))
return msg
url = request.get("ConfigCacheUrl", None) or self.couchUrl
try:
configCache = ConfigCache(url, self.configDBName)
configDocId = request["ConfigCacheID"]
configCache.loadByID(configDocId)
except Exception as ex:
msg = "Cannot find ConfigCache document %s on %s." % (configDocId, url)
logging.error("%s, reason: %s" % (msg, ex))
return msg
return getattr(configCache, processMethod)()
def validateConfigCacheExists(self, configID, couchURL, couchDBName,
getOutputModules = False):
"""
_validateConfigCacheExists_
If we have a configCache, we should probably try and load it.
"""
if configID == '' or configID == ' ':
self.raiseValidationException(msg = "ConfigCacheID is invalid and cannot be loaded")
configCache = ConfigCache(dbURL = couchURL, couchDBName = couchDBName,
id = configID)
try:
configCache.loadByID(configID = configID)
except ConfigCacheException, ex:
self.raiseValidationException(msg = "Failure to load ConfigCache while validating workload")
def createAlgoFromInfo(info):
"""
Create an Algo object from basic information
"""
algo = {
"ApplicationName": info["ApplicationName"],
"ApplicationFamily": info["ApplicationFamily"],
"ApplicationVersion": info["ApplicationVersion"],
"PSetHash": info["PSetHash"],
"PSetContent": None,
"InDBS": info["AlgoInDBS"],
}
configString = info.get("PSetContent")
if configString:
try:
split = configString.split(";;")
cacheURL = split[0]
cacheDB = split[1]
configID = split[2]
except IndexError:
msg = "configCache not properly formatted\n"
msg += "configString\n: %s" % configString
msg += "Not attempting to put configCache content in DBS for this algo"
msg += "AlgoInfo: %s" % algo
logging.error(msg)
return algo
if cacheURL == "None" or cacheDB == "None" or configID == "None":
# No Config for this DB
logging.debug("No configCache for this algo")
return algo
try:
configCache = ConfigCache(cacheURL, cacheDB)
configCache.loadByID(configID)
algo["PSetContent"] = configCache.getConfig()
except Exception as ex:
msg = "Exception in getting configCache from DB\n"
msg += "Ignoring this exception and continuing without config.\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
logging.debug("URL: %s, DB: %s, ID: %s" % (cacheURL, cacheDB, configID))
return algo
def validateConfigCacheExists(self, configID, couchURL, couchDBName,
getOutputModules = False):
"""
_validateConfigCacheExists_
If we have a configCache, we should probably try and load it.
"""
if configID == '' or configID == ' ':
self.raiseValidationException(msg = "ConfigCacheID is invalid and cannot be loaded")
configCache = ConfigCache(dbURL = couchURL, couchDBName = couchDBName,
id = configID)
try:
configCache.loadByID(configID = configID)
except ConfigCacheException:
self.raiseValidationException(msg = "Failure to load ConfigCache while validating workload")
duplicateCheck = {}
try:
outputModuleInfo = configCache.getOutputModuleInfo()
except Exception:
# Something's gone wrong with trying to open the configCache
msg = "Error in getting output modules from ConfigCache during workload validation. Check ConfigCache formatting!"
self.raiseValidationException(msg = msg)
for outputModule in outputModuleInfo.values():
dataTier = outputModule.get('dataTier', None)
filterName = outputModule.get('filterName', None)
if not dataTier:
self.raiseValidationException(msg = "No DataTier in output module.")
# Add dataTier to duplicate dictionary
if not dataTier in duplicateCheck.keys():
duplicateCheck[dataTier] = []
if filterName in duplicateCheck[dataTier]:
# Then we've seen this combination before
self.raiseValidationException(msg = "Duplicate dataTier/filterName combination.")
else:
duplicateCheck[dataTier].append(filterName)
if getOutputModules:
return outputModuleInfo
return
def determineOutputModules(self, scenarioFunc = None, scenarioArgs = None,
configDoc = None, couchURL = None,
couchDBName = None, configCacheUrl = None):
"""
_determineOutputModules_
Determine the output module names and associated metadata for the
given config.
"""
outputModules = {}
if configDoc != None and configDoc != "":
url = configCacheUrl or couchURL
configCache = ConfigCache(url, couchDBName)
configCache.loadByID(configDoc)
outputModules = configCache.getOutputModuleInfo()
else:
if 'outputs' in scenarioArgs and scenarioFunc in [ "promptReco", "expressProcessing", "repack" ]:
for output in scenarioArgs.get('outputs', []):
moduleLabel = output['moduleLabel']
outputModules[moduleLabel] = { 'dataTier' : output['dataTier'] }
if output.has_key('primaryDataset'):
outputModules[moduleLabel]['primaryDataset'] = output['primaryDataset']
if output.has_key('filterName'):
outputModules[moduleLabel]['filterName'] = output['filterName']
elif 'writeTiers' in scenarioArgs and scenarioFunc == "promptReco":
for dataTier in scenarioArgs.get('writeTiers'):
moduleLabel = "%soutput" % dataTier
outputModules[moduleLabel] = { 'dataTier' : dataTier }
elif scenarioFunc == "alcaSkim":
for alcaSkim in scenarioArgs.get('skims',[]):
moduleLabel = "ALCARECOStream%s" % alcaSkim
if alcaSkim == "PromptCalibProd":
dataTier = "ALCAPROMPT"
else:
dataTier = "ALCARECO"
outputModules[moduleLabel] = { 'dataTier' : dataTier,
'primaryDataset' : scenarioArgs.get('primaryDataset'),
'filterName' : alcaSkim }
return outputModules
def determineOutputModules(self, scenarioName = None, scenarioArgs = None,
configDoc = None, couchURL = None,
couchDBName = None):
"""
_determineOutputModules_
Determine the output module names and associated metadata for the
given config.
"""
outputModules = {}
if configDoc != None and configDoc != "":
configCache = ConfigCache(couchURL, couchDBName)
configCache.loadByID(configDoc)
outputModules = configCache.getOutputModuleInfo()
else:
for dataTier in scenarioArgs.get("writeTiers",[]):
outputModuleName = "output%s%s" % (dataTier, dataTier)
outputModules[outputModuleName] = {"dataTier": dataTier,
"filterName": None}
return outputModules
def testA_basicConfig(self):
"""
_basicConfig_
Basic configCache stuff.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
self.assertEqual(configCache2.getPSetTweaks(), PSetTweak)
configCache2.delete()
configCache3 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
testFlag = False
# It should fail to load deleted documents
try:
configCache3.loadByID(configCache.getCouchID())
except ConfigCacheException:
testFlag = True
self.assertTrue(testFlag)
return
def determineOutputModules(
self, scenarioFunc=None, scenarioArgs=None, configDoc=None, couchURL=None, couchDBName=None
):
"""
_determineOutputModules_
Determine the output module names and associated metadata for the
given config.
"""
outputModules = {}
if configDoc != None and configDoc != "":
configCache = ConfigCache(couchURL, couchDBName)
configCache.loadByID(configDoc)
outputModules = configCache.getOutputModuleInfo()
else:
if "outputs" in scenarioArgs and scenarioFunc in ["promptReco", "expressProcessing", "repack"]:
for output in scenarioArgs.get("outputs", []):
moduleLabel = output["moduleLabel"]
outputModules[moduleLabel] = {"dataTier": output["dataTier"]}
if output.has_key("primaryDataset"):
outputModules[moduleLabel]["primaryDataset"] = output["primaryDataset"]
if output.has_key("filterName"):
outputModules[moduleLabel]["filterName"] = output["filterName"]
elif "writeTiers" in scenarioArgs and scenarioFunc == "promptReco":
for dataTier in scenarioArgs.get("writeTiers"):
moduleLabel = "%soutput" % dataTier
outputModules[moduleLabel] = {"dataTier": dataTier}
elif scenarioFunc == "alcaSkim":
for alcaSkim in scenarioArgs.get("skims", []):
moduleLabel = "ALCARECOStream%s" % alcaSkim
outputModules[moduleLabel] = {
"dataTier": "ALCARECO",
"primaryDataset": scenarioArgs.get("primaryDataset"),
"filterName": alcaSkim,
}
return outputModules
def validateConfigCacheExists(self,
configID,
couchURL,
couchDBName,
getOutputModules=False):
"""
_validateConfigCacheExists_
If we have a configCache, we should probably try and load it.
"""
if configID == '' or configID == ' ':
self.raiseValidationException(
msg="ConfigCacheID is invalid and cannot be loaded")
configCache = ConfigCache(dbURL=couchURL,
couchDBName=couchDBName,
id=configID)
try:
configCache.loadByID(configID=configID)
except ConfigCacheException, ex:
self.raiseValidationException(
msg="Failure to load ConfigCache while validating workload")
def testA_basicConfig(self):
"""
_basicConfig_
Basic configCache stuff.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
self.assertEqual(configCache2.getPSetTweaks(), PSetTweak)
configCache2.delete()
configCache3 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
testFlag = False
# It should fail to load deleted documents
try:
configCache3.loadByID(configCache.getCouchID())
except ConfigCacheException:
testFlag = True
self.assertTrue(testFlag)
return
def showTweakFile(self, docId):
""" Makes a link to the dump of the tweakfile """
configCache = ConfigCache(self.couchUrl, self.configDBName)
configCache.loadByID(docId)
return str(configCache.getPSetTweaks()).replace('\n', '<br>')
def showTweakFile(self, docId):
""" Makes a link to the dump of the tweakfile """
configCache = ConfigCache(self.couchUrl, self.configDBName)
configCache.loadByID(docId)
return str(configCache.getPSetTweaks()).replace('\n', '<br>')
messages=request['RequestMessages'],
updateDictList=request['RequestUpdates'])
def _getConfigCache(self, requestName, processMethod):
try:
request = Utilities.requestDetails(requestName)
except Exception, ex:
msg = "Cannot find request %s, check logs." % requestName
logging.error("%s, reason: %s" % (msg, ex))
return msg
url = request.get("ConfigCacheUrl", None) or self.couchUrl
try:
configCache = ConfigCache(url, self.configDBName)
configDocId = request["ConfigCacheID"]
configCache.loadByID(configDocId)
except Exception, ex:
msg = "Cannot find ConfigCache document %s on %s." % (configDocId, url)
logging.error("%s, reason: %s" % (msg, ex))
return msg
return getattr(configCache, processMethod)()
@cherrypy.expose
@cherrypy.tools.secmodv2()
def showOriginalConfig(self, requestName):
"""
Makes a link to the original text of the config document.
"""
self.validate(requestName)
adminHtml=adminHtml,
messages=request['RequestMessages'],
updateDictList=request['RequestUpdates'])
def _getConfigCache(self, requestName, processMethod):
try:
request = Utilities.requestDetails(requestName)
except Exception, ex:
msg = "Cannot find request %s, check logs." % requestName
logging.error("%s, reason: %s" % (msg, ex))
return msg
url = request.get("ConfigCacheUrl", None) or self.couchUrl
try:
configCache = ConfigCache(url, self.configDBName)
configDocId = request["ConfigCacheID"]
configCache.loadByID(configDocId)
except Exception, ex:
msg = "Cannot find ConfigCache document %s on %s." % (configDocId,
url)
logging.error("%s, reason: %s" % (msg, ex))
return msg
return getattr(configCache, processMethod)()
@cherrypy.expose
@cherrypy.tools.secmodv2()
def showOriginalConfig(self, requestName):
"""
Makes a link to the original text of the config document.
"""
self.validate(requestName)
def determineOutputModules(
self, scenarioFunc=None, scenarioArgs=None, configDoc=None, couchURL=None, couchDBName=None, configCacheUrl=None
):
"""
_determineOutputModules_
Determine the output module names and associated metadata for the
given config.
"""
# set default scenarioArgs to empty dictionary if it is None.
scenarioArgs = scenarioArgs or {}
outputModules = {}
if configDoc != None and configDoc != "":
url = configCacheUrl or couchURL
if (url, couchDBName) in self.config_cache:
configCache = self.config_cache[(url, couchDBName)]
else:
configCache = ConfigCache(url, couchDBName, True)
self.config_cache[(url, couchDBName)] = configCache
# TODO: need to change to DataCache
# configCache.loadDocument(configDoc)
configCache.loadByID(configDoc)
outputModules = configCache.getOutputModuleInfo()
else:
if "outputs" in scenarioArgs and scenarioFunc in ["promptReco", "expressProcessing", "repack"]:
for output in scenarioArgs.get("outputs", []):
moduleLabel = output["moduleLabel"]
outputModules[moduleLabel] = {"dataTier": output["dataTier"]}
if "primaryDataset" in output:
outputModules[moduleLabel]["primaryDataset"] = output["primaryDataset"]
if "filterName" in output:
outputModules[moduleLabel]["filterName"] = output["filterName"]
for physicsSkim in scenarioArgs.get("PhysicsSkims", []):
skimToDataTier = {
"LogError": "RAW-RECO",
"LogErrorMonitor": "USER",
"ZElectron": "RAW-RECO",
"ZMu": "RAW-RECO",
"MuTau": "RAW-RECO",
"TopMuEG": "RAW-RECO",
"EcalActivity": "RAW-RECO",
"CosmicSP": "RAW-RECO",
"CosmicTP": "RAW-RECO",
"ZMM": "RAW-RECO",
"Onia": "RECO",
"HighPtJet": "RAW-RECO",
"D0Meson": "RECO",
"Photon": "AOD",
"ZEE": "AOD",
"BJet": "AOD",
"OniaCentral": "RECO",
"OniaPeripheral": "RECO",
"SingleTrack": "AOD",
"MinBias": "AOD",
"OniaUPC": "RAW-RECO",
"HighMET": "RECO",
"BPHSkim": "USER",
}
dataTier = skimToDataTier.get(physicsSkim, "USER")
moduleLabel = "SKIMStream%s" % physicsSkim
outputModules[moduleLabel] = {"dataTier": dataTier, "filterName": physicsSkim}
elif scenarioFunc == "alcaSkim":
for alcaSkim in scenarioArgs.get("skims", []):
moduleLabel = "ALCARECOStream%s" % alcaSkim
if alcaSkim.startswith("PromptCalibProd"):
dataTier = "ALCAPROMPT"
else:
dataTier = "ALCARECO"
outputModules[moduleLabel] = {
"dataTier": dataTier,
"primaryDataset": scenarioArgs.get("primaryDataset"),
"filterName": alcaSkim,
}
return outputModules