def _configCacheId(self, label):
"""Return config cache id for given config label"""
key, cert = self.__class__.reqmgr['requests'].getKeyCert()
configCache = ConfigCache(self.__class__.endpoint + '/couchdb',
'reqmgr_config_cache',
ckey=key,
cert=cert)
try:
configCacheId = configCache.getIDFromLabel(label)
except:
configCacheId = None
if configCacheId:
return configCacheId
# The following will fail if FWCore.ParameterSet not in PYTHONPATH
from PSetTweaks.WMTweak import makeTweak
configCache.createUserGroup('test', 'test')
configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data',
'configs')
configCache.addConfig(os.path.join(configDir, label + '.py'))
configCache.setLabel(label)
configCache.setDescription(label)
modPath = imp.find_module(label, [configDir])
loadedConfig = imp.load_module(label, modPath[0], modPath[1],
modPath[2])
configCache.setPSetTweaks(
makeTweak(loadedConfig.process).jsondictionary())
configCache.save()
return configCache.getIDFromLabel(label)
def testListAllConfigs(self):
"""
_testListAllConfigs_
Verify that the list all configs method works correctly.
"""
configCacheA = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCacheA.createUserGroup(groupname="testGroup", username='******')
configCacheA.setLabel("labelA")
configCacheA.save()
configCacheB = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCacheB.createUserGroup(groupname="testGroup", username='******')
configCacheB.setLabel("labelB")
configCacheB.save()
configs = configCacheA.listAllConfigsByLabel()
self.assertEqual(len(configs.keys()), 2,
"Error: There should be two configs")
self.assertEqual(configs["labelA"], configCacheA.getCouchID(),
"Error: Label A is wrong.")
self.assertEqual(configs["labelB"], configCacheB.getCouchID(),
"Error: Label B is wrong.")
return
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
"process": {
"outputModules_": ["ThisIsAName"],
"ThisIsAName": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
}
}
BadTweak = {
"process": {
"outputModules_": ["ThisIsAName1", "ThisIsAName2"],
"ThisIsAName1": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
"ThisIsAName2": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
}
}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username="******")
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
def testC_testViews(self):
"""
_testViews_
Prototype test for what should be a lot of other tests.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.attachments['attach1'] = attach
configCache.document['md5_hash'] = "somemd5"
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache2.document['md5_hash'] = configCache.document['md5_hash']
configCache2.load()
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache2.delete()
return
def testD_LoadConfigCache(self):
"""
_LoadConfigCache_
Actually load the config cache using plain .load()
Tests to make sure that if we pass in an id field it gets used to load configs
"""
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setLabel("labelA")
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.load()
self.assertEqual(configCache2.document['owner'], {
'group': 'testGroup',
'user': '******'
})
self.assertEqual(configCache2.document['description'], {
'config_desc': None,
'config_label': 'labelA'
})
return
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def testListAllConfigs(self):
"""
_testListAllConfigs_
Verify that the list all configs method works correctly.
"""
configCacheA = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCacheA.createUserGroup(groupname = "testGroup", username = '******')
configCacheA.setLabel("labelA")
configCacheA.save()
configCacheB = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCacheB.createUserGroup(groupname = "testGroup", username = '******')
configCacheB.setLabel("labelB")
configCacheB.save()
configs = configCacheA.listAllConfigsByLabel()
self.assertEqual(len(configs.keys()), 2,
"Error: There should be two configs")
self.assertEqual(configs["labelA"], configCacheA.getCouchID(),
"Error: Label A is wrong.")
self.assertEqual(configs["labelB"], configCacheB.getCouchID(),
"Error: Label B is wrong.")
return
def createConfig(couchDBName):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
'process': {
'outputModules_': ['RECOoutput', 'ALCARECOoutput'],
'RECOoutput': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
},
'ALCARECOoutput': {
'dataset': {
'dataTier': 'ALCARECO',
'filterName': 'AlcaFilter'
}
}
}
}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=couchDBName)
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
def testC_testViews(self):
"""
_testViews_
Prototype test for what should be a lot of other tests.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
configCache.document['md5_hash'] = "somemd5"
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache2.document['md5_hash'] = configCache.document['md5_hash']
configCache2.load()
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache2.delete()
return
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
# Set constants
self.couchDB = "config_test"
self.configURL = "RANDOM;;URL;;NAME"
self.configString = "This is a random string"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMComponent.DBS3Buffer", 'WMCore.Agent.Database'],
useDefault=False)
self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.bufferFactory(
classname="DBSBufferFiles.AddLocation")
locationAction.execute(siteName="se1.cern.ch")
locationAction.execute(siteName="se1.fnal.gov")
locationAction.execute(siteName="malpaquet")
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
# Set up a config cache
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDB)
configCache.createUserGroup(groupname="testGroup", username='******')
self.testDir = self.testInit.generateWorkDir()
psetPath = os.path.join(self.testDir, "PSet.txt")
f = open(psetPath, 'w')
f.write(self.configString)
f.close()
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.save()
self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB,
configCache.getCouchID())
return
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
# Set constants
self.couchDB = "config_test"
self.configURL = "RANDOM;;URL;;NAME"
self.configString = "This is a random string"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules =
["WMComponent.DBS3Buffer",
'WMCore.Agent.Database'],
useDefault = False)
self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
# Set up a config cache
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDB)
configCache.createUserGroup(groupname = "testGroup", username = '******')
self.testDir = self.testInit.generateWorkDir()
psetPath = os.path.join(self.testDir, "PSet.txt")
f = open(psetPath, 'w')
f.write(self.configString)
f.close()
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.save()
self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"],
self.couchDB,
configCache.getCouchID())
return
def getConfig(self):
"""
_getConfig_
Create a test config and put it in the cache
"""
PSetTweak = {'someKey': "Hello, I am a PSetTweak. It's nice to meet you."}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['configFile'] = 'This Is A Test Config'
configCache.save()
return configCache
def upload_to_couch(cfg_name,
section_name,
user_name,
group_name,
test_mode=False,
url=None):
if test_mode:
return "00000000000000000"
if not os.path.exists(cfg_name):
raise RuntimeError("Error: Can't locate config file %s." % cfg_name)
# create a file with the ID inside to avoid multiple injections
oldID = cfg_name + '.couchID'
if os.path.exists(oldID):
f = open(oldID)
the_id = f.readline().replace('\n', '')
f.close()
print cfg_name, 'already uploaded with ID', the_id, 'from', oldID
return the_id
try:
loadedConfig = __loadConfig(cfg_name)
except:
#just try again !!
time.sleep(2)
loadedConfig = __loadConfig(cfg_name)
where = COUCH_DB_ADDRESS
if url:
where = url
configCache = ConfigCache(where, DATABASE_NAME)
configCache.createUserGroup(group_name, user_name)
configCache.addConfig(cfg_name)
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.setLabel(section_name)
configCache.setDescription(section_name)
configCache.save()
print "Added file to the config cache:"
print " DocID: %s" % configCache.document["_id"]
print " Revision: %s" % configCache.document["_rev"]
f = open(oldID, "w")
f.write(configCache.document["_id"])
f.close()
return configCache.document["_id"]
def upload_to_couch(cfg_name, section_name, user_name, group_name, test_mode=False, url=None):
if test_mode:
return "00000000000000000"
if not os.path.exists(cfg_name):
raise RuntimeError("Error: Can't locate config file %s." % cfg_name)
# create a file with the ID inside to avoid multiple injections
oldID = cfg_name + '.couchID'
if os.path.exists(oldID):
f = open(oldID)
the_id = f.readline().replace('\n','')
f.close()
print cfg_name, 'already uploaded with ID', the_id, 'from', oldID
return the_id
try:
loadedConfig = __loadConfig(cfg_name)
except:
#just try again !!
time.sleep(2)
loadedConfig = __loadConfig(cfg_name)
where = COUCH_DB_ADDRESS
if url:
where = url
configCache = ConfigCache(where, DATABASE_NAME)
configCache.createUserGroup(group_name, user_name)
configCache.addConfig(cfg_name)
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.setLabel(section_name)
configCache.setDescription(section_name)
configCache.save()
print "Added file to the config cache:"
print " DocID: %s" % configCache.document["_id"]
print " Revision: %s" % configCache.document["_rev"]
f = open(oldID,"w")
f.write(configCache.document["_id"])
f.close()
return configCache.document["_id"]
def createConfig(couchDBName):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['RECOoutput', 'ALCARECOoutput'],
'RECOoutput': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ALCARECOoutput': {'dataset': {'dataTier': 'ALCARECO',
'filterName': 'AlcaFilter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
def upload(self, configModule, group, userDN,
label="Autogenerated by YAAT",
description="Autogenerated by YAAT",
url="http://se2.accre.vanderbilt.edu:5984",
database="wmagent_configcache"):
if url.endswith('/'):
raise RuntimeError, "URL shouldn't have a trailing slash"
configCache = CMSConfigCache(url, database)
configCache.createUserGroup(group, userDN)
tweaks = makeTweak(configModule.process).jsondictionary()
requiredFields = (('dataTier','USER'),('filterName',''))
for currentModule in tweaks['process']['outputModules_']:
if not 'dataset' in tweaks['process'][currentModule]:
tweaks['process'][currentModule]['dataset'] = {
"parameters_": [
"dataTier",
"filterName"
],
"filterName": "",
"dataTier": "USER"
}
continue
for reqKey, defaultValue in (('dataTier','USER'),('filterName','')):
if not reqKey in tweaks['process'][currentModule]['dataset']['parameters_']:
tweaks['process'][currentModule]['dataset']['parameters_'].append(reqKey)
tweaks['process'][currentModule]['dataset'][reqKey] = defaultValue
_, fileName = tempfile.mkstemp()
try:
filename = self.writeFile(configModule, fileName)
configCache.addConfig(filename)
configCache.setPSetTweaks(tweaks)
configCache.setLabel(label)
configCache.setDescription(description)
configCache.save()
finally:
if os.path.exists(fileName):
os.unlink(fileName)
targetUrl = "%s/%s/%s/configFile" % (url, database, configCache.document["_id"])
return targetUrl
def _configCacheId(self, label):
"""Return config cache id for given config label"""
key, cert = self.__class__.reqmgr['requests'].getKeyCert()
configCache = ConfigCache(self.__class__.endpoint + '/couchdb', 'reqmgr_config_cache', ckey = key, cert = cert)
try:
configCacheId = configCache.getIDFromLabel(label)
except:
configCacheId = None
if configCacheId:
return configCacheId
# The following will fail if FWCore.ParameterSet not in PYTHONPATH
from PSetTweaks.WMTweak import makeTweak
configCache.createUserGroup('test', 'test')
configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data', 'configs')
configCache.addConfig(os.path.join(configDir, label + '.py'))
configCache.setLabel(label)
configCache.setDescription(label)
modPath = imp.find_module(label, [configDir])
loadedConfig = imp.load_module(label, modPath[0], modPath[1], modPath[2])
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.save()
return configCache.getIDFromLabel(label)
def testD_LoadConfigCache(self):
"""
_LoadConfigCache_
Actually load the config cache using plain .load()
Tests to make sure that if we pass in an id field it gets used to load configs
"""
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setLabel("labelA")
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.load()
self.assertEqual(configCache2.document['owner'],
{'group': 'testGroup', 'user': '******'})
self.assertEqual(configCache2.document['description'],
{'config_desc': None, 'config_label': 'labelA'})
return
def testA_basicConfig(self):
"""
_basicConfig_
Basic configCache stuff.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
self.assertEqual(configCache2.getPSetTweaks(), PSetTweak)
configCache2.delete()
configCache3 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
testFlag = False
# It should fail to load deleted documents
try:
configCache3.loadByID(configCache.getCouchID())
except ConfigCacheException:
testFlag = True
self.assertTrue(testFlag)
return
def testA_basicConfig(self):
"""
_basicConfig_
Basic configCache stuff.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
self.assertEqual(configCache2.getPSetTweaks(), PSetTweak)
configCache2.delete()
configCache3 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
testFlag = False
# It should fail to load deleted documents
try:
configCache3.loadByID(configCache.getCouchID())
except ConfigCacheException:
testFlag = True
self.assertTrue(testFlag)
return
