def testListAllConfigs(self):
"""
_testListAllConfigs_
Verify that the list all configs method works correctly.
"""
configCacheA = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCacheA.createUserGroup(groupname="testGroup", username='******')
configCacheA.setLabel("labelA")
configCacheA.save()
configCacheB = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCacheB.createUserGroup(groupname="testGroup", username='******')
configCacheB.setLabel("labelB")
configCacheB.save()
configs = configCacheA.listAllConfigsByLabel()
self.assertEqual(len(configs.keys()), 2,
"Error: There should be two configs")
self.assertEqual(configs["labelA"], configCacheA.getCouchID(),
"Error: Label A is wrong.")
self.assertEqual(configs["labelB"], configCacheB.getCouchID(),
"Error: Label B is wrong.")
return
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def testD_LoadConfigCache(self):
"""
_LoadConfigCache_
Actually load the config cache using plain .load()
Tests to make sure that if we pass in an id field it gets used to load configs
"""
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setLabel("labelA")
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.load()
self.assertEqual(configCache2.document['owner'], {
'group': 'testGroup',
'user': '******'
})
self.assertEqual(configCache2.document['description'], {
'config_desc': None,
'config_label': 'labelA'
})
return
def _configCacheId(self, label):
"""Return config cache id for given config label"""
key, cert = self.__class__.reqmgr['requests'].getKeyCert()
configCache = ConfigCache(self.__class__.endpoint + '/couchdb',
'reqmgr_config_cache',
ckey=key,
cert=cert)
try:
configCacheId = configCache.getIDFromLabel(label)
except:
configCacheId = None
if configCacheId:
return configCacheId
# The following will fail if FWCore.ParameterSet not in PYTHONPATH
from PSetTweaks.WMTweak import makeTweak
configCache.createUserGroup('test', 'test')
configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data',
'configs')
configCache.addConfig(os.path.join(configDir, label + '.py'))
configCache.setLabel(label)
configCache.setDescription(label)
modPath = imp.find_module(label, [configDir])
loadedConfig = imp.load_module(label, modPath[0], modPath[1],
modPath[2])
configCache.setPSetTweaks(
makeTweak(loadedConfig.process).jsondictionary())
configCache.save()
return configCache.getIDFromLabel(label)
def testListAllConfigs(self):
"""
_testListAllConfigs_
Verify that the list all configs method works correctly.
"""
configCacheA = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCacheA.createUserGroup(groupname = "testGroup", username = '******')
configCacheA.setLabel("labelA")
configCacheA.save()
configCacheB = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCacheB.createUserGroup(groupname = "testGroup", username = '******')
configCacheB.setLabel("labelB")
configCacheB.save()
configs = configCacheA.listAllConfigsByLabel()
self.assertEqual(len(configs.keys()), 2,
"Error: There should be two configs")
self.assertEqual(configs["labelA"], configCacheA.getCouchID(),
"Error: Label A is wrong.")
self.assertEqual(configs["labelB"], configCacheB.getCouchID(),
"Error: Label B is wrong.")
return
def upload_to_couch(cfg_name,
section_name,
user_name,
group_name,
test_mode=False,
url=None):
if test_mode:
return "00000000000000000"
if not os.path.exists(cfg_name):
raise RuntimeError("Error: Can't locate config file %s." % cfg_name)
# create a file with the ID inside to avoid multiple injections
oldID = cfg_name + '.couchID'
if os.path.exists(oldID):
f = open(oldID)
the_id = f.readline().replace('\n', '')
f.close()
print cfg_name, 'already uploaded with ID', the_id, 'from', oldID
return the_id
try:
loadedConfig = __loadConfig(cfg_name)
except:
#just try again !!
time.sleep(2)
loadedConfig = __loadConfig(cfg_name)
where = COUCH_DB_ADDRESS
if url:
where = url
configCache = ConfigCache(where, DATABASE_NAME)
configCache.createUserGroup(group_name, user_name)
configCache.addConfig(cfg_name)
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.setLabel(section_name)
configCache.setDescription(section_name)
configCache.save()
print "Added file to the config cache:"
print " DocID: %s" % configCache.document["_id"]
print " Revision: %s" % configCache.document["_rev"]
f = open(oldID, "w")
f.write(configCache.document["_id"])
f.close()
return configCache.document["_id"]
def upload_to_couch(cfg_name, section_name, user_name, group_name, test_mode=False, url=None):
if test_mode:
return "00000000000000000"
if not os.path.exists(cfg_name):
raise RuntimeError("Error: Can't locate config file %s." % cfg_name)
# create a file with the ID inside to avoid multiple injections
oldID = cfg_name + '.couchID'
if os.path.exists(oldID):
f = open(oldID)
the_id = f.readline().replace('\n','')
f.close()
print cfg_name, 'already uploaded with ID', the_id, 'from', oldID
return the_id
try:
loadedConfig = __loadConfig(cfg_name)
except:
#just try again !!
time.sleep(2)
loadedConfig = __loadConfig(cfg_name)
where = COUCH_DB_ADDRESS
if url:
where = url
configCache = ConfigCache(where, DATABASE_NAME)
configCache.createUserGroup(group_name, user_name)
configCache.addConfig(cfg_name)
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.setLabel(section_name)
configCache.setDescription(section_name)
configCache.save()
print "Added file to the config cache:"
print " DocID: %s" % configCache.document["_id"]
print " Revision: %s" % configCache.document["_rev"]
f = open(oldID,"w")
f.write(configCache.document["_id"])
f.close()
return configCache.document["_id"]
def upload(self, configModule, group, userDN,
label="Autogenerated by YAAT",
description="Autogenerated by YAAT",
url="http://se2.accre.vanderbilt.edu:5984",
database="wmagent_configcache"):
if url.endswith('/'):
raise RuntimeError, "URL shouldn't have a trailing slash"
configCache = CMSConfigCache(url, database)
configCache.createUserGroup(group, userDN)
tweaks = makeTweak(configModule.process).jsondictionary()
requiredFields = (('dataTier','USER'),('filterName',''))
for currentModule in tweaks['process']['outputModules_']:
if not 'dataset' in tweaks['process'][currentModule]:
tweaks['process'][currentModule]['dataset'] = {
"parameters_": [
"dataTier",
"filterName"
],
"filterName": "",
"dataTier": "USER"
}
continue
for reqKey, defaultValue in (('dataTier','USER'),('filterName','')):
if not reqKey in tweaks['process'][currentModule]['dataset']['parameters_']:
tweaks['process'][currentModule]['dataset']['parameters_'].append(reqKey)
tweaks['process'][currentModule]['dataset'][reqKey] = defaultValue
_, fileName = tempfile.mkstemp()
try:
filename = self.writeFile(configModule, fileName)
configCache.addConfig(filename)
configCache.setPSetTweaks(tweaks)
configCache.setLabel(label)
configCache.setDescription(description)
configCache.save()
finally:
if os.path.exists(fileName):
os.unlink(fileName)
targetUrl = "%s/%s/%s/configFile" % (url, database, configCache.document["_id"])
return targetUrl
def _configCacheId(self, label):
"""Return config cache id for given config label"""
key, cert = self.__class__.reqmgr['requests'].getKeyCert()
configCache = ConfigCache(self.__class__.endpoint + '/couchdb', 'reqmgr_config_cache', ckey = key, cert = cert)
try:
configCacheId = configCache.getIDFromLabel(label)
except:
configCacheId = None
if configCacheId:
return configCacheId
# The following will fail if FWCore.ParameterSet not in PYTHONPATH
from PSetTweaks.WMTweak import makeTweak
configCache.createUserGroup('test', 'test')
configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data', 'configs')
configCache.addConfig(os.path.join(configDir, label + '.py'))
configCache.setLabel(label)
configCache.setDescription(label)
modPath = imp.find_module(label, [configDir])
loadedConfig = imp.load_module(label, modPath[0], modPath[1], modPath[2])
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.save()
return configCache.getIDFromLabel(label)
def testD_LoadConfigCache(self):
"""
_LoadConfigCache_
Actually load the config cache using plain .load()
Tests to make sure that if we pass in an id field it gets used to load configs
"""
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setLabel("labelA")
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.load()
self.assertEqual(configCache2.document['owner'],
{'group': 'testGroup', 'user': '******'})
self.assertEqual(configCache2.document['description'],
{'config_desc': None, 'config_label': 'labelA'})
return
