def testC_testViews(self):
"""
_testViews_
Prototype test for what should be a lot of other tests.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.attachments['attach1'] = attach
configCache.document['md5_hash'] = "somemd5"
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache2.document['md5_hash'] = configCache.document['md5_hash']
configCache2.load()
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache2.delete()
return
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def _configCacheId(self, label):
"""Return config cache id for given config label"""
key, cert = self.__class__.reqmgr['requests'].getKeyCert()
configCache = ConfigCache(self.__class__.endpoint + '/couchdb',
'reqmgr_config_cache',
ckey=key,
cert=cert)
try:
configCacheId = configCache.getIDFromLabel(label)
except:
configCacheId = None
if configCacheId:
return configCacheId
# The following will fail if FWCore.ParameterSet not in PYTHONPATH
from PSetTweaks.WMTweak import makeTweak
configCache.createUserGroup('test', 'test')
configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data',
'configs')
configCache.addConfig(os.path.join(configDir, label + '.py'))
configCache.setLabel(label)
configCache.setDescription(label)
modPath = imp.find_module(label, [configDir])
loadedConfig = imp.load_module(label, modPath[0], modPath[1],
modPath[2])
configCache.setPSetTweaks(
makeTweak(loadedConfig.process).jsondictionary())
configCache.save()
return configCache.getIDFromLabel(label)
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def testC_testViews(self):
"""
_testViews_
Prototype test for what should be a lot of other tests.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
configCache.document['md5_hash'] = "somemd5"
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache2.document['md5_hash'] = configCache.document['md5_hash']
configCache2.load()
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache2.delete()
return
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
# Set constants
self.couchDB = "config_test"
self.configURL = "RANDOM;;URL;;NAME"
self.configString = "This is a random string"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMComponent.DBS3Buffer", 'WMCore.Agent.Database'],
useDefault=False)
self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.bufferFactory(
classname="DBSBufferFiles.AddLocation")
locationAction.execute(siteName="se1.cern.ch")
locationAction.execute(siteName="se1.fnal.gov")
locationAction.execute(siteName="malpaquet")
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
# Set up a config cache
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDB)
configCache.createUserGroup(groupname="testGroup", username='******')
self.testDir = self.testInit.generateWorkDir()
psetPath = os.path.join(self.testDir, "PSet.txt")
f = open(psetPath, 'w')
f.write(self.configString)
f.close()
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.save()
self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB,
configCache.getCouchID())
return
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
# Set constants
self.couchDB = "config_test"
self.configURL = "RANDOM;;URL;;NAME"
self.configString = "This is a random string"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules =
["WMComponent.DBS3Buffer",
'WMCore.Agent.Database'],
useDefault = False)
self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
# Set up a config cache
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDB)
configCache.createUserGroup(groupname = "testGroup", username = '******')
self.testDir = self.testInit.generateWorkDir()
psetPath = os.path.join(self.testDir, "PSet.txt")
f = open(psetPath, 'w')
f.write(self.configString)
f.close()
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.save()
self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"],
self.couchDB,
configCache.getCouchID())
return
def upload_to_couch(cfg_name,
section_name,
user_name,
group_name,
test_mode=False,
url=None):
if test_mode:
return "00000000000000000"
if not os.path.exists(cfg_name):
raise RuntimeError("Error: Can't locate config file %s." % cfg_name)
# create a file with the ID inside to avoid multiple injections
oldID = cfg_name + '.couchID'
if os.path.exists(oldID):
f = open(oldID)
the_id = f.readline().replace('\n', '')
f.close()
print cfg_name, 'already uploaded with ID', the_id, 'from', oldID
return the_id
try:
loadedConfig = __loadConfig(cfg_name)
except:
#just try again !!
time.sleep(2)
loadedConfig = __loadConfig(cfg_name)
where = COUCH_DB_ADDRESS
if url:
where = url
configCache = ConfigCache(where, DATABASE_NAME)
configCache.createUserGroup(group_name, user_name)
configCache.addConfig(cfg_name)
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.setLabel(section_name)
configCache.setDescription(section_name)
configCache.save()
print "Added file to the config cache:"
print " DocID: %s" % configCache.document["_id"]
print " Revision: %s" % configCache.document["_rev"]
f = open(oldID, "w")
f.write(configCache.document["_id"])
f.close()
return configCache.document["_id"]
def upload_to_couch(cfg_name, section_name, user_name, group_name, test_mode=False, url=None):
if test_mode:
return "00000000000000000"
if not os.path.exists(cfg_name):
raise RuntimeError("Error: Can't locate config file %s." % cfg_name)
# create a file with the ID inside to avoid multiple injections
oldID = cfg_name + '.couchID'
if os.path.exists(oldID):
f = open(oldID)
the_id = f.readline().replace('\n','')
f.close()
print cfg_name, 'already uploaded with ID', the_id, 'from', oldID
return the_id
try:
loadedConfig = __loadConfig(cfg_name)
except:
#just try again !!
time.sleep(2)
loadedConfig = __loadConfig(cfg_name)
where = COUCH_DB_ADDRESS
if url:
where = url
configCache = ConfigCache(where, DATABASE_NAME)
configCache.createUserGroup(group_name, user_name)
configCache.addConfig(cfg_name)
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.setLabel(section_name)
configCache.setDescription(section_name)
configCache.save()
print "Added file to the config cache:"
print " DocID: %s" % configCache.document["_id"]
print " Revision: %s" % configCache.document["_rev"]
f = open(oldID,"w")
f.write(configCache.document["_id"])
f.close()
return configCache.document["_id"]
def upload(self, configModule, group, userDN,
label="Autogenerated by YAAT",
description="Autogenerated by YAAT",
url="http://se2.accre.vanderbilt.edu:5984",
database="wmagent_configcache"):
if url.endswith('/'):
raise RuntimeError, "URL shouldn't have a trailing slash"
configCache = CMSConfigCache(url, database)
configCache.createUserGroup(group, userDN)
tweaks = makeTweak(configModule.process).jsondictionary()
requiredFields = (('dataTier','USER'),('filterName',''))
for currentModule in tweaks['process']['outputModules_']:
if not 'dataset' in tweaks['process'][currentModule]:
tweaks['process'][currentModule]['dataset'] = {
"parameters_": [
"dataTier",
"filterName"
],
"filterName": "",
"dataTier": "USER"
}
continue
for reqKey, defaultValue in (('dataTier','USER'),('filterName','')):
if not reqKey in tweaks['process'][currentModule]['dataset']['parameters_']:
tweaks['process'][currentModule]['dataset']['parameters_'].append(reqKey)
tweaks['process'][currentModule]['dataset'][reqKey] = defaultValue
_, fileName = tempfile.mkstemp()
try:
filename = self.writeFile(configModule, fileName)
configCache.addConfig(filename)
configCache.setPSetTweaks(tweaks)
configCache.setLabel(label)
configCache.setDescription(description)
configCache.save()
finally:
if os.path.exists(fileName):
os.unlink(fileName)
targetUrl = "%s/%s/%s/configFile" % (url, database, configCache.document["_id"])
return targetUrl
def _configCacheId(self, label):
"""Return config cache id for given config label"""
key, cert = self.__class__.reqmgr['requests'].getKeyCert()
configCache = ConfigCache(self.__class__.endpoint + '/couchdb', 'reqmgr_config_cache', ckey = key, cert = cert)
try:
configCacheId = configCache.getIDFromLabel(label)
except:
configCacheId = None
if configCacheId:
return configCacheId
# The following will fail if FWCore.ParameterSet not in PYTHONPATH
from PSetTweaks.WMTweak import makeTweak
configCache.createUserGroup('test', 'test')
configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data', 'configs')
configCache.addConfig(os.path.join(configDir, label + '.py'))
configCache.setLabel(label)
configCache.setDescription(label)
modPath = imp.find_module(label, [configDir])
loadedConfig = imp.load_module(label, modPath[0], modPath[1], modPath[2])
configCache.setPSetTweaks(makeTweak(loadedConfig.process).jsondictionary())
configCache.save()
return configCache.getIDFromLabel(label)
