def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def testListAllConfigs(self):
"""
_testListAllConfigs_
Verify that the list all configs method works correctly.
"""
configCacheA = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCacheA.createUserGroup(groupname="testGroup", username='******')
configCacheA.setLabel("labelA")
configCacheA.save()
configCacheB = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCacheB.createUserGroup(groupname="testGroup", username='******')
configCacheB.setLabel("labelB")
configCacheB.save()
configs = configCacheA.listAllConfigsByLabel()
self.assertEqual(len(configs.keys()), 2,
"Error: There should be two configs")
self.assertEqual(configs["labelA"], configCacheA.getCouchID(),
"Error: Label A is wrong.")
self.assertEqual(configs["labelB"], configCacheB.getCouchID(),
"Error: Label B is wrong.")
return
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def testListAllConfigs(self):
"""
_testListAllConfigs_
Verify that the list all configs method works correctly.
"""
configCacheA = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCacheA.createUserGroup(groupname = "testGroup", username = '******')
configCacheA.setLabel("labelA")
configCacheA.save()
configCacheB = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCacheB.createUserGroup(groupname = "testGroup", username = '******')
configCacheB.setLabel("labelB")
configCacheB.save()
configs = configCacheA.listAllConfigsByLabel()
self.assertEqual(len(configs.keys()), 2,
"Error: There should be two configs")
self.assertEqual(configs["labelA"], configCacheA.getCouchID(),
"Error: Label A is wrong.")
self.assertEqual(configs["labelB"], configCacheB.getCouchID(),
"Error: Label B is wrong.")
return
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
"process": {
"outputModules_": ["ThisIsAName"],
"ThisIsAName": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
}
}
BadTweak = {
"process": {
"outputModules_": ["ThisIsAName1", "ThisIsAName2"],
"ThisIsAName1": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
"ThisIsAName2": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
}
}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username="******")
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
def createConfig(couchDBName):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
'process': {
'outputModules_': ['RECOoutput', 'ALCARECOoutput'],
'RECOoutput': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
},
'ALCARECOoutput': {
'dataset': {
'dataTier': 'ALCARECO',
'filterName': 'AlcaFilter'
}
}
}
}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=couchDBName)
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
def testD_LoadConfigCache(self):
"""
_LoadConfigCache_
Actually load the config cache using plain .load()
Tests to make sure that if we pass in an id field it gets used to load configs
"""
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setLabel("labelA")
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.load()
self.assertEqual(configCache2.document['owner'], {
'group': 'testGroup',
'user': '******'
})
self.assertEqual(configCache2.document['description'], {
'config_desc': None,
'config_label': 'labelA'
})
return
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
# Set constants
self.couchDB = "config_test"
self.configURL = "RANDOM;;URL;;NAME"
self.configString = "This is a random string"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMComponent.DBS3Buffer", 'WMCore.Agent.Database'],
useDefault=False)
self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.bufferFactory(
classname="DBSBufferFiles.AddLocation")
locationAction.execute(siteName="se1.cern.ch")
locationAction.execute(siteName="se1.fnal.gov")
locationAction.execute(siteName="malpaquet")
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
# Set up a config cache
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDB)
configCache.createUserGroup(groupname="testGroup", username='******')
self.testDir = self.testInit.generateWorkDir()
psetPath = os.path.join(self.testDir, "PSet.txt")
f = open(psetPath, 'w')
f.write(self.configString)
f.close()
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.save()
self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB,
configCache.getCouchID())
return
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
# Set constants
self.couchDB = "config_test"
self.configURL = "RANDOM;;URL;;NAME"
self.configString = "This is a random string"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules =
["WMComponent.DBS3Buffer",
'WMCore.Agent.Database'],
useDefault = False)
self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
# Set up a config cache
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDB)
configCache.createUserGroup(groupname = "testGroup", username = '******')
self.testDir = self.testInit.generateWorkDir()
psetPath = os.path.join(self.testDir, "PSet.txt")
f = open(psetPath, 'w')
f.write(self.configString)
f.close()
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.save()
self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"],
self.couchDB,
configCache.getCouchID())
return
def testA_basicConfig(self):
"""
_basicConfig_
Basic configCache stuff.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
self.assertEqual(configCache2.getPSetTweaks(), PSetTweak)
configCache2.delete()
configCache3 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
testFlag = False
# It should fail to load deleted documents
try:
configCache3.loadByID(configCache.getCouchID())
except ConfigCacheException:
testFlag = True
self.assertTrue(testFlag)
return
def testA_basicConfig(self):
"""
_basicConfig_
Basic configCache stuff.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
self.assertEqual(configCache2.getPSetTweaks(), PSetTweak)
configCache2.delete()
configCache3 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
testFlag = False
# It should fail to load deleted documents
try:
configCache3.loadByID(configCache.getCouchID())
except ConfigCacheException:
testFlag = True
self.assertTrue(testFlag)
return
def createConfig(couchDBName):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['RECOoutput', 'ALCARECOoutput'],
'RECOoutput': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ALCARECOoutput': {'dataset': {'dataTier': 'ALCARECO',
'filterName': 'AlcaFilter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
def testD_LoadConfigCache(self):
"""
_LoadConfigCache_
Actually load the config cache using plain .load()
Tests to make sure that if we pass in an id field it gets used to load configs
"""
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setLabel("labelA")
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.load()
self.assertEqual(configCache2.document['owner'],
{'group': 'testGroup', 'user': '******'})
self.assertEqual(configCache2.document['description'],
{'config_desc': None, 'config_label': 'labelA'})
return
