def testB_addingConfigsAndTweaks(self): """ _addingConfigsAndTweaks_ Test adding config files and tweak files """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." attach = "Hello, I am an attachment" configCache = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test') configCache.createUserGroup(groupname="testGroup", username='******') configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.attachments['attach1'] = attach psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt") configCache.addConfig(newConfig=psetPath, psetHash=None) configCache.setLabel("sample-label") configCache.setDescription("describe this config here") configCache.save() configString1 = configCache.getConfig() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test', id=configCache.getCouchID(), rev=configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) configString2 = configCache2.getConfig() self.assertEqual(configString1, configString2) self.assertEqual(configCache2.attachments.get('attach1', None), attach) configCache.delete() return
def testListAllConfigs(self): """ _testListAllConfigs_ Verify that the list all configs method works correctly. """ configCacheA = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test') configCacheA.createUserGroup(groupname="testGroup", username='******') configCacheA.setLabel("labelA") configCacheA.save() configCacheB = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test') configCacheB.createUserGroup(groupname="testGroup", username='******') configCacheB.setLabel("labelB") configCacheB.save() configs = configCacheA.listAllConfigsByLabel() self.assertEqual(len(configs.keys()), 2, "Error: There should be two configs") self.assertEqual(configs["labelA"], configCacheA.getCouchID(), "Error: Label A is wrong.") self.assertEqual(configs["labelB"], configCacheB.getCouchID(), "Error: Label B is wrong.") return
def testB_addingConfigsAndTweaks(self): """ _addingConfigsAndTweaks_ Test adding config files and tweak files """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." attach = "Hello, I am an attachment" configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache.createUserGroup(groupname = "testGroup", username = '******') configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.attachments['attach1'] = attach psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt") configCache.addConfig(newConfig = psetPath, psetHash = None) configCache.setLabel("sample-label") configCache.setDescription("describe this config here") configCache.save() configString1 = configCache.getConfig() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test', id = configCache.getCouchID(), rev = configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) configString2 = configCache2.getConfig() self.assertEqual(configString1, configString2) self.assertEqual(configCache2.attachments.get('attach1', None), attach) configCache.delete() return
def testListAllConfigs(self): """ _testListAllConfigs_ Verify that the list all configs method works correctly. """ configCacheA = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCacheA.createUserGroup(groupname = "testGroup", username = '******') configCacheA.setLabel("labelA") configCacheA.save() configCacheB = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCacheB.createUserGroup(groupname = "testGroup", username = '******') configCacheB.setLabel("labelB") configCacheB.save() configs = configCacheA.listAllConfigsByLabel() self.assertEqual(len(configs.keys()), 2, "Error: There should be two configs") self.assertEqual(configs["labelA"], configCacheA.getCouchID(), "Error: Label A is wrong.") self.assertEqual(configs["labelB"], configCacheB.getCouchID(), "Error: Label B is wrong.") return
def createConfig(self, bad=False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = { "process": { "outputModules_": ["ThisIsAName"], "ThisIsAName": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}}, } } BadTweak = { "process": { "outputModules_": ["ThisIsAName1", "ThisIsAName2"], "ThisIsAName1": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}}, "ThisIsAName2": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}}, } } configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName) configCache.createUserGroup(groupname="testGroup", username="******") if bad: configCache.setPSetTweaks(PSetTweak=BadTweak) else: configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.save() return configCache.getCouchID()
def createConfig(couchDBName): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = { 'process': { 'outputModules_': ['RECOoutput', 'ALCARECOoutput'], 'RECOoutput': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } }, 'ALCARECOoutput': { 'dataset': { 'dataTier': 'ALCARECO', 'filterName': 'AlcaFilter' } } } } configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=couchDBName) configCache.createUserGroup(groupname="testGroup", username='******') configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.save() return configCache.getCouchID()
def createConfig(self, bad = False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = {'process': {'outputModules_': ['ThisIsAName'], 'ThisIsAName': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}, 'ThisIsAName2': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName) configCache.createUserGroup(groupname = "testGroup", username = '******') if bad: configCache.setPSetTweaks(PSetTweak = BadTweak) else: configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.save() return configCache.getCouchID()
def testD_LoadConfigCache(self): """ _LoadConfigCache_ Actually load the config cache using plain .load() Tests to make sure that if we pass in an id field it gets used to load configs """ configCache = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test') configCache.createUserGroup(groupname="testGroup", username='******') configCache.setLabel("labelA") configCache.save() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test', id=configCache.getCouchID(), rev=configCache.getCouchRev()) configCache2.load() self.assertEqual(configCache2.document['owner'], { 'group': 'testGroup', 'user': '******' }) self.assertEqual(configCache2.document['description'], { 'config_desc': None, 'config_label': 'labelA' }) return
def setUp(self): """ _setUp_ setUp function for unittest """ # Set constants self.couchDB = "config_test" self.configURL = "RANDOM;;URL;;NAME" self.configString = "This is a random string" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema( customModules=["WMComponent.DBS3Buffer", 'WMCore.Agent.Database'], useDefault=False) self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache") myThread = threading.currentThread() self.bufferFactory = DAOFactory( package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi) self.buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.bufferFactory( classname="DBSBufferFiles.AddLocation") locationAction.execute(siteName="se1.cern.ch") locationAction.execute(siteName="se1.fnal.gov") locationAction.execute(siteName="malpaquet") # Set heartbeat self.componentName = 'JobSubmitter' self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() # Set up a config cache configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDB) configCache.createUserGroup(groupname="testGroup", username='******') self.testDir = self.testInit.generateWorkDir() psetPath = os.path.join(self.testDir, "PSet.txt") f = open(psetPath, 'w') f.write(self.configString) f.close() configCache.addConfig(newConfig=psetPath, psetHash=None) configCache.save() self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB, configCache.getCouchID()) return
def setUp(self): """ _setUp_ setUp function for unittest """ # Set constants self.couchDB = "config_test" self.configURL = "RANDOM;;URL;;NAME" self.configString = "This is a random string" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer", 'WMCore.Agent.Database'], useDefault = False) self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache") myThread = threading.currentThread() self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation") locationAction.execute(siteName = "se1.cern.ch") locationAction.execute(siteName = "se1.fnal.gov") locationAction.execute(siteName = "malpaquet") # Set heartbeat self.componentName = 'JobSubmitter' self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() # Set up a config cache configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDB) configCache.createUserGroup(groupname = "testGroup", username = '******') self.testDir = self.testInit.generateWorkDir() psetPath = os.path.join(self.testDir, "PSet.txt") f = open(psetPath, 'w') f.write(self.configString) f.close() configCache.addConfig(newConfig = psetPath, psetHash = None) configCache.save() self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB, configCache.getCouchID()) return
def testA_basicConfig(self): """ _basicConfig_ Basic configCache stuff. """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." configCache = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test') configCache.createUserGroup(groupname="testGroup", username='******') configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.save() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test', id=configCache.getCouchID(), rev=configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) self.assertEqual(configCache2.getPSetTweaks(), PSetTweak) configCache2.delete() configCache3 = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test', id=configCache.getCouchID(), rev=configCache.getCouchRev()) testFlag = False # It should fail to load deleted documents try: configCache3.loadByID(configCache.getCouchID()) except ConfigCacheException: testFlag = True self.assertTrue(testFlag) return
def testA_basicConfig(self): """ _basicConfig_ Basic configCache stuff. """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache.createUserGroup(groupname = "testGroup", username = '******') configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.save() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test', id = configCache.getCouchID(), rev = configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) self.assertEqual(configCache2.getPSetTweaks(), PSetTweak) configCache2.delete() configCache3 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test', id = configCache.getCouchID(), rev = configCache.getCouchRev()) testFlag = False # It should fail to load deleted documents try: configCache3.loadByID(configCache.getCouchID()) except ConfigCacheException: testFlag = True self.assertTrue(testFlag) return
def createConfig(couchDBName): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = {'process': {'outputModules_': ['RECOoutput', 'ALCARECOoutput'], 'RECOoutput': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}, 'ALCARECOoutput': {'dataset': {'dataTier': 'ALCARECO', 'filterName': 'AlcaFilter'}}}} configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = couchDBName) configCache.createUserGroup(groupname = "testGroup", username = '******') configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.save() return configCache.getCouchID()
def testD_LoadConfigCache(self): """ _LoadConfigCache_ Actually load the config cache using plain .load() Tests to make sure that if we pass in an id field it gets used to load configs """ configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache.createUserGroup(groupname = "testGroup", username = '******') configCache.setLabel("labelA") configCache.save() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test', id = configCache.getCouchID(), rev = configCache.getCouchRev()) configCache2.load() self.assertEqual(configCache2.document['owner'], {'group': 'testGroup', 'user': '******'}) self.assertEqual(configCache2.document['description'], {'config_desc': None, 'config_label': 'labelA'}) return