class RequestManagerTest(RESTBaseUnitTest): """ Test RequestMgr Service client It will start RequestMgr RESTService Server DB is whatever env is set This checks whether DS call makes without error and return the results. This test only test service call returns without error. The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py """ def initialize(self): self.couchDBName = "reqmgr_t_0" self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') dbUrl = os.environ.get("DATABASE", None) self.config.setDBUrl(dbUrl) self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(8888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def setUp(self): RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) self.jsonSender.put('user/[email protected]') self.jsonSender.put('group/PeopleLikeMe') self.jsonSender.put('group/PeopleLikeMe/me') self.jsonSender.put('version/CMSSW_3_5_8/slc5_ia32_gcc434') schema = ReReco.getTestArguments() schema['RequestName'] = 'TestReReco' schema['RequestType'] = 'ReReco' schema['CmsPath'] = "/uscmst1/prod/sw/cms" schema['Requestor'] = '%s' % "me" schema['Group'] = '%s' % "PeopleLikeMe" schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha'] schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta'] schema['Campaign'] = 'MyTestCampaign' try: r = self.jsonSender.put('request/' + schema['RequestName'], schema) except Exception, ex: print "Exception during set up, investigate exception instance attributes:" print dir(ex) return self.requestName = r[0]['RequestName']
class RequestManagerTest(RESTBaseUnitTest): """ Test RequestMgr Service client It will start RequestMgr RESTService Server DB is whatever env is set This checks whether DS call makes without error and return the results. This test only test service call returns without error. The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py """ def initialize(self): self.couchDBName = "reqmgr_t_0" self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') dbUrl = os.environ.get("DATABASE", None) self.config.setDBUrl(dbUrl) self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName=self.couchDBName) self.config.setPort(8888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def setUp(self): RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") # logging stuff from TestInit is broken, setting myself l = logging.getLogger() l.setLevel(logging.DEBUG) self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['ConfigCacheID'] = self.createConfig() schema['CouchDBName'] = self.couchDBName try: r = self.jsonSender.put('request/' + schema['RequestName'], schema) self.requestName = r[0]['RequestName'] except Exception, ex: print "Exception during set up, reason: %s" % ex raise ex
class RequestManagerTest(RESTBaseUnitTest): """ Test RequestMgr Service client It will start RequestMgr RESTService Server DB is whatever env is set This checks whether DS call makes without error and return the results. This test only test service call returns without error. The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py """ def initialize(self): self.couchDBName = "reqmgr_t_0" self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') dbUrl = os.environ.get("DATABASE", None) self.config.setDBUrl(dbUrl) self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(8888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def setUp(self): RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") # logging stuff from TestInit is broken, setting myself l = logging.getLogger() l.setLevel(logging.DEBUG) self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) try: r = self.jsonSender.put('request/' + schema['RequestName'], schema) except Exception, ex: print "Exception during set up, reason: %s" % ex return self.requestName = r[0]['RequestName']
class ReqMgrWorkloadTest(RESTBaseUnitTest): """ Test that sets up and checks the validations of the various main WMSpec.StdSpecs This is mostly a simple set of tests which can be very repetitive. """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(12888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def createConfig(self, bad = False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = {'process': {'outputModules_': ['ThisIsAName'], 'ThisIsAName': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}, 'ThisIsAName2': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName) configCache.createUserGroup(groupname = "testGroup", username = '******') if bad: configCache.setPSetTweaks(PSetTweak = BadTweak) else: configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.save() return configCache.getCouchID() def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload @attr('integration') def testA_makeReRecoWorkload(self): """ _makeReRecoWorkload_ Check that you can make a basic ReReco workload """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) del schema['GlobalTag'] raises = False try: self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Missing required " "field 'GlobalTag' in workload validation!" in ex.result) self.assertTrue(raises) schema = utils.getSchema(groupName = groupName, userName = userName) schema['InputDataset'] = '/Nothing' raises = False try: self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) print ex.result
class RequestManagerTest(RESTBaseUnitTest): """ Test RequestMgr Service client It will start RequestMgr RESTService Server DB is whatever env is set This checks whether DS call makes without error and return the results. This test only test service call returns without error. The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py """ def initialize(self): self.couchDBName = "reqmgr_t_0" self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') dbUrl = os.environ.get("DATABASE", None) self.config.setDBUrl(dbUrl) self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName=self.couchDBName) self.config.setPort(8899) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def setUp(self): RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") # logging stuff from TestInit is broken, setting myself l = logging.getLogger() l.setLevel(logging.DEBUG) self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['ConfigCacheID'] = self.createConfig() schema['CouchDBName'] = self.couchDBName schema['CouchWorkloadDBName'] = self.couchDBName try: r = self.jsonSender.put('request', schema) try: self.requestName = r[0]['RequestName'] except: self.requestName = r[0].values()[0]['RequestName'] except Exception as ex: msg = traceback.format_exc() print "Exception during set up, reason: %s" % msg raise ex def tearDown(self): self.config.deleteWorkloadCache() RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def createConfig(self, bad=False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = { 'process': { 'outputModules_': ['ThisIsAName'], 'ThisIsAName': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } } } } BadTweak = { 'process': { 'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } }, 'ThisIsAName2': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } } } } configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName) configCache.createUserGroup(groupname="testGroup", username='******') if bad: configCache.setPSetTweaks(PSetTweak=BadTweak) else: configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.save() return configCache.getCouchID() @attr("integration") def testA_RequestManagerService(self): requestName = self.requestName request = self.reqService.getRequest(requestName) # minimal test : it's return type and the some value inside self.assertEqual(type(request), dict) self.assertTrue(len(request) > 0) # Test putTeam self.reqService.putTeam("team_usa") self.assertTrue('team_usa' in self.jsonSender.get('team')[0]) self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName)) request = self.reqService.getAssignment(teamName="team_usa") self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) request = self.reqService.getAssignment(request=requestName) self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) self.reqService.sendMessage(requestName, "error") self.reqService.putWorkQueue(requestName, "http://test_url") self.reqService.reportRequestProgress(requestName, percent_complete=100, percent_success=90) self.reqService.updateRequestStatus(requestName, "running-open")
class AssignTest(RESTBaseUnitTest): """ _AssignTest_ """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self, initRoot=False) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.Assign') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName=self.couchDBName) self.config.setPort(12888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload def testA_SiteWhitelist(self): """ _SiteWhitelist_ Test to see if we can get the siteWhitelist to work properly. """ secconfig = getattr(self.config, "SecurityModule") cherrypy.server.environment = 'test' cherrypy.tools.secmodv2 = NullAuth(secconfig) self.config.UnitTests.views.active.rest.templates = 'templateDir' self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot' self.config.UnitTests.views.active.rest.wildcardKeys = { 'T1*': 'T1_*', 'T2*': 'T2_*', 'T3*': 'T3_*', 'US*': '_US_' } from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign assign = Assign(config=self.config.UnitTests.views.active.rest, noSiteDB=True) siteList = [ 'T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT' ] self.assertEqual(assign.sites, []) assign.sites.extend(siteList) Utilities.addSiteWildcards(assign.wildcardKeys, assign.sites, assign.wildcardSites) for s in siteList: self.assertTrue(s in assign.sites) self.assertTrue('T1*' in assign.sites) self.assertTrue('T2*' in assign.sites) self.assertFalse('T3*' in assign.sites) self.assertTrue('US*' in assign.sites) self.assertEqual(assign.wildcardSites['T1*'], ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL']) self.assertEqual(assign.wildcardSites['T2*'], ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']) self.assertEqual( assign.wildcardSites['US*'], ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
class ReqMgrPriorityTest(RESTBaseUnitTest): """ _ReqMgrPriorityTest_ Basic test for setting the priority in ReqMgr Services """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName=self.couchDBName) self.config.setPort(8888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload def changeStatusAndCheck(self, requestName, statusName): """ _changeStatusAndCheck_ Change the status of a request and make sure that the request actually did it. """ self.jsonSender.put('request/%s?status=%s' % (requestName, statusName)) result = self.jsonSender.get('request/%s' % requestName) self.assertEqual(result[0]['RequestStatus'], statusName) def testA_RequestPriority(self): """ _priorityChanges_ Do some fairly standard priority changes to the Request and see how things react """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), 0) # Reset user, group priorities to 0 self.jsonSender.post('user/%s?priority=0' % userName) self.jsonSender.post('group/%s?priority=0' % groupName) # Set priority == 5 priority = 5 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) # Set priority == 100 priority = 100 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) # Set priority == -1 priority = -1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) # Let's move the request around a bit self.changeStatusAndCheck(requestName=requestName, statusName='testing-approved') self.changeStatusAndCheck(requestName=requestName, statusName='testing') self.changeStatusAndCheck(requestName=requestName, statusName='tested') self.changeStatusAndCheck(requestName=requestName, statusName='assignment-approved') self.jsonSender.put( urllib.quote('assignment/%s/%s' % (teamName, requestName))) # Set priority == 99 priority = 99 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) def testB_InvalidPriority(self): """ _InvalidPriority_ Put in a bunch of invalid values for priorities and see what the code makes of them. """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] raises = False try: priority = sys.maxint + 1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Priority must have abs() less then MAXINT!" in ex.result) self.assertTrue(raises) raises = False try: priority = -1 - sys.maxint self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) print ex.result self.assertTrue( "Priority must have abs() less then MAXINT!" in ex.result)
class AssignTest(RESTBaseUnitTest): def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self, initRoot = False) self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.Assign') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(12888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload def testA_SiteWhitelist(self): """ _SiteWhitelist_ Test to see if we can get the siteWhitelist to work properly. """ secconfig = getattr(self.config, "SecurityModule") cherrypy.server.environment = 'test' cherrypy.tools.secmodv2 = NullAuth(secconfig) self.config.UnitTests.views.active.rest.templates = 'templateDir' self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot' self.config.UnitTests.views.active.rest.wildcardKeys = {'T1*': 'T1_*', 'T2*': 'T2_*', 'T3*': 'T3_*', 'US*': '_US_'} from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign assign = Assign(config = self.config.UnitTests.views.active.rest, noSiteDB = True) siteList = ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'] self.assertEqual(assign.sites, []) assign.sites.extend(siteList) Utilities.addSiteWildcards(assign.wildcardKeys,assign.sites,assign.wildcardSites) for s in siteList: self.assertTrue(s in assign.sites) self.assertTrue('T1*' in assign.sites) self.assertTrue('T2*' in assign.sites) self.assertFalse('T3*' in assign.sites) self.assertTrue('US*' in assign.sites) self.assertEqual(assign.wildcardSites['T1*'], ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL']) self.assertEqual(assign.wildcardSites['T2*'], ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']) self.assertEqual(assign.wildcardSites['US*'], ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
class AdminTest(RESTBaseUnitTest): """ _AdminTest_ Test for the lower-level DB code in the admin section """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) return def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(12888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] return def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() return def testA_SoftwareManagement(self): """ _SoftwareManagement_ Test the SoftwareManagement code """ self.assertEqual(SoftwareManagement.listSoftware(), {}) softwareVersions = ReqMgrWebTools.allScramArchsAndVersions() ReqMgrWebTools.updateScramArchsAndCMSSWVersions() result = SoftwareManagement.listSoftware() for scramArch in result.keys(): self.assertEqual(set(result[scramArch]), set(softwareVersions[scramArch])) # Now for each scramArch insert a blank set # Because of the way that updateSoftware works, this interprets a blank list # as telling it that no softwareVersions are available. # It deletes every software version it is not handed, so it should give nothing out. for scramArch in result.keys(): SoftwareManagement.updateSoftware(softwareNames = [], scramArch = scramArch) self.assertEqual(SoftwareManagement.listSoftware(), {}) from WMCore.HTTPFrontEnd.RequestManager import Admin setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase) self.config.section_('templates') self.config.section_('html') admin = Admin.Admin(self.config) ReqMgrWebTools.updateScramArchsAndCMSSWVersions() self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs()) return
class ReqMgrWorkloadTest(RESTBaseUnitTest): """ Test that sets up and checks the validations of the various main WMSpec.StdSpecs This is mostly a simple set of tests which can be very repetitive. """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName=self.couchDBName) self.config.setPort(12888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def createConfig(self, bad=False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = { 'process': { 'outputModules_': ['ThisIsAName'], 'ThisIsAName': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } } } } BadTweak = { 'process': { 'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } }, 'ThisIsAName2': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } } } } configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName) configCache.createUserGroup(groupname="testGroup", username='******') if bad: configCache.setPSetTweaks(PSetTweak=BadTweak) else: configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.save() return configCache.getCouchID() def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload @attr('integration') def testA_makeReRecoWorkload(self): """ _makeReRecoWorkload_ Check that you can make a basic ReReco workload """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) del schema['GlobalTag'] configID = self.createConfig(bad=False) schema['ConfigCacheID'] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName raises = False try: self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Argument GlobalTag is required." in ex.result) self.assertTrue(raises) schema = utils.getSchema(groupName=groupName, userName=userName) schema['InputDataset'] = '/Nothing' raises = False try: self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue(raises) schema = utils.getSchema(groupName=groupName, userName=userName) raises = False del schema['ConfigCacheID'] try: self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Create request failed, 'ConfigCacheID'" in ex.result) self.assertTrue(raises) configID = self.createConfig(bad=True) schema = utils.getSchema(groupName=groupName, userName=userName) schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName raises = False try: self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: " "Duplicate dataTier/filterName combination" in ex.result) self.assertTrue(raises) schema = utils.getSchema(groupName=groupName, userName=userName) configID = self.createConfig(bad=False) schema['ConfigCacheID'] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName try: result = self.jsonSender.put('request', schema) except Exception as ex: raise self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) @attr('integration') def testB_Analysis(self): """ _Analysis_ Test Analysis workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "Analysis" try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue(raises) # Put the right things in the schema # And watch it fail, because we can't open a secure connection to CERN # in a unittest well schema['RequestorDN'] = 'SomeDN' #result = self.jsonSender.put('request/testRequest', schema) #requestName = result[0]['RequestName'] #result = self.jsonSender.get('request/%s' % requestName) #request = result[0] #self.assertEqual(request['CMSSWVersion'], CMSSWVersion) #self.assertEqual(request['Group'], groupName) #self.assertEqual(request['Requestor'], userName) @attr('integration') def testD_ReDigi(self): """ _ReDigi_ Test ReDigi workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "ReDigi" try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Create request failed, 'StepOneConfigCacheID'" in ex.result) self.assertTrue(raises) schema["StepOneConfigCacheID"] = "fakeID" schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) configID = self.createConfig() schema["StepOneConfigCacheID"] = configID result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['SizePerEvent'], 512) self.assertEqual(request['RequestNumEvents'], 0) self.assertEqual(request['RequestSizeFiles'], 0) @attr('integration') def testE_StoreResults(self): """ _StoreResults_ Test StoreResults workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "StoreResults" schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Argument CmsPath is required." in ex.result) self.assertTrue(raises) schema['DbsUrl'] = 'http://fake.dbs.url/dbs' schema['CmsPath'] = '/fake/tmp/path' schema['AcquisitionEra'] = 'era' schema['ProcessingVersion'] = 1 result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['DbsUrl'], schema['DbsUrl']) @attr('integration') def testF_TaskChain(self): """ _TaskChain_ Test the monstrous TaskChain workflow This will be a long one NOTE: This test is so complicated that all I do is take code from TaskChain_t and make sure it still produces and actual request """ couchServer = CouchServer(os.environ["COUCHURL"]) configDatabase = couchServer.connectDatabase(self.couchDBName) generatorDoc = makeGeneratorConfig(configDatabase) processorDocs = makeProcessingConfigs(configDatabase) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getSchema(userName=userName) schema["CouchURL"] = os.environ["COUCHURL"] schema["CouchDBName"] = self.couchDBName schema["CouchWorkloadDBName"] = self.couchDBName schema["SiteWhitelist"] = ["T1_CH_CERN", "T1_US_FNAL"] schema["TaskChain"] = 5 chains = { "Task1": { "TaskName": "GenSim", "ConfigCacheID": generatorDoc, "SplittingAlgo": "EventBased", "EventsPerJob": 250, "RequestNumEvents": 10000, "PrimaryDataset": "RelValTTBar" }, "Task2": { "TaskName": "DigiHLT", "InputTask": "GenSim", "InputFromOutputModule": "writeGENSIM", "ConfigCacheID": processorDocs['DigiHLT'], "SplittingAlgo": "FileBased" }, "Task3": { "TaskName": "Reco", "InputTask": "DigiHLT", "InputFromOutputModule": "writeRAWDIGI", "ConfigCacheID": processorDocs['Reco'], "SplittingAlgo": "FileBased" }, "Task4": { "TaskName": "ALCAReco", "InputTask": "Reco", "InputFromOutputModule": "writeALCA", "ConfigCacheID": processorDocs['ALCAReco'], "SplittingAlgo": "FileBased" }, "Task5": { "TaskName": "Skims", "InputTask": "Reco", "InputFromOutputModule": "writeRECO", "ConfigCacheID": processorDocs['Skims'], "SplittingAlgo": "FileBased", "FilesPerJob": 10 } } schema.update(chains) args = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema.update(args) # this is necessary and after all updates to the schema are made, # otherwise this item will get overwritten schema['RequestType'] = "TaskChain" schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) workload = self.loadWorkload(requestName) self.assertEqual(workload.data.request.schema.Task1["EventsPerJob"], 250) @attr('integration') def testG_MonteCarloFromGEN(self): """ _MonteCarloFromGEN_ Test MonteCarloFromGEN workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "MonteCarloFromGEN" try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) schema["ConfigCacheID"] = "fakeID" schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) configID = self.createConfig() schema["ConfigCacheID"] = configID result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['SizePerEvent'], 512) self.assertEqual(request['RequestNumEvents'], 0) self.assertEqual(request['RequestSizeFiles'], 0) def testH_MonteCarlo(self): """ _MonteCarlo_ Test MonteCarlo workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "MonteCarlo" schema['DbsUrl'] = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' # Set some versions schema['ProcessingVersion'] = '2012' schema['AcquisitionEra'] = 'ae2012' try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) schema["ConfigCacheID"] = "fakeID" schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName schema["PrimaryDataset"] = "ReallyFake" schema["RequestNumEvents"] = 100 try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue(raises) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["FilterEfficiency"] = -0.5 try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) # until exception handling is redone (New REST API) #self.assertTrue("Negative filter efficiency for MC workflow" in ex.result) self.assertTrue(raises) schema["FilterEfficiency"] = 1.0 result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['DbsUrl'], schema['DbsUrl']) self.assertEqual(request['SizePerEvent'], 512) self.assertEqual(request['RequestNumEvents'], 100) self.assertEqual(request['RequestSizeFiles'], 0) def testJ_Resubmission(self): """ _Resubmission_ Test Resubmission """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "ReReco" configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] # user, group schema already set up schema = utils.getSchema(groupName=groupName, userName=userName) schema['RequestType'] = "Resubmission" schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Validation failed: InitialTaskPath is mendatory" in ex.result) self.assertTrue(raises) schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName schema["ACDCServer"] = os.environ.get("COUCHURL") schema["ACDCDatabase"] = self.couchDBName schema["CollectionName"] = "SomeOtherName" # Here we just make sure that real result goes through result = self.jsonSender.put('request', schema) resubmitName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % resubmitName) couchServer = CouchServer(self.testInit.couchUrl) reqmgrCouch = couchServer.connectDatabase(self.couchDBName) result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests', {}, [requestName])['rows'] self.assertEqual(len(result), 1) self.assertEqual(result[0]['key'], requestName) self.assertEqual(result[0]['id'], resubmitName)
class ReqMgrWorkloadTest(RESTBaseUnitTest): """ Test that sets up and checks the validations of the various main WMSpec.StdSpecs This is mostly a simple set of tests which can be very repetitive. """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName=self.couchDBName) self.config.setPort(12888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def createConfig(self, bad=False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = { 'process': { 'outputModules_': ['ThisIsAName'], 'ThisIsAName': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } } } } BadTweak = { 'process': { 'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } }, 'ThisIsAName2': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } } } } configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName) configCache.createUserGroup(groupname="testGroup", username='******') if bad: configCache.setPSetTweaks(PSetTweak=BadTweak) else: configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.save() return configCache.getCouchID() def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload @attr('integration') def testA_makeReRecoWorkload(self): """ _makeReRecoWorkload_ Check that you can make a basic ReReco workload """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) del schema['GlobalTag'] configID = self.createConfig(bad=False) schema['ConfigCacheID'] = configID raises = False try: self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Argument GlobalTag is required." in ex.result) self.assertTrue(raises) schema = utils.getSchema(groupName=groupName, userName=userName) schema['InputDataset'] = '/Nothing' raises = False try: self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) print ex.result
class AssignTest(RESTBaseUnitTest): """ _AssignTest_ """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self, initRoot = False) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) return def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.Assign') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(12888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] return def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() return def setupSchema(self, groupName = 'PeopleLikeMe', userName = '******', teamName = 'White Sox', CMSSWVersion = 'CMSSW_3_5_8'): """ _setupSchema_ Set up a test schema so that we can run a test request. Standardization! """ self.jsonSender.put('user/%[email protected]' % userName) self.jsonSender.put('group/%s' % groupName) self.jsonSender.put('group/%s/%s' % (groupName, userName)) self.jsonSender.put(urllib.quote('team/%s' % teamName)) self.jsonSender.put('version/%s' % CMSSWVersion) schema = ReReco.getTestArguments() schema['RequestName'] = 'TestReReco' schema['RequestType'] = 'ReReco' schema['CmsPath'] = "/uscmst1/prod/sw/cms" schema['Requestor'] = '%s' % userName schema['Group'] = '%s' % groupName return schema def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload def testA_SiteWhitelist(self): """ _SiteWhitelist_ Test to see if we can get the siteWhitelist to work properly. """ secconfig = getattr(self.config, "SecurityModule") cherrypy.server.environment = 'test' cherrypy.tools.secmodv2 = NullAuth(secconfig) self.config.UnitTests.views.active.rest.templates = 'templateDir' self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot' self.config.UnitTests.views.active.rest.wildcardKeys = {'T1*': 'T1_*', 'T2*': 'T2_*', 'T3*': 'T3_*', 'US*': '_US_'} from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign assign = Assign(config = self.config.UnitTests.views.active.rest, noSiteDB = True) siteList = ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'] self.assertEqual(assign.sites, []) assign.sites.extend(siteList) Utilities.addSiteWildcards(assign.wildcardKeys,assign.sites,assign.wildcardSites) for s in siteList: self.assertTrue(s in assign.sites) self.assertTrue('T1*' in assign.sites) self.assertTrue('T2*' in assign.sites) self.assertFalse('T3*' in assign.sites) self.assertTrue('US*' in assign.sites) self.assertEqual(assign.wildcardSites['T1*'], ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL']) self.assertEqual(assign.wildcardSites['T2*'], ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']) self.assertEqual(assign.wildcardSites['US*'], ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']) return
class ReqMgrPriorityTest(RESTBaseUnitTest): """ Basic test for setting the priority in ReqMgr Services """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") self.testInit.setupCouch("%s_acdc" % self.couchDBName, "ACDC") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig("WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel") self.config.setFormatter("WMCore.WebTools.RESTFormatter") self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName=self.couchDBName) self.config.setPort(8888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = "%s/%s/%s/spec" % (os.environ["COUCHURL"], self.couchDBName, requestName) workload.load(url) return workload def createConfig(self, bad=False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = { "process": { "outputModules_": ["ThisIsAName"], "ThisIsAName": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}}, } } BadTweak = { "process": { "outputModules_": ["ThisIsAName1", "ThisIsAName2"], "ThisIsAName1": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}}, "ThisIsAName2": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}}, } } configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName) configCache.createUserGroup(groupname="testGroup", username="******") if bad: configCache.setPSetTweaks(PSetTweak=BadTweak) else: configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.save() return configCache.getCouchID() def changeStatusAndCheck(self, requestName, statusName): """ _changeStatusAndCheck_ Change the status of a request and make sure that the request actually did it. """ self.jsonSender.put("request/%s?status=%s" % (requestName, statusName)) result = self.jsonSender.get("request/%s" % requestName) self.assertEqual(result[0]["RequestStatus"], statusName) def testA_RequestPriority(self): """ _priorityChanges_ Do some fairly standard priority changes to the Request and see how things react """ userName = "******" groupName = "Li" teamName = "Tang" schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put("request/testRequest", schema) self.assertEqual(result[1], 200) requestName = result[0]["RequestName"] workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), 0) # Set priority == 5 priority = 5 self.jsonSender.put("request/%s?priority=%s" % (requestName, priority)) request = self.jsonSender.get("request/%s" % requestName)[0] self.assertEqual(request["RequestPriority"], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) # Set priority == 100 priority = 100 self.jsonSender.put("request/%s?priority=%s" % (requestName, priority)) request = self.jsonSender.get("request/%s" % requestName)[0] self.assertEqual(request["RequestPriority"], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) # Set priority == -1 priority = -1 self.jsonSender.put("request/%s?priority=%s" % (requestName, priority)) request = self.jsonSender.get("request/%s" % requestName)[0] self.assertEqual(request["RequestPriority"], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) # Let's move the request around a bit self.changeStatusAndCheck(requestName=requestName, statusName="testing-approved") self.changeStatusAndCheck(requestName=requestName, statusName="testing") self.changeStatusAndCheck(requestName=requestName, statusName="tested") self.changeStatusAndCheck(requestName=requestName, statusName="assignment-approved") self.jsonSender.put(urllib.quote("assignment/%s/%s" % (teamName, requestName))) # Set priority == 99 priority = 99 self.jsonSender.put("request/%s?priority=%s" % (requestName, priority)) request = self.jsonSender.get("request/%s" % requestName)[0] self.assertEqual(request["RequestPriority"], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) def testB_InvalidPriority(self): """ _InvalidPriority_ Put in a bunch of invalid values for priorities and see what the code makes of them. """ userName = "******" groupName = "Li" teamName = "Tang" schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put("request/testRequest", schema) self.assertEqual(result[1], 200) requestName = result[0]["RequestName"] raises = False try: priority = sys.maxsize + 1 self.jsonSender.put("request/%s?priority=%s" % (requestName, priority)) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result) self.assertTrue(raises) raises = False try: priority = -1 - sys.maxsize self.jsonSender.put("request/%s?priority=%s" % (requestName, priority)) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) print(ex.result) self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result) self.assertTrue(raises)
class RequestManagerTest(RESTBaseUnitTest): """ Test RequestMgr Service client It will start RequestMgr RESTService Server DB is whatever env is set This checks whether DS call makes without error and return the results. This test only test service call returns without error. The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py """ def initialize(self): self.couchDBName = "reqmgr_t_0" self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') dbUrl = os.environ.get("DATABASE", None) self.config.setDBUrl(dbUrl) self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(8888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] return def setUp(self): """ setUP global values """ RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) self.requestSchema = getRequestSchema() self.jsonSender.put('group/PeopleLikeMe') self.jsonSender.put('user/[email protected]') self.jsonSender.put('group/PeopleLikeMe/me') self.jsonSender.put('version/CMSSW_3_5_8') r = self.jsonSender.put('request/' + self.requestSchema['RequestName'], self.requestSchema) self.requestName = r[0]['RequestName'] def tearDown(self): self.config.deleteWorkloadCache() RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() @attr("integration") def testA_RequestManagerService(self): requestName = self.requestName request = self.reqService.getRequest(requestName) # minimal test : it's return type and the some value inside self.assertEqual(type(request), dict) self.assertTrue(len(request) > 0) # Test putTeam self.reqService.putTeam("team_usa") self.assertTrue('team_usa' in self.jsonSender.get('team')[0]) self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName)) request = self.reqService.getAssignment(teamName = "team_usa") self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) request = self.reqService.getAssignment(request = requestName) self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) self.reqService.sendMessage(requestName,"error") self.reqService.putWorkQueue(requestName, "http://test_url") self.reqService.reportRequestProgress(requestName) self.reqService.reportRequestProgress(requestName, percent_complete = 100, percent_success = 90) self.reqService.reportRequestStatus(requestName, "running") return
class AdminTest(RESTBaseUnitTest): """ Test for the lower-level DB code in the admin section """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName=self.couchDBName) self.config.setPort(12888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def testA_SoftwareManagement(self): """ _SoftwareManagement_ Test the SoftwareManagement code """ self.assertEqual(SoftwareManagement.listSoftware(), {}) softwareVersions = ReqMgrWebTools.allScramArchsAndVersions() ReqMgrWebTools.updateScramArchsAndCMSSWVersions() result = SoftwareManagement.listSoftware() for scramArch in result.keys(): self.assertEqual(set(result[scramArch]), set(softwareVersions[scramArch])) # Now for each scramArch insert a blank set # Because of the way that updateSoftware works, this interprets a blank list # as telling it that no softwareVersions are available. # It deletes every software version it is not handed, so it should give nothing out. for scramArch in result.keys(): SoftwareManagement.updateSoftware(softwareNames=[], scramArch=scramArch) self.assertEqual(SoftwareManagement.listSoftware(), {}) # import has to be here, otherwise getting: # AttributeError: 'Toolbox' object has no attribute 'secmodv2' from the Admin module from WMCore.HTTPFrontEnd.RequestManager import Admin setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase) self.config.section_('templates') self.config.section_('html') admin = Admin.Admin(self.config) ReqMgrWebTools.updateScramArchsAndCMSSWVersions() self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs())
class ReqMgrPriorityTest(RESTBaseUnitTest): """ Basic test for setting the priority in ReqMgr Services """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") self.testInit.setupCouch("%s_acdc" % self.couchDBName, "ACDC") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(8888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload def changeStatusAndCheck(self, requestName, statusName): """ _changeStatusAndCheck_ Change the status of a request and make sure that the request actually did it. """ self.jsonSender.put('request/%s?status=%s' % (requestName, statusName)) result = self.jsonSender.get('request/%s' % requestName) self.assertEqual(result[0]['RequestStatus'], statusName) def testA_RequestPriority(self): """ _priorityChanges_ Do some fairly standard priority changes to the Request and see how things react """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), 0) # Set priority == 5 priority = 5 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['RequestPriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority) # Set priority == 100 priority = 100 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['RequestPriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority) # Set priority == -1 priority = -1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['RequestPriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority) # Let's move the request around a bit self.changeStatusAndCheck(requestName = requestName, statusName = 'testing-approved') self.changeStatusAndCheck(requestName = requestName, statusName = 'testing') self.changeStatusAndCheck(requestName = requestName, statusName = 'tested') self.changeStatusAndCheck(requestName = requestName, statusName = 'assignment-approved') self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName))) # Set priority == 99 priority = 99 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['RequestPriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority) def testB_InvalidPriority(self): """ _InvalidPriority_ Put in a bunch of invalid values for priorities and see what the code makes of them. """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] raises = False try: priority = sys.maxint + 1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result) self.assertTrue(raises) raises = False try: priority = -1 - sys.maxint self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) print ex.result self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
class ReqMgrWorkloadTest(RESTBaseUnitTest): """ Test that sets up and checks the validations of the various main WMSpec.StdSpecs This is mostly a simple set of tests which can be very repetitive. """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(12888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def createConfig(self, bad = False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = {'process': {'outputModules_': ['ThisIsAName'], 'ThisIsAName': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}, 'ThisIsAName2': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName) configCache.createUserGroup(groupname = "testGroup", username = '******') if bad: configCache.setPSetTweaks(PSetTweak = BadTweak) else: configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.save() return configCache.getCouchID() def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload @attr('integration') def testA_makeReRecoWorkload(self): """ _makeReRecoWorkload_ Check that you can make a basic ReReco workload """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) del schema['GlobalTag'] configID = self.createConfig(bad = False) schema['ConfigCacheID'] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName raises = False try: self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Argument GlobalTag is required." in ex.result) self.assertTrue(raises) schema = utils.getSchema(groupName = groupName, userName = userName) schema['InputDataset'] = '/Nothing' raises = False try: self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue(raises) schema = utils.getSchema(groupName = groupName, userName = userName) raises = False del schema['ConfigCacheID'] try: self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Create request failed, 'ConfigCacheID'" in ex.result) self.assertTrue(raises) configID = self.createConfig(bad = True) schema = utils.getSchema(groupName = groupName, userName = userName) schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName raises = False try: self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: " "Duplicate dataTier/filterName combination" in ex.result) self.assertTrue(raises) schema = utils.getSchema(groupName = groupName, userName = userName) configID = self.createConfig(bad = False) schema['ConfigCacheID'] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName try: result = self.jsonSender.put('request', schema) except Exception as ex: raise self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) @attr('integration') def testB_Analysis(self): """ _Analysis_ Test Analysis workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "Analysis" try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue(raises) # Put the right things in the schema # And watch it fail, because we can't open a secure connection to CERN # in a unittest well schema['RequestorDN'] = 'SomeDN' #result = self.jsonSender.put('request/testRequest', schema) #requestName = result[0]['RequestName'] #result = self.jsonSender.get('request/%s' % requestName) #request = result[0] #self.assertEqual(request['CMSSWVersion'], CMSSWVersion) #self.assertEqual(request['Group'], groupName) #self.assertEqual(request['Requestor'], userName) @attr('integration') def testD_ReDigi(self): """ _ReDigi_ Test ReDigi workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "ReDigi" try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Create request failed, 'StepOneConfigCacheID'" in ex.result) self.assertTrue(raises) schema["StepOneConfigCacheID"] = "fakeID" schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) configID = self.createConfig() schema["StepOneConfigCacheID"] = configID result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['SizePerEvent'], 512) self.assertEqual(request['RequestNumEvents'], 0) self.assertEqual(request['RequestSizeFiles'], 0) @attr('integration') def testE_StoreResults(self): """ _StoreResults_ Test StoreResults workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "StoreResults" schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Argument CmsPath is required." in ex.result) self.assertTrue(raises) schema['DbsUrl'] = 'http://fake.dbs.url/dbs' schema['CmsPath'] = '/fake/tmp/path' schema['AcquisitionEra'] = 'era' schema['ProcessingVersion'] = 1 result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['DbsUrl'], schema['DbsUrl']) @attr('integration') def testF_TaskChain(self): """ _TaskChain_ Test the monstrous TaskChain workflow This will be a long one NOTE: This test is so complicated that all I do is take code from TaskChain_t and make sure it still produces and actual request """ couchServer = CouchServer(os.environ["COUCHURL"]) configDatabase = couchServer.connectDatabase(self.couchDBName) generatorDoc = makeGeneratorConfig(configDatabase) processorDocs = makeProcessingConfigs(configDatabase) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getSchema(userName = userName) schema["CouchURL"] = os.environ["COUCHURL"] schema["CouchDBName"] = self.couchDBName schema["CouchWorkloadDBName"] = self.couchDBName schema["SiteWhitelist"] = ["T1_CH_CERN", "T1_US_FNAL"] schema["TaskChain"] = 5 chains = {"Task1" : {"TaskName" : "GenSim", "ConfigCacheID" : generatorDoc, "SplittingAlgo" : "EventBased", "EventsPerJob" : 250, "RequestNumEvents" : 10000, "PrimaryDataset" : "RelValTTBar"}, "Task2" : {"TaskName" : "DigiHLT", "InputTask" : "GenSim", "InputFromOutputModule" : "writeGENSIM", "ConfigCacheID" : processorDocs['DigiHLT'], "SplittingAlgo" : "FileBased"}, "Task3" : {"TaskName" : "Reco", "InputTask" : "DigiHLT", "InputFromOutputModule" : "writeRAWDIGI", "ConfigCacheID" : processorDocs['Reco'], "SplittingAlgo" : "FileBased"}, "Task4" : {"TaskName" : "ALCAReco", "InputTask" : "Reco", "InputFromOutputModule" : "writeALCA", "ConfigCacheID" : processorDocs['ALCAReco'], "SplittingAlgo" : "FileBased"}, "Task5" : {"TaskName" : "Skims", "InputTask" : "Reco", "InputFromOutputModule" : "writeRECO", "ConfigCacheID" : processorDocs['Skims'], "SplittingAlgo" : "FileBased", "FilesPerJob" : 10 } } schema.update(chains) args = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema.update(args) # this is necessary and after all updates to the schema are made, # otherwise this item will get overwritten schema['RequestType'] = "TaskChain" schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) workload = self.loadWorkload(requestName) self.assertEqual(workload.data.request.schema.Task1["EventsPerJob"], 250) @attr('integration') def testG_MonteCarloFromGEN(self): """ _MonteCarloFromGEN_ Test MonteCarloFromGEN workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "MonteCarloFromGEN" try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) schema["ConfigCacheID"] = "fakeID" schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) configID = self.createConfig() schema["ConfigCacheID"] = configID result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['SizePerEvent'], 512) self.assertEqual(request['RequestNumEvents'], 0) self.assertEqual(request['RequestSizeFiles'], 0) def testH_MonteCarlo(self): """ _MonteCarlo_ Test MonteCarlo workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "MonteCarlo" schema['DbsUrl'] = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' # Set some versions schema['ProcessingVersion'] = '2012' schema['AcquisitionEra'] = 'ae2012' try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) schema["ConfigCacheID"] = "fakeID" schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName schema["PrimaryDataset"] = "ReallyFake" schema["RequestNumEvents"] = 100 try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue(raises) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["FilterEfficiency"] = -0.5 try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) # until exception handling is redone (New REST API) #self.assertTrue("Negative filter efficiency for MC workflow" in ex.result) self.assertTrue(raises) schema["FilterEfficiency"] = 1.0 result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['DbsUrl'], schema['DbsUrl']) self.assertEqual(request['SizePerEvent'], 512) self.assertEqual(request['RequestNumEvents'], 100) self.assertEqual(request['RequestSizeFiles'], 0) def testJ_Resubmission(self): """ _Resubmission_ Test Resubmission """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "ReReco" configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] # user, group schema already set up schema = utils.getSchema(groupName = groupName, userName = userName) schema['RequestType'] = "Resubmission" schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Validation failed: InitialTaskPath is mendatory" in ex.result) self.assertTrue(raises) schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName schema["ACDCServer"] = os.environ.get("COUCHURL") schema["ACDCDatabase"] = self.couchDBName schema["CollectionName"] = "SomeOtherName" # Here we just make sure that real result goes through result = self.jsonSender.put('request', schema) resubmitName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % resubmitName) couchServer = CouchServer(self.testInit.couchUrl) reqmgrCouch = couchServer.connectDatabase(self.couchDBName) result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests', {}, [requestName])['rows'] self.assertEqual(len(result), 1) self.assertEqual(result[0]['key'], requestName) self.assertEqual(result[0]['id'], resubmitName)
class RequestManagerTest(RESTBaseUnitTest): """ Test RequestMgr Service client It will start RequestMgr RESTService Server DB is whatever env is set This checks whether DS call makes without error and return the results. This test only test service call returns without error. The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py """ def initialize(self): self.couchDBName = "reqmgr_t_0" self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') dbUrl = os.environ.get("DATABASE", None) self.config.setDBUrl(dbUrl) self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(8899) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def setUp(self): RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") # logging stuff from TestInit is broken, setting myself l = logging.getLogger() l.setLevel(logging.DEBUG) self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['ConfigCacheID'] = self.createConfig() schema['CouchDBName'] = self.couchDBName schema['CouchWorkloadDBName'] = self.couchDBName try: r = self.jsonSender.put('request', schema) try: self.requestName = r[0]['RequestName'] except: self.requestName = r[0].values()[0]['RequestName'] except Exception as ex: msg = traceback.format_exc() print("Exception during set up, reason: %s" % msg) raise ex def tearDown(self): self.config.deleteWorkloadCache() RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def createConfig(self, bad = False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = {'process': {'outputModules_': ['ThisIsAName'], 'ThisIsAName': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}, 'ThisIsAName2': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName) configCache.createUserGroup(groupname = "testGroup", username = '******') if bad: configCache.setPSetTweaks(PSetTweak = BadTweak) else: configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.save() return configCache.getCouchID() @attr("integration") def testA_RequestManagerService(self): requestName = self.requestName request = self.reqService.getRequest(requestName) # minimal test : it's return type and the some value inside self.assertEqual(type(request), dict) self.assertTrue(len(request) > 0) # Test putTeam self.reqService.putTeam("team_usa") self.assertTrue('team_usa' in self.jsonSender.get('team')[0]) self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName)) request = self.reqService.getAssignment(teamName = "team_usa") self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) request = self.reqService.getAssignment(request = requestName) self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) self.reqService.sendMessage(requestName,"error") self.reqService.putWorkQueue(requestName, "http://test_url") self.reqService.reportRequestProgress(requestName, percent_complete = 100, percent_success = 90) self.reqService.updateRequestStatus(requestName, "running-open")
class ReqMgrPriorityTest(RESTBaseUnitTest): """ Basic test for setting the priority in ReqMgr Services """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") self.testInit.setupCouch("%s_acdc" % self.couchDBName, "ACDC") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName=self.couchDBName) self.config.setPort(8888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload def createConfig(self, bad=False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = { 'process': { 'outputModules_': ['ThisIsAName'], 'ThisIsAName': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } } } } BadTweak = { 'process': { 'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } }, 'ThisIsAName2': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } } } } configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName) configCache.createUserGroup(groupname="testGroup", username='******') if bad: configCache.setPSetTweaks(PSetTweak=BadTweak) else: configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.save() return configCache.getCouchID() def changeStatusAndCheck(self, requestName, statusName): """ _changeStatusAndCheck_ Change the status of a request and make sure that the request actually did it. """ self.jsonSender.put('request/%s?status=%s' % (requestName, statusName)) result = self.jsonSender.get('request/%s' % requestName) self.assertEqual(result[0]['RequestStatus'], statusName) def testA_RequestPriority(self): """ _priorityChanges_ Do some fairly standard priority changes to the Request and see how things react """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), 0) # Set priority == 5 priority = 5 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['RequestPriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) # Set priority == 100 priority = 100 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['RequestPriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) # Set priority == -1 priority = -1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['RequestPriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) # Let's move the request around a bit self.changeStatusAndCheck(requestName=requestName, statusName='testing-approved') self.changeStatusAndCheck(requestName=requestName, statusName='testing') self.changeStatusAndCheck(requestName=requestName, statusName='tested') self.changeStatusAndCheck(requestName=requestName, statusName='assignment-approved') self.jsonSender.put( urllib.quote('assignment/%s/%s' % (teamName, requestName))) # Set priority == 99 priority = 99 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['RequestPriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) def testB_InvalidPriority(self): """ _InvalidPriority_ Put in a bunch of invalid values for priorities and see what the code makes of them. """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] raises = False try: priority = sys.maxint + 1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Priority must have abs() less then MAXINT!" in ex.result) self.assertTrue(raises) raises = False try: priority = -1 - sys.maxint self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) print ex.result self.assertTrue( "Priority must have abs() less then MAXINT!" in ex.result)
class ReqMgrPriorityTest(RESTBaseUnitTest): """ _ReqMgrPriorityTest_ Basic test for setting the priority in ReqMgr Services """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) return def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(8888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] return def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() return def setupSchema(self, groupName = 'PeopleLikeMe', userName = '******', teamName = 'White Sox', CMSSWVersion = 'CMSSW_3_5_8', scramArch = 'slc5_ia32_gcc434'): """ _setupSchema_ Set up a test schema so that we can run a test request. Standardization! """ self.jsonSender.put('user/%[email protected]' % userName) self.jsonSender.put('group/%s' % groupName) self.jsonSender.put('group/%s/%s' % (groupName, userName)) self.jsonSender.put(urllib.quote('team/%s' % teamName)) self.jsonSender.put('version/%s/%s' % (CMSSWVersion, scramArch)) schema = ReReco.getTestArguments() schema['RequestName'] = 'TestReReco' schema['RequestType'] = 'ReReco' schema['CmsPath'] = "/uscmst1/prod/sw/cms" schema['Requestor'] = '%s' % userName schema['Group'] = '%s' % groupName return schema def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload def changeStatusAndCheck(self, requestName, statusName): """ _changeStatusAndCheck_ Change the status of a request and make sure that the request actually did it. """ self.jsonSender.put('request/%s?status=%s' % (requestName, statusName)) result = self.jsonSender.get('request/%s' % requestName) self.assertEqual(result[0]['RequestStatus'], statusName) return def testA_RequestPriority(self): """ _priorityChanges_ Do some fairly standard priority changes to the Request and see how things react """ userName = '******' groupName = 'Li' teamName = 'Tang' CMSSWVersion = 'CMSSW_3_5_8' schema = self.setupSchema(userName = userName, groupName = groupName, teamName = teamName, CMSSWVersion = CMSSWVersion) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), 0) # Reset user, group priorities to 0 self.jsonSender.post('user/%s?priority=0' % userName) self.jsonSender.post('group/%s?priority=0' % groupName) # Set priority == 5 priority = 5 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority) # Set priority == 100 priority = 100 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority) # Set priority == -1 priority = -1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority) # Let's move the request around a bit self.changeStatusAndCheck(requestName = requestName, statusName = 'testing-approved') self.changeStatusAndCheck(requestName = requestName, statusName = 'testing') self.changeStatusAndCheck(requestName = requestName, statusName = 'tested') self.changeStatusAndCheck(requestName = requestName, statusName = 'assignment-approved') self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName))) # Set priority == 99 priority = 99 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority) return def testB_InvalidPriority(self): """ _InvalidPriority_ Put in a bunch of invalid values for priorities and see what the code makes of them. """ userName = '******' groupName = 'Li' teamName = 'Tang' CMSSWVersion = 'CMSSW_3_5_8' schema = self.setupSchema(userName = userName, groupName = groupName, teamName = teamName, CMSSWVersion = CMSSWVersion) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] raises = False try: priority = sys.maxint + 1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result) self.assertTrue(raises) raises = False try: priority = -1 - sys.maxint self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) print ex.result self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
class ReqMgrWorkloadTest(RESTBaseUnitTest): """ _ReqMgrWorkloadTest_ Test that sets up and checks the validations of the various main WMSpec.StdSpecs This is mostly a simple set of tests which can be very repetitive. """ def setUp(self): """ setUP global values Database setUp is done in base class """ self.couchDBName = "reqmgr_t_0" RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") reqMgrHost = self.config.getServerUrl() self.jsonSender = JSONRequests(reqMgrHost) return def initialize(self): self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(12888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] return def tearDown(self): """ _tearDown_ Basic tear down of database """ RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() return def createConfig(self, bad = False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = {'process': {'outputModules_': ['ThisIsAName'], 'ThisIsAName': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}, 'ThisIsAName2': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName) configCache.createUserGroup(groupname = "testGroup", username = '******') if bad: configCache.setPSetTweaks(PSetTweak = BadTweak) else: configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.save() return configCache.getCouchID() def setupSchema(self, groupName = 'PeopleLikeMe', userName = '******', teamName = 'White Sox', CMSSWVersion = 'CMSSW_3_5_8', typename = 'ReReco', setupDB = True, scramArch = 'slc5_ia32_gcc434'): """ _setupSchema_ Set up a test schema so that we can run a test request. Standardization! """ if setupDB: self.jsonSender.put('user/%[email protected]' % userName) self.jsonSender.put('group/%s' % groupName) try: self.jsonSender.put('group/%s/%s' % (groupName, userName)) except: # Done this already pass self.jsonSender.put(urllib.quote('team/%s' % teamName)) self.jsonSender.put('version/%s/%s' % (CMSSWVersion, scramArch)) schema = ReReco.getTestArguments() schema['RequestName'] = 'TestReReco' schema['RequestType'] = typename schema['CmsPath'] = "/uscmst1/prod/sw/cms" schema['Requestor'] = '%s' % userName schema['Group'] = '%s' % groupName return schema def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload @attr('integration') def testA_makeReRecoWorkload(self): """ _makeReRecoWorkload_ Check that you can make a basic ReReco workload """ userName = '******' groupName = 'Li' teamName = 'Tang' CMSSWVersion = 'CMSSW_3_5_8' # Okay, we can make one. Shouldn't surprise us. Let's try # and make a bad one. schema = self.setupSchema(userName = userName, groupName = groupName, teamName = teamName, CMSSWVersion = CMSSWVersion, setupDB = True) del schema['GlobalTag'] raises = False try: self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Missing required field GlobalTag in workload validation" in ex.result) self.assertTrue(raises) schema = self.setupSchema(userName = userName, groupName = groupName, teamName = teamName, CMSSWVersion = CMSSWVersion, setupDB = False) schema['InputDataset'] = '/Nothing' raises = False try: self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Bad value for InputDataset" in ex.result)