def initialize(self):
self.config = RequestManagerConfig("WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel")
self.config.setFormatter("WMCore.WebTools.RESTFormatter")
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.Assign')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
return
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8899)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
class ReqMgrPriorityTest(RESTBaseUnitTest):
"""
Basic test for setting the priority in ReqMgr Services
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
self.testInit.setupCouch("%s_acdc" % self.couchDBName, "ACDC")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def changeStatusAndCheck(self, requestName, statusName):
"""
_changeStatusAndCheck_
Change the status of a request and make sure that
the request actually did it.
"""
self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], statusName)
def testA_RequestPriority(self):
"""
_priorityChanges_
Do some fairly standard priority changes to the Request and
see how things react
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), 0)
# Set priority == 5
priority = 5
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == 100
priority = 100
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == -1
priority = -1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Let's move the request around a bit
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing-approved')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'tested')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assignment-approved')
self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName)))
# Set priority == 99
priority = 99
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxint + 1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
raises = False
try:
priority = -1 - sys.maxint
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
class ReqMgrWorkloadTest(RESTBaseUnitTest):
"""
Test that sets up and checks the validations of the various main WMSpec.StdSpecs
This is mostly a simple set of tests which can be very repetitive.
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
@attr('integration')
def testA_makeReRecoWorkload(self):
"""
_makeReRecoWorkload_
Check that you can make a basic ReReco workload
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
del schema['GlobalTag']
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Missing required "
"field 'GlobalTag' in workload validation!" in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['InputDataset'] = '/Nothing'
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
class AssignTest(RESTBaseUnitTest):
"""
_AssignTest_
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self, initRoot=False)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.Assign')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def testA_SiteWhitelist(self):
"""
_SiteWhitelist_
Test to see if we can get the siteWhitelist to work properly.
"""
secconfig = getattr(self.config, "SecurityModule")
cherrypy.server.environment = 'test'
cherrypy.tools.secmodv2 = NullAuth(secconfig)
self.config.UnitTests.views.active.rest.templates = 'templateDir'
self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot'
self.config.UnitTests.views.active.rest.wildcardKeys = {
'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*',
'US*': '_US_'
}
from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign
assign = Assign(config=self.config.UnitTests.views.active.rest,
noSiteDB=True)
siteList = [
'T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL',
'T2_US_CIT'
]
self.assertEqual(assign.sites, [])
assign.sites.extend(siteList)
Utilities.addSiteWildcards(assign.wildcardKeys, assign.sites,
assign.wildcardSites)
for s in siteList:
self.assertTrue(s in assign.sites)
self.assertTrue('T1*' in assign.sites)
self.assertTrue('T2*' in assign.sites)
self.assertFalse('T3*' in assign.sites)
self.assertTrue('US*' in assign.sites)
self.assertEqual(assign.wildcardSites['T1*'],
['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL'])
self.assertEqual(assign.wildcardSites['T2*'],
['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
self.assertEqual(
assign.wildcardSites['US*'],
['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
class ReqMgrPriorityTest(RESTBaseUnitTest):
"""
_ReqMgrPriorityTest_
Basic test for setting the priority in ReqMgr Services
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def changeStatusAndCheck(self, requestName, statusName):
"""
_changeStatusAndCheck_
Change the status of a request and make sure that
the request actually did it.
"""
self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], statusName)
def testA_RequestPriority(self):
"""
_priorityChanges_
Do some fairly standard priority changes to the Request and
see how things react
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), 0)
# Reset user, group priorities to 0
self.jsonSender.post('user/%s?priority=0' % userName)
self.jsonSender.post('group/%s?priority=0' % groupName)
# Set priority == 5
priority = 5
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == 100
priority = 100
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == -1
priority = -1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Let's move the request around a bit
self.changeStatusAndCheck(requestName=requestName,
statusName='testing-approved')
self.changeStatusAndCheck(requestName=requestName,
statusName='testing')
self.changeStatusAndCheck(requestName=requestName, statusName='tested')
self.changeStatusAndCheck(requestName=requestName,
statusName='assignment-approved')
self.jsonSender.put(
urllib.quote('assignment/%s/%s' % (teamName, requestName)))
# Set priority == 99
priority = 99
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxint + 1
self.jsonSender.put('request/%s?priority=%s' %
(requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
raises = False
try:
priority = -1 - sys.maxint
self.jsonSender.put('request/%s?priority=%s' %
(requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue(
"Priority must have abs() less then MAXINT!" in ex.result)
class ReqMgrWorkloadTest(RESTBaseUnitTest):
"""
_ReqMgrWorkloadTest_
Test that sets up and checks the validations of the various main WMSpec.StdSpecs
This is mostly a simple set of tests which can be very repetitive.
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
return
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
return
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
return
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
def setupSchema(self, groupName = 'PeopleLikeMe',
userName = '******', teamName = 'White Sox',
CMSSWVersion = 'CMSSW_3_5_8',
typename = 'ReReco', setupDB = True,
scramArch = 'slc5_ia32_gcc434'):
"""
_setupSchema_
Set up a test schema so that we can run a test request.
Standardization!
"""
if setupDB:
self.jsonSender.put('user/%[email protected]' % userName)
self.jsonSender.put('group/%s' % groupName)
try:
self.jsonSender.put('group/%s/%s' % (groupName, userName))
except:
# Done this already
pass
self.jsonSender.put(urllib.quote('team/%s' % teamName))
self.jsonSender.put('version/%s/%s' % (CMSSWVersion, scramArch))
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = typename
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = '%s' % userName
schema['Group'] = '%s' % groupName
return schema
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
@attr('integration')
def testA_makeReRecoWorkload(self):
"""
_makeReRecoWorkload_
Check that you can make a basic ReReco workload
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
CMSSWVersion = 'CMSSW_3_5_8'
# Okay, we can make one. Shouldn't surprise us. Let's try
# and make a bad one.
schema = self.setupSchema(userName = userName,
groupName = groupName,
teamName = teamName,
CMSSWVersion = CMSSWVersion,
setupDB = True)
del schema['GlobalTag']
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Missing required field GlobalTag in workload validation" in ex.result)
self.assertTrue(raises)
schema = self.setupSchema(userName = userName,
groupName = groupName,
teamName = teamName,
CMSSWVersion = CMSSWVersion,
setupDB = False)
schema['InputDataset'] = '/Nothing'
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Bad value for InputDataset" in ex.result)
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
return
def setUp(self):
"""
setUP global values
"""
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
self.requestSchema = getRequestSchema()
self.jsonSender.put('group/PeopleLikeMe')
self.jsonSender.put('user/[email protected]')
self.jsonSender.put('group/PeopleLikeMe/me')
self.jsonSender.put('version/CMSSW_3_5_8')
r = self.jsonSender.put('request/' + self.requestSchema['RequestName'],
self.requestSchema)
self.requestName = r[0]['RequestName']
def tearDown(self):
self.config.deleteWorkloadCache()
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
@attr("integration")
def testA_RequestManagerService(self):
requestName = self.requestName
request = self.reqService.getRequest(requestName)
# minimal test : it's return type and the some value inside
self.assertEqual(type(request), dict)
self.assertTrue(len(request) > 0)
# Test putTeam
self.reqService.putTeam("team_usa")
self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
request = self.reqService.getAssignment(teamName = "team_usa")
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
request = self.reqService.getAssignment(request = requestName)
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
self.reqService.sendMessage(requestName,"error")
self.reqService.putWorkQueue(requestName, "http://test_url")
self.reqService.reportRequestProgress(requestName)
self.reqService.reportRequestProgress(requestName,
percent_complete = 100, percent_success = 90)
self.reqService.reportRequestStatus(requestName, "running")
return
class AdminTest(RESTBaseUnitTest):
"""
_AdminTest_
Test for the lower-level DB code in the admin section
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
return
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
return
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
return
def testA_SoftwareManagement(self):
"""
_SoftwareManagement_
Test the SoftwareManagement code
"""
self.assertEqual(SoftwareManagement.listSoftware(), {})
softwareVersions = ReqMgrWebTools.allScramArchsAndVersions()
ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
result = SoftwareManagement.listSoftware()
for scramArch in result.keys():
self.assertEqual(set(result[scramArch]), set(softwareVersions[scramArch]))
# Now for each scramArch insert a blank set
# Because of the way that updateSoftware works, this interprets a blank list
# as telling it that no softwareVersions are available.
# It deletes every software version it is not handed, so it should give nothing out.
for scramArch in result.keys():
SoftwareManagement.updateSoftware(softwareNames = [], scramArch = scramArch)
self.assertEqual(SoftwareManagement.listSoftware(), {})
from WMCore.HTTPFrontEnd.RequestManager import Admin
setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase)
self.config.section_('templates')
self.config.section_('html')
admin = Admin.Admin(self.config)
ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs())
return
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
self.jsonSender.put('user/[email protected]')
self.jsonSender.put('group/PeopleLikeMe')
self.jsonSender.put('group/PeopleLikeMe/me')
self.jsonSender.put('version/CMSSW_3_5_8/slc5_ia32_gcc434')
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = 'ReReco'
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = '%s' % "me"
schema['Group'] = '%s' % "PeopleLikeMe"
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
schema['Campaign'] = 'MyTestCampaign'
try:
r = self.jsonSender.put('request/' + schema['RequestName'], schema)
except Exception, ex:
print "Exception during set up, investigate exception instance attributes:"
print dir(ex)
return
self.requestName = r[0]['RequestName']
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
try:
r = self.jsonSender.put('request/' + schema['RequestName'], schema)
except Exception, ex:
print "Exception during set up, reason: %s" % ex
return
self.requestName = r[0]['RequestName']
class ReqMgrWorkloadTest(RESTBaseUnitTest):
"""
Test that sets up and checks the validations of the various main WMSpec.StdSpecs
This is mostly a simple set of tests which can be very repetitive.
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache",
"ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
'process': {
'outputModules_': ['ThisIsAName'],
'ThisIsAName': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
BadTweak = {
'process': {
'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
},
'ThisIsAName2': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username='******')
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
@attr('integration')
def testA_makeReRecoWorkload(self):
"""
_makeReRecoWorkload_
Check that you can make a basic ReReco workload
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
del schema['GlobalTag']
configID = self.createConfig(bad=False)
schema['ConfigCacheID'] = configID
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Argument GlobalTag is required."
in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName=groupName, userName=userName)
schema['InputDataset'] = '/Nothing'
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
class AssignTest(RESTBaseUnitTest):
"""
_AssignTest_
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self, initRoot = False)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
return
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.Assign')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
return
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
return
def setupSchema(self, groupName = 'PeopleLikeMe',
userName = '******', teamName = 'White Sox',
CMSSWVersion = 'CMSSW_3_5_8'):
"""
_setupSchema_
Set up a test schema so that we can run a test request.
Standardization!
"""
self.jsonSender.put('user/%[email protected]' % userName)
self.jsonSender.put('group/%s' % groupName)
self.jsonSender.put('group/%s/%s' % (groupName, userName))
self.jsonSender.put(urllib.quote('team/%s' % teamName))
self.jsonSender.put('version/%s' % CMSSWVersion)
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = 'ReReco'
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = '%s' % userName
schema['Group'] = '%s' % groupName
return schema
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def testA_SiteWhitelist(self):
"""
_SiteWhitelist_
Test to see if we can get the siteWhitelist to work properly.
"""
secconfig = getattr(self.config, "SecurityModule")
cherrypy.server.environment = 'test'
cherrypy.tools.secmodv2 = NullAuth(secconfig)
self.config.UnitTests.views.active.rest.templates = 'templateDir'
self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot'
self.config.UnitTests.views.active.rest.wildcardKeys = {'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*',
'US*': '_US_'}
from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign
assign = Assign(config = self.config.UnitTests.views.active.rest, noSiteDB = True)
siteList = ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']
self.assertEqual(assign.sites, [])
assign.sites.extend(siteList)
Utilities.addSiteWildcards(assign.wildcardKeys,assign.sites,assign.wildcardSites)
for s in siteList:
self.assertTrue(s in assign.sites)
self.assertTrue('T1*' in assign.sites)
self.assertTrue('T2*' in assign.sites)
self.assertFalse('T3*' in assign.sites)
self.assertTrue('US*' in assign.sites)
self.assertEqual(assign.wildcardSites['T1*'], ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL'])
self.assertEqual(assign.wildcardSites['T2*'], ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
self.assertEqual(assign.wildcardSites['US*'], ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
return
class ReqMgrPriorityTest(RESTBaseUnitTest):
"""
Basic test for setting the priority in ReqMgr Services
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
self.testInit.setupCouch("%s_acdc" % self.couchDBName, "ACDC")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig("WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel")
self.config.setFormatter("WMCore.WebTools.RESTFormatter")
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = "%s/%s/%s/spec" % (os.environ["COUCHURL"], self.couchDBName, requestName)
workload.load(url)
return workload
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
"process": {
"outputModules_": ["ThisIsAName"],
"ThisIsAName": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
}
}
BadTweak = {
"process": {
"outputModules_": ["ThisIsAName1", "ThisIsAName2"],
"ThisIsAName1": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
"ThisIsAName2": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
}
}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username="******")
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
def changeStatusAndCheck(self, requestName, statusName):
"""
_changeStatusAndCheck_
Change the status of a request and make sure that
the request actually did it.
"""
self.jsonSender.put("request/%s?status=%s" % (requestName, statusName))
result = self.jsonSender.get("request/%s" % requestName)
self.assertEqual(result[0]["RequestStatus"], statusName)
def testA_RequestPriority(self):
"""
_priorityChanges_
Do some fairly standard priority changes to the Request and
see how things react
"""
userName = "******"
groupName = "Li"
teamName = "Tang"
schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request/testRequest", schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), 0)
# Set priority == 5
priority = 5
self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(request["RequestPriority"], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == 100
priority = 100
self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(request["RequestPriority"], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == -1
priority = -1
self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(request["RequestPriority"], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Let's move the request around a bit
self.changeStatusAndCheck(requestName=requestName, statusName="testing-approved")
self.changeStatusAndCheck(requestName=requestName, statusName="testing")
self.changeStatusAndCheck(requestName=requestName, statusName="tested")
self.changeStatusAndCheck(requestName=requestName, statusName="assignment-approved")
self.jsonSender.put(urllib.quote("assignment/%s/%s" % (teamName, requestName)))
# Set priority == 99
priority = 99
self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(request["RequestPriority"], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = "******"
groupName = "Li"
teamName = "Tang"
schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request/testRequest", schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
raises = False
try:
priority = sys.maxsize + 1
self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
raises = False
try:
priority = -1 - sys.maxsize
self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
print(ex.result)
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
class RequestManagerTest(RESTBaseUnitTest):
"""
Test RequestMgr Service client
It will start RequestMgr RESTService
Server DB is whatever env is set
This checks whether DS call makes without error and return the results.
This test only test service call returns without error.
The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py
"""
def initialize(self):
self.couchDBName = "reqmgr_t_0"
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8899)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
schema['CouchWorkloadDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request', schema)
try:
self.requestName = r[0]['RequestName']
except:
self.requestName = r[0].values()[0]['RequestName']
except Exception as ex:
msg = traceback.format_exc()
print("Exception during set up, reason: %s" % msg)
raise ex
def tearDown(self):
self.config.deleteWorkloadCache()
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_RequestManagerService(self):
requestName = self.requestName
request = self.reqService.getRequest(requestName)
# minimal test : it's return type and the some value inside
self.assertEqual(type(request), dict)
self.assertTrue(len(request) > 0)
# Test putTeam
self.reqService.putTeam("team_usa")
self.assertTrue('team_usa' in self.jsonSender.get('team')[0])
self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName))
request = self.reqService.getAssignment(teamName = "team_usa")
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
request = self.reqService.getAssignment(request = requestName)
self.assertEqual(type(request), list)
self.assertTrue(len(request) > 0)
self.reqService.sendMessage(requestName,"error")
self.reqService.putWorkQueue(requestName, "http://test_url")
self.reqService.reportRequestProgress(requestName,
percent_complete = 100, percent_success = 90)
self.reqService.updateRequestStatus(requestName, "running-open")
class ReqMgrWorkloadTest(RESTBaseUnitTest):
"""
Test that sets up and checks the validations of the various main WMSpec.StdSpecs
This is mostly a simple set of tests which can be very repetitive.
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
@attr('integration')
def testA_makeReRecoWorkload(self):
"""
_makeReRecoWorkload_
Check that you can make a basic ReReco workload
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
del schema['GlobalTag']
configID = self.createConfig(bad = False)
schema['ConfigCacheID'] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument GlobalTag is required." in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['InputDataset'] = '/Nothing'
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(raises)
schema = utils.getSchema(groupName = groupName, userName = userName)
raises = False
del schema['ConfigCacheID']
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Create request failed, 'ConfigCacheID'" in ex.result)
self.assertTrue(raises)
configID = self.createConfig(bad = True)
schema = utils.getSchema(groupName = groupName, userName = userName)
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: "
"Duplicate dataTier/filterName combination" in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName = groupName, userName = userName)
configID = self.createConfig(bad = False)
schema['ConfigCacheID'] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
try:
result = self.jsonSender.put('request', schema)
except Exception as ex:
raise
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
@attr('integration')
def testB_Analysis(self):
"""
_Analysis_
Test Analysis workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "Analysis"
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(raises)
# Put the right things in the schema
# And watch it fail, because we can't open a secure connection to CERN
# in a unittest well
schema['RequestorDN'] = 'SomeDN'
#result = self.jsonSender.put('request/testRequest', schema)
#requestName = result[0]['RequestName']
#result = self.jsonSender.get('request/%s' % requestName)
#request = result[0]
#self.assertEqual(request['CMSSWVersion'], CMSSWVersion)
#self.assertEqual(request['Group'], groupName)
#self.assertEqual(request['Requestor'], userName)
@attr('integration')
def testD_ReDigi(self):
"""
_ReDigi_
Test ReDigi workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "ReDigi"
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Create request failed, 'StepOneConfigCacheID'" in ex.result)
self.assertTrue(raises)
schema["StepOneConfigCacheID"] = "fakeID"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Failure to load ConfigCache while validating workload" in ex.result)
self.assertTrue(raises)
configID = self.createConfig()
schema["StepOneConfigCacheID"] = configID
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['SizePerEvent'], 512)
self.assertEqual(request['RequestNumEvents'], 0)
self.assertEqual(request['RequestSizeFiles'], 0)
@attr('integration')
def testE_StoreResults(self):
"""
_StoreResults_
Test StoreResults workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "StoreResults"
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument CmsPath is required." in ex.result)
self.assertTrue(raises)
schema['DbsUrl'] = 'http://fake.dbs.url/dbs'
schema['CmsPath'] = '/fake/tmp/path'
schema['AcquisitionEra'] = 'era'
schema['ProcessingVersion'] = 1
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['DbsUrl'], schema['DbsUrl'])
@attr('integration')
def testF_TaskChain(self):
"""
_TaskChain_
Test the monstrous TaskChain workflow
This will be a long one
NOTE: This test is so complicated that all I do is
take code from TaskChain_t and make sure it still
produces and actual request
"""
couchServer = CouchServer(os.environ["COUCHURL"])
configDatabase = couchServer.connectDatabase(self.couchDBName)
generatorDoc = makeGeneratorConfig(configDatabase)
processorDocs = makeProcessingConfigs(configDatabase)
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getSchema(userName = userName)
schema["CouchURL"] = os.environ["COUCHURL"]
schema["CouchDBName"] = self.couchDBName
schema["CouchWorkloadDBName"] = self.couchDBName
schema["SiteWhitelist"] = ["T1_CH_CERN", "T1_US_FNAL"]
schema["TaskChain"] = 5
chains = {"Task1" : {"TaskName" : "GenSim",
"ConfigCacheID" : generatorDoc,
"SplittingAlgo" : "EventBased",
"EventsPerJob" : 250,
"RequestNumEvents" : 10000,
"PrimaryDataset" : "RelValTTBar"},
"Task2" : {"TaskName" : "DigiHLT",
"InputTask" : "GenSim",
"InputFromOutputModule" : "writeGENSIM",
"ConfigCacheID" : processorDocs['DigiHLT'],
"SplittingAlgo" : "FileBased"},
"Task3" : {"TaskName" : "Reco",
"InputTask" : "DigiHLT",
"InputFromOutputModule" : "writeRAWDIGI",
"ConfigCacheID" : processorDocs['Reco'],
"SplittingAlgo" : "FileBased"},
"Task4" : {"TaskName" : "ALCAReco",
"InputTask" : "Reco",
"InputFromOutputModule" : "writeALCA",
"ConfigCacheID" : processorDocs['ALCAReco'],
"SplittingAlgo" : "FileBased"},
"Task5" : {"TaskName" : "Skims",
"InputTask" : "Reco",
"InputFromOutputModule" : "writeRECO",
"ConfigCacheID" : processorDocs['Skims'],
"SplittingAlgo" : "FileBased",
"FilesPerJob" : 10 } }
schema.update(chains)
args = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema.update(args)
# this is necessary and after all updates to the schema are made,
# otherwise this item will get overwritten
schema['RequestType'] = "TaskChain"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
workload = self.loadWorkload(requestName)
self.assertEqual(workload.data.request.schema.Task1["EventsPerJob"],
250)
@attr('integration')
def testG_MonteCarloFromGEN(self):
"""
_MonteCarloFromGEN_
Test MonteCarloFromGEN workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "MonteCarloFromGEN"
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Failure to load ConfigCache while validating workload" in ex.result)
self.assertTrue(raises)
schema["ConfigCacheID"] = "fakeID"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Failure to load ConfigCache while validating workload" in ex.result)
self.assertTrue(raises)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['SizePerEvent'], 512)
self.assertEqual(request['RequestNumEvents'], 0)
self.assertEqual(request['RequestSizeFiles'], 0)
def testH_MonteCarlo(self):
"""
_MonteCarlo_
Test MonteCarlo workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "MonteCarlo"
schema['DbsUrl'] = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
# Set some versions
schema['ProcessingVersion'] = '2012'
schema['AcquisitionEra'] = 'ae2012'
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Failure to load ConfigCache while validating workload" in ex.result)
self.assertTrue(raises)
schema["ConfigCacheID"] = "fakeID"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
schema["PrimaryDataset"] = "ReallyFake"
schema["RequestNumEvents"] = 100
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(raises)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["FilterEfficiency"] = -0.5
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
# until exception handling is redone (New REST API)
#self.assertTrue("Negative filter efficiency for MC workflow" in ex.result)
self.assertTrue(raises)
schema["FilterEfficiency"] = 1.0
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['DbsUrl'], schema['DbsUrl'])
self.assertEqual(request['SizePerEvent'], 512)
self.assertEqual(request['RequestNumEvents'], 100)
self.assertEqual(request['RequestSizeFiles'], 0)
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "ReReco"
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['RequestType'] = "Resubmission"
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Validation failed: InitialTaskPath is mendatory" in ex.result)
self.assertTrue(raises)
schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName
schema["ACDCServer"] = os.environ.get("COUCHURL")
schema["ACDCDatabase"] = self.couchDBName
schema["CollectionName"] = "SomeOtherName"
# Here we just make sure that real result goes through
result = self.jsonSender.put('request', schema)
resubmitName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % resubmitName)
couchServer = CouchServer(self.testInit.couchUrl)
reqmgrCouch = couchServer.connectDatabase(self.couchDBName)
result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests', {}, [requestName])['rows']
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['key'], requestName)
self.assertEqual(result[0]['id'], resubmitName)
class ReqMgrPriorityTest(RESTBaseUnitTest):
"""
Basic test for setting the priority in ReqMgr Services
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache",
"ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
self.testInit.setupCouch("%s_acdc" % self.couchDBName, "ACDC")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
'process': {
'outputModules_': ['ThisIsAName'],
'ThisIsAName': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
BadTweak = {
'process': {
'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
},
'ThisIsAName2': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username='******')
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
def changeStatusAndCheck(self, requestName, statusName):
"""
_changeStatusAndCheck_
Change the status of a request and make sure that
the request actually did it.
"""
self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], statusName)
def testA_RequestPriority(self):
"""
_priorityChanges_
Do some fairly standard priority changes to the Request and
see how things react
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), 0)
# Set priority == 5
priority = 5
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == 100
priority = 100
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == -1
priority = -1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
# Let's move the request around a bit
self.changeStatusAndCheck(requestName=requestName,
statusName='testing-approved')
self.changeStatusAndCheck(requestName=requestName,
statusName='testing')
self.changeStatusAndCheck(requestName=requestName, statusName='tested')
self.changeStatusAndCheck(requestName=requestName,
statusName='assignment-approved')
self.jsonSender.put(
urllib.quote('assignment/%s/%s' % (teamName, requestName)))
# Set priority == 99
priority = 99
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxint + 1
self.jsonSender.put('request/%s?priority=%s' %
(requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
raises = False
try:
priority = -1 - sys.maxint
self.jsonSender.put('request/%s?priority=%s' %
(requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue(
"Priority must have abs() less then MAXINT!" in ex.result)
class ReqMgrPriorityTest(RESTBaseUnitTest):
"""
_ReqMgrPriorityTest_
Basic test for setting the priority in ReqMgr Services
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
return
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(8888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
return
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
return
def setupSchema(self, groupName = 'PeopleLikeMe',
userName = '******', teamName = 'White Sox',
CMSSWVersion = 'CMSSW_3_5_8',
scramArch = 'slc5_ia32_gcc434'):
"""
_setupSchema_
Set up a test schema so that we can run a test request.
Standardization!
"""
self.jsonSender.put('user/%[email protected]' % userName)
self.jsonSender.put('group/%s' % groupName)
self.jsonSender.put('group/%s/%s' % (groupName, userName))
self.jsonSender.put(urllib.quote('team/%s' % teamName))
self.jsonSender.put('version/%s/%s' % (CMSSWVersion, scramArch))
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = 'ReReco'
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = '%s' % userName
schema['Group'] = '%s' % groupName
return schema
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def changeStatusAndCheck(self, requestName, statusName):
"""
_changeStatusAndCheck_
Change the status of a request and make sure that
the request actually did it.
"""
self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], statusName)
return
def testA_RequestPriority(self):
"""
_priorityChanges_
Do some fairly standard priority changes to the Request and
see how things react
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
CMSSWVersion = 'CMSSW_3_5_8'
schema = self.setupSchema(userName = userName,
groupName = groupName,
teamName = teamName,
CMSSWVersion = CMSSWVersion)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), 0)
# Reset user, group priorities to 0
self.jsonSender.post('user/%s?priority=0' % userName)
self.jsonSender.post('group/%s?priority=0' % groupName)
# Set priority == 5
priority = 5
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == 100
priority = 100
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Set priority == -1
priority = -1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
# Let's move the request around a bit
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing-approved')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'tested')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assignment-approved')
self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName)))
# Set priority == 99
priority = 99
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
return
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
CMSSWVersion = 'CMSSW_3_5_8'
schema = self.setupSchema(userName = userName,
groupName = groupName,
teamName = teamName,
CMSSWVersion = CMSSWVersion)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxint + 1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
raises = False
try:
priority = -1 - sys.maxint
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
class AssignTest(RESTBaseUnitTest):
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self, initRoot = False)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.Assign')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def testA_SiteWhitelist(self):
"""
_SiteWhitelist_
Test to see if we can get the siteWhitelist to work properly.
"""
secconfig = getattr(self.config, "SecurityModule")
cherrypy.server.environment = 'test'
cherrypy.tools.secmodv2 = NullAuth(secconfig)
self.config.UnitTests.views.active.rest.templates = 'templateDir'
self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot'
self.config.UnitTests.views.active.rest.wildcardKeys = {'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*',
'US*': '_US_'}
from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign
assign = Assign(config = self.config.UnitTests.views.active.rest, noSiteDB = True)
siteList = ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']
self.assertEqual(assign.sites, [])
assign.sites.extend(siteList)
Utilities.addSiteWildcards(assign.wildcardKeys,assign.sites,assign.wildcardSites)
for s in siteList:
self.assertTrue(s in assign.sites)
self.assertTrue('T1*' in assign.sites)
self.assertTrue('T2*' in assign.sites)
self.assertFalse('T3*' in assign.sites)
self.assertTrue('US*' in assign.sites)
self.assertEqual(assign.wildcardSites['T1*'], ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL'])
self.assertEqual(assign.wildcardSites['T2*'], ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
self.assertEqual(assign.wildcardSites['US*'], ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
