class ReqMgrTest(RESTBaseUnitTest):
"""
_ReqMgrTest_
Basic test for the ReqMgr services.
Setup is done off-screen in RESTBaseUnitTest - this makes
things confusing
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName,
"GroupUser", "ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
"WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
tearDown
Tear down everything
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_testBasicSetUp(self):
"""
_testBasicSetUp_
Moving the tests that were in the setUp category out of it,
mostly because I want to make sure that they don't fail inside
the setUp statement.
"""
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = self.jsonSender.get('group/PeopleLikeMe')[0]['users']
self.assertTrue('me' in users)
groups = self.jsonSender.get('user/me')[0]['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to dealwith
schema = utils.getSchema()
version = "version/" + schema["CMSSWVersion"]
self.assertTrue(self.jsonSender.put(version)[1] == 200)
self.assertTrue(schema["CMSSWVersion"] in self.jsonSender.get('version')[0])
@attr("integration")
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = utils.getAndSetupSchema(self)
schema['RequestNumEvents'] = 100
schema['RequestEventSize'] = 101
self.doRequest(schema)
def doRequest(self, schema):
"""
_doRequest_
Run all tests on a basic ReReco workflow
"""
requestName = schema['RequestName']
self.assertRaises(HTTPException, self.jsonSender.delete, 'request/%s' % requestName)
result = self.jsonSender.put('request/%s' % (requestName), schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['RequestName'], requestName)
self.jsonSender.put('request/%s?status=assignment-approved' % requestName)
me = self.jsonSender.get('user/me')[0]
self.assertTrue(requestName in me['requests'])
self.assertEqual(self.jsonSender.put('request/%s?priority=5' % requestName)[1], 200)
self.assertEqual(self.jsonSender.post('user/me?priority=6')[1], 200)
self.assertEqual(self.jsonSender.post('group/PeopleLikeMe?priority=7')[1], 200)
# default priority of group and user of 1
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], 5)
self.assertEqual(request['ReqMgrRequestorBasePriority'], 6)
self.assertEqual(request['ReqMgrGroupBasePriority'], 7)
self.assertEqual(request['RequestPriority'], 5+6+7)
# Check LFN Bases
self.assertEqual(request['UnmergedLFNBase'], '/store/unmerged')
self.assertEqual(request['MergedLFNBase'], '/store/data')
# Check random other
self.assertEqual(request['CustodialSite'], 'US_T1_FNAL')
# Check Num events
self.assertEqual(request['RequestNumEvents'], 100)
self.assertEqual(request['RequestEventSize'], 101)
# only certain transitions allowed
#self.assertEqual(self.jsonSender.put('request/%s?status=running' % requestName)[1], 400)
self.assertRaises(HTTPException, self.jsonSender.put,'request/%s?status=running' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'assignment-approved')
self.assertTrue(self.jsonSender.put(urllib.quote('assignment/White Sox/%s' % requestName))[1] == 200)
requestsAndSpecs = self.jsonSender.get(urllib.quote('assignment/White Sox'))[0]
self.assertTrue(requestName in requestsAndSpecs[0])
workloadHelper = WMWorkloadHelper()
workloadHelper.load(requestsAndSpecs[0][1])
self.assertEqual(workloadHelper.getOwner()['Requestor'], "me")
self.assertEqual(self.jsonSender.get('assignment?request=%s'% requestName)[0], ['White Sox'])
self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['teams'], ['White Sox'])
agentUrl = 'http://cmssrv96.fnal.gov/workqueue'
self.jsonSender.put('workQueue/%s?url=%s'% (requestName, urllib.quote(agentUrl)) )
self.assertEqual(self.jsonSender.get('workQueue/%s' % requestName)[0][0], agentUrl)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'acquired')
self.jsonSender.post('request/%s?events_written=10&files_merged=1' % requestName)
self.jsonSender.post('request/%s?events_written=20&files_merged=2&percent_success=99.9' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(len(request['RequestUpdates']), 2)
self.assertEqual(request['RequestUpdates'][0]['files_merged'], 1)
self.assertEqual(request['RequestUpdates'][1]['events_written'], 20)
self.assertEqual(request['RequestUpdates'][1]['percent_success'], 99.9)
message = "The sheriff is near"
jsonMessage = json.dumps(message)
self.jsonSender.put('message/%s' % requestName, message)
messages = self.jsonSender.get('message/%s' % requestName)
#self.assertEqual(messages[0][0][0], message)
for status in ['running', 'completed']:
self.jsonSender.put('request/%s?status=%s' % (requestName, status))
# campaign
self.jsonSender.put('campaign/%s' % 'TestCampaign')
campaigns = self.jsonSender.get('campaign')[0]
self.assertTrue('TestCampaign' in campaigns.keys())
self.jsonSender.put('campaign/%s/%s' % ('TestCampaign', requestName))
requestsInCampaign = self.jsonSender.get('campaign/%s' % 'TestCampaign')[0]
self.assertTrue(requestName in requestsInCampaign.keys())
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['Campaign'], 'TestCampaign')
self.jsonSender.delete('request/%s' % requestName)
@attr("integration")
def testC_404Errors(self):
"""
_404Errors_
Do some things that generate 404 errors. This
should be limited to requests for objects that do not
exist.
"""
badName = 'ThereIsNoWayThisNameShouldExist'
# First, try to find a non-existant request
# This should throw a 404 error.
# The request name should not be in it
self.checkForError(cls = 'request', badName = badName, exitCode = 404,
message = 'Given requestName not found')
# Now look for non-existant user
self.checkForError(cls = 'user', badName = badName, exitCode = 404,
message = 'Cannot find user')
# Now try non-existant group
self.checkForError(cls = 'group', badName = badName, exitCode = 404,
message = "Cannot find group/group priority")
# Now try non-existant campaign
self.checkForError(cls = 'campaign', badName = badName, exitCode = 404,
message = "Cannot find campaign")
# Now try invalid message
# This raises a requestName error becuase it searches for the request
self.checkForError(cls = 'message', badName = badName, exitCode = 404,
message = "Given requestName not found", testEmpty = False)
# Check for assignments (no teams or requests)
# This raises a team error because it tries to load teams out first
self.checkForError(cls = 'assignment', badName = badName, exitCode = 404,
message = 'Cannot find team')
@attr("integration")
def testD_400Errors(self):
"""
_400Errors_
These are failures created by invalid input, such as sending
args to a request when it doesn't accept any. They should
generatore 400 Errors
"""
badName = 'ThereIsNoWayThisNameShouldExist'
# Attempt to send arguments to a function that doesn't accept them.
self.checkForError(cls = 'team', badName = badName, exitCode = 400,
message = "Invalid input: Arguments added where none allowed")
# Recheck for versions
self.checkForError(cls = 'version', badName = badName, exitCode = 400,
message = "Invalid input: Arguments added where none allowed")
# Break the validation
self.checkForError(cls = 'user', badName = '!', exitCode = 400,
message = 'Invalid input: Input data failed validation')
def checkForError(self, cls, badName, exitCode, message, testEmpty = True):
"""
_checkForError_
Generic function for checking for errors in JSON commands
Does a basic check on type cls searching for name badName which hopefull
does not exist.
Checks to make sure that it exits with code exitCode, and that
the error contains the string message.
Also checks to make sure that name badName is NOT in the output
testEmpty for those that don't handle calls to the main (i.e., who require
an argument)
"""
raises = False
# First assert that the test to be tested is empty
if testEmpty:
result = self.jsonSender.get(cls)
self.assertTrue(type(result[0]) in [type([]), type({})])
# Next, test
try:
result = self.jsonSender.get('%s/%s' % (cls, badName))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, exitCode)
self.assertTrue(message in ex.result)
self.assertFalse(badName in ex.result)
self.assertTrue(raises)
class ReqMgrTest(RESTBaseUnitTest):
"""
_ReqMgrTest_
Basic test for the ReqMgr services.
Setup is done off-screen in RESTBaseUnitTest - this makes
things confusing
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
return
def initialize(self):
self.config = RequestManagerConfig("WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel")
self.config.setFormatter("WMCore.WebTools.RESTFormatter")
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
return
def tearDown(self):
"""
tearDown
Tear down everything
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
return
def setupSchema(
self,
groupName="PeopleLikeMe",
userName="******",
teamName="White Sox",
CMSSWVersion="CMSSW_3_5_8",
scramArch="slc5_ia32_gcc434",
):
"""
_setupSchema_
Set up a test schema so that we can run a test request.
Standardization!
"""
self.jsonSender.put("user/%[email protected]" % userName)
self.jsonSender.put("group/%s" % groupName)
self.jsonSender.put("group/%s/%s" % (groupName, userName))
self.jsonSender.put(urllib.quote("team/%s" % teamName))
self.jsonSender.put("version/%s/%s" % (CMSSWVersion, scramArch))
schema = ReReco.getTestArguments()
schema["RequestName"] = "TestReReco"
schema["RequestType"] = "ReReco"
schema["CmsPath"] = "/uscmst1/prod/sw/cms"
schema["Requestor"] = "%s" % userName
schema["Group"] = "%s" % groupName
schema["CustodialSite"] = "US_T1_FNAL"
schema["TimePerEvent"] = "12"
schema["Memory"] = 3000
schema["SizePerEvent"] = 512
return schema
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
"process": {
"outputModules_": ["ThisIsAName"],
"ThisIsAName": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
}
}
BadTweak = {
"process": {
"outputModules_": ["ThisIsAName1", "ThisIsAName2"],
"ThisIsAName1": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
"ThisIsAName2": {"dataset": {"dataTier": "RECO", "filterName": "Filter"}},
}
}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username="******")
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_testBasicSetUp(self):
"""
_testBasicSetUp_
Moving the tests that were in the setUp category out of it,
mostly because I want to make sure that they don't fail inside
the setUp statement.
"""
if "me" in self.jsonSender.get("user")[0]:
self.jsonSender.delete("user/me")
self.assertFalse("me" in self.jsonSender.get("user")[0])
self.assertEqual(self.jsonSender.put("user/[email protected]")[1], 200)
self.assertTrue("me" in self.jsonSender.get("user")[0])
if "PeopleLikeMe" in self.jsonSender.get("group")[0]:
self.jsonSender.delete("group/PeopleLikeMe")
self.assertFalse("PeopleLikeMe" in self.jsonSender.get("group")[0])
self.assertEqual(self.jsonSender.put("group/PeopleLikeMe")[1], 200)
self.assertTrue("PeopleLikeMe" in self.jsonSender.get("group")[0])
self.jsonSender.put("group/PeopleLikeMe/me")
users = self.jsonSender.get("group/PeopleLikeMe")[0]["users"]
self.assertTrue("me" in users)
groups = self.jsonSender.get("user/me")[0]["groups"]
self.assertTrue("PeopleLikeMe" in groups)
groups2 = self.jsonSender.get("group?user=me")[0]
self.assertTrue("PeopleLikeMe" in groups2)
if "White Sox" in self.jsonSender.get("team")[0]:
self.jsonSender.delete(urllib.quote("team/White Sox"))
self.assertFalse("White Sox" in self.jsonSender.get("team")[0])
self.assertEqual(self.jsonSender.put(urllib.quote("team/White Sox"))[1], 200)
self.assertTrue("White Sox" in self.jsonSender.get("team")[0])
# some foreign key stuff to dealwith
self.assertTrue(self.jsonSender.put("version/CMSSW_3_5_8")[1] == 200)
self.assertTrue("CMSSW_3_5_8" in self.jsonSender.get("version")[0])
return
@attr("integration")
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = self.setupSchema()
schema["RequestNumEvents"] = 100
schema["RequestEventSize"] = 101
self.doRequest(schema)
return
def doRequest(self, schema):
"""
_doRequest_
Run all tests on a basic ReReco workflow
"""
requestName = schema["RequestName"]
self.assertRaises(HTTPException, self.jsonSender.delete, "request/%s" % requestName)
result = self.jsonSender.put("request/%s" % (requestName), schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
self.assertEqual(self.jsonSender.get("request/%s" % requestName)[0]["RequestName"], requestName)
self.jsonSender.put("request/%s?status=assignment-approved" % requestName)
me = self.jsonSender.get("user/me")[0]
self.assertTrue(requestName in me["requests"])
self.assertEqual(self.jsonSender.put("request/%s?priority=5" % requestName)[1], 200)
self.assertEqual(self.jsonSender.post("user/me?priority=6")[1], 200)
self.assertEqual(self.jsonSender.post("group/PeopleLikeMe?priority=7")[1], 200)
# default priority of group and user of 1
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(request["ReqMgrRequestBasePriority"], 5)
self.assertEqual(request["ReqMgrRequestorBasePriority"], 6)
self.assertEqual(request["ReqMgrGroupBasePriority"], 7)
self.assertEqual(request["RequestPriority"], 5 + 6 + 7)
# Check LFN Bases
self.assertEqual(request["UnmergedLFNBase"], "/store/unmerged")
self.assertEqual(request["MergedLFNBase"], "/store/data")
# Check random other
self.assertEqual(request["CustodialSite"], "US_T1_FNAL")
# Check Num events
self.assertEqual(request["RequestNumEvents"], 100)
self.assertEqual(request["RequestEventSize"], 101)
# only certain transitions allowed
# self.assertEqual(self.jsonSender.put('request/%s?status=running' % requestName)[1], 400)
self.assertRaises(HTTPException, self.jsonSender.put, "request/%s?status=running" % requestName)
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(request["RequestStatus"], "assignment-approved")
self.assertTrue(self.jsonSender.put(urllib.quote("assignment/White Sox/%s" % requestName))[1] == 200)
requestsAndSpecs = self.jsonSender.get(urllib.quote("assignment/White Sox"))[0]
self.assertTrue(requestName in requestsAndSpecs[0])
workloadHelper = WMWorkloadHelper()
workloadHelper.load(requestsAndSpecs[0][1])
self.assertEqual(workloadHelper.getOwner()["Requestor"], "me")
self.assertEqual(self.jsonSender.get("assignment?request=%s" % requestName)[0], ["White Sox"])
self.assertEqual(self.jsonSender.get("request/%s" % requestName)[0]["teams"], ["White Sox"])
agentUrl = "http://cmssrv96.fnal.gov/workqueue"
self.jsonSender.put("workQueue/%s?url=%s" % (requestName, urllib.quote(agentUrl)))
self.assertEqual(self.jsonSender.get("workQueue/%s" % requestName)[0][0], agentUrl)
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(request["RequestStatus"], "acquired")
self.jsonSender.post("request/%s?events_written=10&files_merged=1" % requestName)
self.jsonSender.post("request/%s?events_written=20&files_merged=2&percent_success=99.9" % requestName)
request = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(len(request["RequestUpdates"]), 2)
self.assertEqual(request["RequestUpdates"][0]["files_merged"], 1)
self.assertEqual(request["RequestUpdates"][1]["events_written"], 20)
self.assertEqual(request["RequestUpdates"][1]["percent_success"], 99.9)
message = "The sheriff is near"
jsonMessage = json.dumps(message)
self.jsonSender.put("message/%s" % requestName, message)
messages = self.jsonSender.get("message/%s" % requestName)
# self.assertEqual(messages[0][0][0], message)
for status in ["running", "completed"]:
self.jsonSender.put("request/%s?status=%s" % (requestName, status))
# campaign
self.jsonSender.put("campaign/%s" % "TestCampaign")
campaigns = self.jsonSender.get("campaign")[0]
self.assertTrue("TestCampaign" in campaigns.keys())
self.jsonSender.put("campaign/%s/%s" % ("TestCampaign", requestName))
requestsInCampaign = self.jsonSender.get("campaign/%s" % "TestCampaign")[0]
self.assertTrue(requestName in requestsInCampaign.keys())
req = self.jsonSender.get("request/%s" % requestName)[0]
self.assertEqual(req["Campaign"], "TestCampaign")
self.jsonSender.delete("request/%s" % requestName)
return
@attr("integration")
def testC_404Errors(self):
"""
_404Errors_
Do some things that generate 404 errors. This
should be limited to requests for objects that do not
exist.
"""
badName = "ThereIsNoWayThisNameShouldExist"
# First, try to find a non-existant request
# This should throw a 404 error.
# The request name should not be in it
self.checkForError(cls="request", badName=badName, exitCode=404, message="Given requestName not found")
# Now look for non-existant user
self.checkForError(cls="user", badName=badName, exitCode=404, message="Cannot find user")
# Now try non-existant group
self.checkForError(cls="group", badName=badName, exitCode=404, message="Cannot find group/group priority")
# Now try non-existant campaign
self.checkForError(cls="campaign", badName=badName, exitCode=404, message="Cannot find campaign")
# Now try invalid message
# This raises a requestName error becuase it searches for the request
self.checkForError(
cls="message", badName=badName, exitCode=404, message="Given requestName not found", testEmpty=False
)
# Check for assignments (no teams or requests)
# This raises a team error because it tries to load teams out first
self.checkForError(cls="assignment", badName=badName, exitCode=404, message="Cannot find team")
return
@attr("integration")
def testD_400Errors(self):
"""
_400Errors_
These are failures created by invalid input, such as sending
args to a request when it doesn't accept any. They should
generatore 400 Errors
"""
badName = "ThereIsNoWayThisNameShouldExist"
# Attempt to send arguments to a function that doesn't accept them.
self.checkForError(
cls="team", badName=badName, exitCode=400, message="Invalid input: Arguments added where none allowed"
)
# Recheck for versions
self.checkForError(
cls="version", badName=badName, exitCode=400, message="Invalid input: Arguments added where none allowed"
)
# Break the validation
self.checkForError(cls="user", badName="!", exitCode=400, message="Invalid input: Input data failed validation")
return
def checkForError(self, cls, badName, exitCode, message, testEmpty=True):
"""
_checkForError_
Generic function for checking for errors in JSON commands
Does a basic check on type cls searching for name badName which hopefull
does not exist.
Checks to make sure that it exits with code exitCode, and that
the error contains the string message.
Also checks to make sure that name badName is NOT in the output
testEmpty for those that don't handle calls to the main (i.e., who require
an argument)
"""
raises = False
# First assert that the test to be tested is empty
if testEmpty:
result = self.jsonSender.get(cls)
self.assertTrue(type(result[0]) in [type([]), type({})])
# Next, test
try:
result = self.jsonSender.get("%s/%s" % (cls, badName))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, exitCode)
self.assertTrue(message in ex.result)
self.assertFalse(badName in ex.result)
self.assertTrue(raises)
return
class ReqMgrTest(RESTBaseUnitTest):
"""
Basic test for the ReqMgr services.
Setup is done off-screen in RESTBaseUnitTest - this makes
things confusing
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
"WMStats")
self.testInit.setupCouch("%s_acdc" % self.couchDBName,
"ACDC", "GroupUser")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
self.params = {}
self.params['endpoint'] = reqMgrHost
self.reqService = RequestManager(self.params)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
tearDown
Tear down everything
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_testBasicSetUp(self):
"""
_testBasicSetUp_
Moving the tests that were in the setUp category out of it,
mostly because I want to make sure that they don't fail inside
the setUp statement.
"""
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = self.jsonSender.get('group/PeopleLikeMe')[0]['users']
self.assertTrue('me' in users)
groups = self.jsonSender.get('user/me')[0]['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to deal with
schema = utils.getSchema()
version = "version/" + schema["CMSSWVersion"]
self.assertTrue(self.jsonSender.put(version)[1] == 200)
self.assertTrue(schema["CMSSWVersion"] in self.jsonSender.get('version')[0])
@attr("integration")
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = utils.getAndSetupSchema(self)
schema['RequestNumEvents'] = 100
schema['SizePerEvent'] = 101
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
self.doRequest(schema)
def doRequest(self, schema):
"""
_doRequest_
Run all tests on a basic ReReco workflow
"""
requestName = schema['RequestName']
self.assertRaises(HTTPException, self.jsonSender.delete, 'request/%s' % requestName)
result = self.jsonSender.put('request/%s' % (requestName), schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['RequestName'], requestName)
self.jsonSender.put('request/%s?status=assignment-approved' % requestName)
me = self.jsonSender.get('user/me')[0]
self.assertTrue(requestName in me['requests'])
self.assertEqual(self.jsonSender.put('request/%s?priority=5' % requestName)[1], 200)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], 5)
# Check LFN Bases
self.assertEqual(request['UnmergedLFNBase'], '/store/unmerged')
self.assertEqual(request['MergedLFNBase'], '/store/data')
# Check Num events
self.assertEqual(request['RequestNumEvents'], 100)
self.assertEqual(request['SizePerEvent'], 101)
# only certain transitions allowed
#self.assertEqual(self.jsonSender.put('request/%s?status=running' % requestName)[1], 400)
self.assertRaises(HTTPException, self.jsonSender.put,'request/%s?status=running' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'assignment-approved')
self.assertTrue(self.jsonSender.put(urllib.quote('assignment/White Sox/%s' % requestName))[1] == 200)
requestsAndSpecs = self.jsonSender.get(urllib.quote('assignment/White Sox'))[0]
self.assertTrue(requestName in requestsAndSpecs[0])
workloadHelper = WMWorkloadHelper()
workloadHelper.load(requestsAndSpecs[0][1])
self.assertEqual(workloadHelper.getOwner()['Requestor'], "me")
self.assertEqual(self.jsonSender.get('assignment?request=%s'% requestName)[0], ['White Sox'])
self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['teams'], ['White Sox'])
agentUrl = 'http://cmssrv96.fnal.gov/workqueue'
self.jsonSender.put('workQueue/%s?url=%s'% (requestName, urllib.quote(agentUrl)) )
self.assertEqual(self.jsonSender.get('workQueue/%s' % requestName)[0][0], agentUrl)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'acquired')
self.jsonSender.post('request/%s?events_written=10&files_merged=1' % requestName)
self.jsonSender.post('request/%s?events_written=20&files_merged=2&percent_success=99.9' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(len(request['RequestUpdates']), 2)
self.assertEqual(request['RequestUpdates'][0]['files_merged'], 1)
self.assertEqual(request['RequestUpdates'][1]['events_written'], 20)
self.assertEqual(request['RequestUpdates'][1]['percent_success'], 99.9)
message = "The sheriff is near"
jsonMessage = json.dumps(message)
self.jsonSender.put('message/%s' % requestName, message)
messages = self.jsonSender.get('message/%s' % requestName)
#self.assertEqual(messages[0][0][0], message)
for status in ['running-open', 'running-closed', 'completed']:
self.jsonSender.put('request/%s?status=%s' % (requestName, status))
# campaign
self.jsonSender.put('campaign/%s' % 'TestCampaign')
campaigns = self.jsonSender.get('campaign')[0]
self.assertTrue('TestCampaign' in campaigns.keys())
self.jsonSender.put('campaign/%s/%s' % ('TestCampaign', requestName))
requestsInCampaign = self.jsonSender.get('campaign/%s' % 'TestCampaign')[0]
self.assertTrue(requestName in requestsInCampaign.keys())
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['Campaign'], 'TestCampaign')
self.jsonSender.delete('request/%s' % requestName)
@attr("integration")
def testC_404Errors(self):
"""
_404Errors_
Do some things that generate 404 errors. This
should be limited to requests for objects that do not
exist.
"""
badName = 'ThereIsNoWayThisNameShouldExist'
# First, try to find a non-existent request
# This should throw a 404 error.
# The request name should not be in it
self.checkForError(cls = 'request', badName = badName, exitCode = 404,
message = 'Given requestName not found')
# Now look for non-existent user
self.checkForError(cls = 'user', badName = badName, exitCode = 404,
message = 'Cannot find user')
# Now try non-existent campaign
self.checkForError(cls = 'campaign', badName = badName, exitCode = 404,
message = "Cannot find campaign")
# Now try invalid message
# This raises a requestName error because it searches for the request
self.checkForError(cls = 'message', badName = badName, exitCode = 404,
message = "Given requestName not found", testEmpty = False)
# Check for assignments (no teams or requests)
# This raises a team error because it tries to load teams out first
self.checkForError(cls = 'assignment', badName = badName, exitCode = 404,
message = 'Cannot find team')
@attr("integration")
def testD_400Errors(self):
"""
_400Errors_
These are failures created by invalid input, such as sending
args to a request when it doesn't accept any. They should
generatore 400 Errors
"""
badName = 'ThereIsNoWayThisNameShouldExist'
# Attempt to send arguments to a function that doesn't accept them.
self.checkForError(cls = 'team', badName = badName, exitCode = 400,
message = "Invalid input: Arguments added where none allowed")
# Recheck for versions
self.checkForError(cls = 'version', badName = badName, exitCode = 400,
message = "Invalid input: Arguments added where none allowed")
# Break the validation
self.checkForError(cls = 'user', badName = '!', exitCode = 400,
message = 'Invalid input: Input data failed validation')
def checkForError(self, cls, badName, exitCode, message, testEmpty = True):
"""
_checkForError_
Generic function for checking for errors in JSON commands
Does a basic check on type cls searching for name badName which hopefull
does not exist.
Checks to make sure that it exits with code exitCode, and that
the error contains the string message.
Also checks to make sure that name badName is NOT in the output
testEmpty for those that don't handle calls to the main (i.e., who require
an argument)
"""
raises = False
# First assert that the test to be tested is empty
if testEmpty:
result = self.jsonSender.get(cls)
self.assertTrue(type(result[0]) in [type([]), type({})])
# Next, test
try:
result = self.jsonSender.get('%s/%s' % (cls, badName))
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, exitCode)
self.assertTrue(message in ex.result)
self.assertFalse(badName in ex.result)
self.assertTrue(raises)
@attr("integration")
def testE_CheckStatusChanges(self):
"""
_CheckStatusChanges_
Check status changes for a single request. See whether
we can move the request through the proper chain. Figure
out what happens when we fail.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
# There should only be one request in the DB
result = GetRequest.requestID(requestName = requestName)
self.assertEqual(result, 1)
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['Group'], groupName)
self.assertEqual(result[0]['Requestor'], userName)
# Let's see what we can do in terms of setting status
self.changeStatusAndCheck(requestName = requestName,
statusName = 'new')
# Let's try an illegal status change, just for the hell of it
raises = False
try:
self.jsonSender.put('request/%s?status=assigned' % requestName)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
self.assertTrue(raises)
# Now, let's try a totally bogus status
raises = False
try:
self.jsonSender.put('request/%s?status=bogus' % requestName)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
self.assertTrue(raises)
# We should still be in new
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], 'new')
# Let's go on in a full loop
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing-approved')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'tested')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assignment-approved')
# This should fail, as you cannot assign a request without a team
raises = False
try:
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assigned')
except HTTPException as ex:
raises = True
self.assertTrue('Cannot change status without a team' in ex.result)
self.assertTrue(raises)
self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName)))
self.changeStatusAndCheck(requestName = requestName,
statusName = 'acquired')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'running-open')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'running-closed')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'completed')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'closed-out')
def changeStatusAndCheck(self, requestName, statusName):
"""
_changeStatusAndCheck_
Change the status of a request and make sure that
the request actually did it.
"""
self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], statusName)
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def testF_TestWhitelistBlacklist(self):
"""
_TestWhitelistBlacklist_
Test whether or not we can assign the block/run blacklist/whitelist
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RunWhitelist'] = [1, 2, 3]
schema['RunBlacklist'] = [4, 5, 6]
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema['RunWhitelist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema['RunBlacklist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema['BlockWhitelist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema['BlockBlacklist'])
req = self.jsonSender.get('request/%s' % requestName)
self.assertTrue('Site Blacklist' in req[0])
self.assertTrue('Site Whitelist' in req[0])
schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'}
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
print(ex.result)
self.assertTrue("Error in Workload Validation: Argument BlockBlacklist type is incorrect in schema." in ex.result)
pass
self.assertTrue(raises)
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
schema['RunWhitelist'] = {'1': '/dataset/dataset/dataset#beta'}
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument RunWhitelist type is incorrect in schema." in ex.result)
pass
self.assertTrue(raises)
schema['RunWhitelist'] = ['hello', 'how', 'are', 'you']
try:
raises = True
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument RunWhitelist doesn't pass validation." in ex.result)
pass
self.assertTrue(raises)
def testG_AddDuplicateUser(self):
"""
_AddDuplicateUser_
Test and see if we get a sensible error when adding a duplicate user.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
raises = False
try:
self.jsonSender.put('group/%s/%s' % (groupName, userName))
except HTTPException as ex:
self.assertTrue("User/Group Already Linked in DB" in ex.result)
self.assertEqual(ex.status, 400)
raises = True
self.assertTrue(raises)
def testH_RemoveSoftwareVersion(self):
"""
_RemoveSoftwareVersion_
Remove the software version after submitting the request. See what that does.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
# Delete software versions and make sure they're gone from the DB
SoftwareManagement.removeSoftware(softwareName = schema["CMSSWVersion"],
scramArch = schema["ScramArch"])
versions = myThread.dbi.processData("SELECT * FROM reqmgr_software")[0].fetchall()
self.assertEqual(versions, [])
assocs = myThread.dbi.processData("SELECT * FROM reqmgr_software_dependency")[0].fetchall()
self.assertEqual(assocs, [])
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
def testI_CheckConfigIDs(self):
"""
_CheckConfigIDs_
Check to see if we can pull out the ConfigIDs by request
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
# Set some versions
schema['ProcessingVersion'] = '2012'
schema['AcquisitionEra'] = 'ae2012'
schema["PrimaryDataset"] = "ReallyFake"
schema["RequestNumEvents"] = 100
configID = self.createConfig()
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["ConfigCacheID"] = configID
schema["InputDataset"] = '/MinimumBias/Run2010B-RelValRawSkim-v1/RAW'
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
result = self.jsonSender.get('configIDs?prim=MinimumBias&proc=Run2010B-RelValRawSkim-v1&tier=RAW')[0]
print(result)
self.assertTrue(requestName in result.keys())
self.assertTrue(configID in result[requestName][0])
def testJ_CheckRequestCloning(self):
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["AcquisitionEra"] = "NewEra"
result = self.jsonSender.put("request", schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
acquisitionEra = result[0]["AcquisitionEra"]
self.assertTrue(schema["AcquisitionEra"], acquisitionEra)
# set some non-default priority
# when cloning a request which had some non default priority,
# the priority values were lost when creating a cloned request, the
# default values were lost. Change it here to specifically catch this case.
priority = 300
result = self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))
self.assertEqual(result[1], 200)
# get the original request from the server, although the variable result
# shall have the same stuff in
response = self.jsonSender.get("request/%s" % requestName)
origRequest = response[0]
self.assertEqual(origRequest["AcquisitionEra"], acquisitionEra)
# test that the priority was correctly set in the brand-new request
self.assertEqual(origRequest["RequestPriority"], priority)
# test cloning not existing request
self.assertRaises(HTTPException, self.jsonSender.put, "clone/%s" % "NotExistingRequestName")
# correct attempt to clone the request
# this is the new request, it'll have different name
result = self.jsonSender.put("clone/%s" % requestName)
# get the cloned request from the server
respose = self.jsonSender.get("request/%s" % result[0]["RequestName"])
clonedRequest = respose[0]
# these request arguments shall differ in the cloned request:
toDiffer = ["RequestName", "RequestStatus"]
for differ in toDiffer:
self.assertNotEqual(origRequest[differ], clonedRequest[differ])
# check the desired status of the cloned request
self.assertEqual(clonedRequest["RequestStatus"], "assignment-approved",
"Cloned request status should be 'assignment-approved', not '%s'." %
clonedRequest["RequestStatus"])
# don't care about these two (they will likely be the same in the unittest
# since the brand new request injection as well as the cloning probably
# happen at roughly the same time)
toDiffer.extend(["RequestDate", "timeStamp", "RequestWorkflow"])
for differ in toDiffer:
del origRequest[differ]
del clonedRequest[differ]
# check the request dictionaries
self.assertEqual(len(origRequest), len(clonedRequest))
for k1, k2 in zip(sorted(origRequest.keys()), sorted(clonedRequest.keys())):
msg = ("Request values: original: %s: %s cloned: %s: %s differ" %
(k1, origRequest[k1], k2, clonedRequest[k2]))
self.assertEqual(origRequest[k1], clonedRequest[k2], msg)
def testK_CheckRequestFailsInjectionForbiddenInputArg(self):
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
from WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel import deprecatedRequestArgs
for deprec in deprecatedRequestArgs:
schema = utils.getSchema(groupName=groupName, userName=userName)
schema[deprec] = "something"
self.assertRaises(HTTPException, self.jsonSender.put, "request", schema)
def setupACDCDatabase(self, collectionName, taskPath,
user, group):
"""
_setupACDCDatabase_
Populate an ACDC database with bogus records
associated to certain collection name, user and task path.
"""
acdcServer = CouchService(url = self.testInit.couchUrl,
database = "%s_acdc" % self.couchDBName)
owner = acdcServer.newOwner(group, user)
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = collectionName)
testCollection.setOwner(owner)
testFileset = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = taskPath)
testCollection.addFileset(testFileset)
testFiles = []
for _ in range(5):
testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
events = random.randint(1024, 4096))
testFiles.append(testFile)
testFileset.add(testFiles)
def testL_CascadeCloseOutAnnnouncement(self):
"""
_testL_CascadeCloseOutAnnouncement_
Test the cascade closeout REST call, also
check that when announced a request deletes all ACDC records in the system.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
result['Requestor'], result['Group'])
depth = 2
nReq = 3
requests = [originalRequest]
def createChildrenRequest(parentRequest, i, nReq):
createdRequests = []
resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
groupName, userName)
result = self.jsonSender.put("request", resubSchema)[0]
requestName = result['RequestName']
self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
createdRequests.append(requestName)
if i:
for _ in range(nReq):
createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
return createdRequests
requests.extend(createChildrenRequest(originalRequest, depth, nReq))
for request in requests:
self.changeStatusAndCheck(request, 'assignment-approved')
for request in requests:
self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
for status in ['acquired',
'running-open', 'running-closed',
'completed']:
for request in requests:
self.changeStatusAndCheck(request, status)
self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
svc = CouchService(url = self.testInit.couchUrl,
database = "%s_acdc" % self.couchDBName)
owner = svc.newOwner(groupName, userName)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'closed-out')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertNotEqual(len(testCollection["filesets"]), 0)
self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'announced')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertEqual(len(testCollection["filesets"]), 0)
def testM_PutRequestStats(self):
userName = '******'
groupName = 'Bryant'
teamName = 'Lakers'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
stats = {'total_jobs': 100, 'input_events': 100, 'input_lumis': 100, 'input_num_files': 100}
result = self.reqService.putRequestStats(originalRequest, stats)
self.assertEqual(result['RequestName'], originalRequest)
class TestReqMgr(RESTBaseUnitTest):
"""
_TestReqMgr_
Basic test for the ReqMgr services
"""
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
dbUrl = os.environ.get("DATABASE", None)
if dbUrl == None:
raise RuntimeError, """The DATABASE environment is not set. WARNING: setting this variable will cause the database to be deleted during cleanup! """
self.config.setDBUrl(dbUrl)
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
# mysql example
#self.config.setDBUrl('mysql://[email protected]:3306/TestDB')
#self.config.setDBSocket('/var/lib/mysql/mysql.sock')
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
# Do this is you don't want to remake and delete the schema
#self.schemaModules = []
def setUp(self):
"""
setUP global values
"""
RESTBaseUnitTest.setUp(self)
reqMgrHost = self.config.getServerUrl()
self.requestSchema = getRequestSchema()
print reqMgrHost
self.jsonSender = JSONRequests(reqMgrHost)
#self.requestTypes = ['ReReco', 'StoreResults', 'CmsGen', 'Reco']
#self.requestTypes = ['ReReco', 'MonteCarlo']
self.requestTypes = ['ReReco']
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = json.loads(self.jsonSender.get('group/PeopleLikeMe')[0])['users']
self.assertTrue('me' in users)
groups = json.loads(self.jsonSender.get('user/me')[0])['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to dealwith
#self.assertFalse('CMSSW_X_Y_Z' in self.jsonSender.get('version')[0])
self.assertTrue(self.jsonSender.put('version/CMSSW_3_5_8')[1] == 200)
self.assertTrue('CMSSW_3_5_8' in self.jsonSender.get('version')[0])
@attr("integration")
def testReReco(self):
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = 'ReReco'
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
self.doRequest(schema)
def doRequest(self, schema):
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = 'me'
schema['Group'] = 'PeopleLikeMe'
requestName = schema['RequestName']
self.assertRaises(HTTPException, self.jsonSender.delete, 'request/%s' % requestName)
self.assertEqual(self.jsonSender.put('request/%s' % requestName, schema)[1], 200)
self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['RequestName'], requestName)
self.assertTrue(requestName in self.jsonSender.get('user/me')[0])
self.jsonSender.put('request/%s?status=assignment-approved' % requestName)
meJSON = self.jsonSender.get('user/me')[0]
me = json.loads(meJSON)
self.assertTrue(requestName in me['requests'])
self.assertEqual(self.jsonSender.put('request/%s?priority=5' % requestName)[1], 200)
self.assertEqual(self.jsonSender.post('user/me?priority=6')[1], 200)
self.assertEqual(self.jsonSender.post('group/PeopleLikeMe?priority=7')[1], 200)
# default priority of group and user of 1
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], 5)
self.assertEqual(request['ReqMgrRequestorBasePriority'], 6)
self.assertEqual(request['ReqMgrGroupBasePriority'], 7)
self.assertEqual(request['RequestPriority'], 5+6+7)
# only certain transitions allowed
#self.assertEqual(self.jsonSender.put('request/%s?status=running' % requestName)[1], 400)
self.assertRaises(HTTPException, self.jsonSender.put,'request/%s?status=running' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'assignment-approved')
self.assertTrue(self.jsonSender.put(urllib.quote('assignment/White Sox/%s' % requestName))[1] == 200)
requestsAndSpecs = self.jsonSender.get(urllib.quote('assignment/White Sox'))[0]
self.assertTrue(requestName in requestsAndSpecs.keys())
#workloadHelper = WMWorkloadCache.loadFromURL(requestsAndSpecs[requestName])
workloadHelper = WMWorkloadHelper()
workloadHelper.load(requestsAndSpecs[requestName])
self.assertEqual(workloadHelper.getOwner()['Requestor'], "me")
self.assertTrue(self.jsonSender.get('assignment?request=%s'% requestName)[0] == ['White Sox'])
agentUrl = 'http://cmssrv96.fnal.gov/workqueue'
self.jsonSender.put('workQueue/%s?url=%s'% (requestName, urllib.quote(agentUrl)) )
self.assertEqual(self.jsonSender.get('workQueue/%s' % requestName)[0][0], agentUrl)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'acquired')
self.jsonSender.post('request/%s?events_written=10&files_merged=1' % requestName)
self.jsonSender.post('request/%s?events_written=20&files_merged=2&percent_success=99.9' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(len(request['RequestUpdates']), 2)
self.assertEqual(request['RequestUpdates'][0]['files_merged'], 1)
self.assertEqual(request['RequestUpdates'][1]['events_written'], 20)
self.assertEqual(request['RequestUpdates'][1]['percent_success'], 99.9)
message = "The sheriff is near"
jsonMessage = json.dumps(message)
self.jsonSender.put('message/%s' % requestName, message)
messages = self.jsonSender.get('message/%s' % requestName)
#self.assertEqual(messages[0][0][0], message)
for status in ['running', 'completed']:
self.jsonSender.put('request/%s?status=%s' % (requestName, status))
# campaign
self.jsonSender.put('campaign/%s' % 'TestCampaign')
campaigns = self.jsonSender.get('campaign')[0]
self.assertTrue('TestCampaign' in campaigns.keys())
self.jsonSender.put('campaign/%s/%s' % ('TestCampaign', requestName))
requestsInCampaign = self.jsonSender.get('campaign/%s' % 'TestCampaign')[0]
self.assertTrue(requestName in requestsInCampaign.keys())
self.jsonSender.delete('request/%s' % requestName)
class ReqMgrTester(object):
def __init__(self, reqMgrUrl):
self.reqMgrUrl = reqMgrUrl
self.restSender = JSONRequests(reqMgrUrl)
d = dict(endpoint = self.reqMgrUrl)
self.reqMgrService = RequestManager(d)
def queryAllRequests(self):
"""
Returns all requests stored at ReqMgr instance.
"""
logging.info("Querying all requests at ReqMgr instance ...")
r = self.reqMgrService.getRequestNames()
print "Found %s requests:" % len(r)
for req in r:
print req
def queryRequest(self, requestName):
"""
Query a specific request according to the input argument.
"""
urlQuery = "request/%s" % requestName
logging.info("Querying request '%s'" % requestName)
logging.info("Query: '%s':" % urlQuery)
r = self.restSender.get(urlQuery)
print str(r)
def createRequests(self, numRequests):
"""
Inject new numRequests into ReqMgr instance.
(see ReqMgr_t testE how to create a request)
"""
logging.info("Creating %s new requests ..." % numRequests)
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = 'ReReco'
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = '%s' % "testinguser"
schema['Group'] = '%s' % "PeopleLikeMe"
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
schema['Campaign'] = 'MyTestCampaign'
for i in range(numRequests):
urlQuery = "request/testRequest"
logging.info("Query: '%s':" % urlQuery)
r = self.restSender.put(urlQuery, schema)
# print "request creating response: ", r
print "created: ", r[0]["RequestName"]
def deleteRequest(self, requestNames):
"""
Delete requests specified in the input, more request names
are comma-separated.
"""
logging.info("Deleting requests ...")
for reqName in requestNames.split(','):
reqName = reqName.strip()
urlQuery = "request/%s" % reqName
logging.info("Deleting request (request_name): '%s'" % reqName)
logging.info("Query: '%s':" % urlQuery)
r = self.restSender.delete(urlQuery)
def injectOpsClipboard(self, reqName, couchUrl, couchDbName):
"""
Once a request reaches "ops-hold" state, it can be injected into
CouchDB, application OpsClipboard, for further manipulation.
Do this here with the reqName request.
OpsClipboard.inject() method which does in the CouchDB injection
is called from WMCore/HTTPFrontEnd/RequestManager/Assign.py
handleAssignmentPage method (which currently, 2012-01, doesn't
have any unittest nor REST API) (used only from the Assignment webpage)
Works when running locally accessing CouchDB behind frontend:
py test/data/ReqMgr/reqmgr-load_example_data.py -u https://localhost:2000/reqmgr/reqMgr/ \
-t testinguser_120131_213320_2161 -i -o https://localhost:2000/couchdb/ \
-a ops_clipboard
"""
# find out campaign name associated with this request
r = self.restSender.get("request/%s" % reqName)
campaign = r[0]["Campaign"]
logging.info("Campaign: %s" % campaign)
requests = [{u"RequestName": reqName, u"CampaignName": campaign}]
OpsClipboard.inject(couchUrl, couchDbName, *requests)
def requestChangeStates(self, reqName, injectOpsClipboard, couchUrl, couchDbName):
"""
Route the request (spec. by the request name) in the input
through a series of possible request states.
"""
logging.info("Changing state of request %s ..." % reqName)
def changeState(requestName, urlQuery):
logging.info("Query: '%s' ..." % urlQuery)
r = self.restSender.put(urlQuery)
r = self.restSender.get("request/%s" % requestName)
#assert r[0]["RequestStatus"] == statusName
logging.info("Querying modified request, new state: %s" % r[0]["RequestStatus"])
# once a request is created, it's in 'new' state
# states transition has to be an allowed one as defined here:
# WMCore/RequestManager/RequestDB/Settings/RequestStatus.py
statesQueries = ["request/%s?status=%s" % (reqName, "testing-approved"),
"request/%s?status=%s" % (reqName, "testing"),
"request/%s?status=%s" % (reqName, "tested"),
"request/%s?status=%s" % (reqName, "assignment-approved"),
# create an assignment now
# need quote because of space in the team name
urllib.quote("assignment/%s/%s" % (TEAM_NAME, reqName)),
"request/%s?status=%s" % (reqName, "ops-hold")]
for query in statesQueries:
changeState(reqName, query)
if injectOpsClipboard:
self.injectOpsClipboard(reqName, couchUrl, couchDbName)
def setup(self):
"""
Setup ReqMgr instance for dealing with requests - needs to create
a user, group, SW releases entries, etc.
as done in test/python/WMCore_t/RequestManager_t/ReqMgr_t.py
"""
logging.info("ReqMgr setup ...")
def doQuery(urlQuery):
logging.info("Query: '%s' ..." % urlQuery)
r = None
try:
r = self.restSender.put(urlQuery)
except Exception as ex:
print "exception"
print str(ex)
print "response:", r
queries = ["user/[email protected]",
"group/PeopleLikeMe",
"group/PeopleLikeMe/testinguser",
urllib.quote("team/" + TEAM_NAME),
"version/%s" % "CMSSW_3_5_8"]
for q in queries:
doQuery(q)
logging.info("ReqMgr setup finished, listing known users ...")
q = "user/"
r = self.restSender.get(q)
print r
class ReqMgrTest(RESTBaseUnitTest):
"""
Basic test for the ReqMgr services.
Setup is done off-screen in RESTBaseUnitTest - this makes
things confusing
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName,
"WMStats")
self.testInit.setupCouch("%s_acdc" % self.couchDBName,
"ACDC", "GroupUser")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
self.params = {}
self.params['endpoint'] = reqMgrHost
self.reqService = RequestManager(self.params)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName = self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
tearDown
Tear down everything
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad = False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {'process': {'outputModules_': ['ThisIsAName'],
'ThisIsAName': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}},
'ThisIsAName2': {'dataset': {'dataTier': 'RECO',
'filterName': 'Filter'}}}}
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName)
configCache.createUserGroup(groupname = "testGroup", username = '******')
if bad:
configCache.setPSetTweaks(PSetTweak = BadTweak)
else:
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_testBasicSetUp(self):
"""
_testBasicSetUp_
Moving the tests that were in the setUp category out of it,
mostly because I want to make sure that they don't fail inside
the setUp statement.
"""
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = self.jsonSender.get('group/PeopleLikeMe')[0]['users']
self.assertTrue('me' in users)
groups = self.jsonSender.get('user/me')[0]['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to deal with
schema = utils.getSchema()
version = "version/" + schema["CMSSWVersion"]
self.assertTrue(self.jsonSender.put(version)[1] == 200)
self.assertTrue(schema["CMSSWVersion"] in self.jsonSender.get('version')[0])
@attr("integration")
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = utils.getAndSetupSchema(self)
schema['RequestNumEvents'] = 100
schema['SizePerEvent'] = 101
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
self.doRequest(schema)
def doRequest(self, schema):
"""
_doRequest_
Run all tests on a basic ReReco workflow
"""
requestName = schema['RequestName']
self.assertRaises(HTTPException, self.jsonSender.delete, 'request/%s' % requestName)
result = self.jsonSender.put('request/%s' % (requestName), schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['RequestName'], requestName)
self.jsonSender.put('request/%s?status=assignment-approved' % requestName)
me = self.jsonSender.get('user/me')[0]
self.assertTrue(requestName in me['requests'])
self.assertEqual(self.jsonSender.put('request/%s?priority=5' % requestName)[1], 200)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestPriority'], 5)
# Check LFN Bases
self.assertEqual(request['UnmergedLFNBase'], '/store/unmerged')
self.assertEqual(request['MergedLFNBase'], '/store/data')
# Check Num events
self.assertEqual(request['RequestNumEvents'], 100)
self.assertEqual(request['SizePerEvent'], 101)
# only certain transitions allowed
#self.assertEqual(self.jsonSender.put('request/%s?status=running' % requestName)[1], 400)
self.assertRaises(HTTPException, self.jsonSender.put,'request/%s?status=running' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'assignment-approved')
self.assertTrue(self.jsonSender.put(urllib.quote('assignment/White Sox/%s' % requestName))[1] == 200)
requestsAndSpecs = self.jsonSender.get(urllib.quote('assignment/White Sox'))[0]
self.assertTrue(requestName in requestsAndSpecs[0])
workloadHelper = WMWorkloadHelper()
workloadHelper.load(requestsAndSpecs[0][1])
self.assertEqual(workloadHelper.getOwner()['Requestor'], "me")
self.assertEqual(self.jsonSender.get('assignment?request=%s'% requestName)[0], ['White Sox'])
self.assertEqual(self.jsonSender.get('request/%s' % requestName)[0]['teams'], ['White Sox'])
agentUrl = 'http://cmssrv96.fnal.gov/workqueue'
self.jsonSender.put('workQueue/%s?url=%s'% (requestName, urllib.quote(agentUrl)) )
self.assertEqual(self.jsonSender.get('workQueue/%s' % requestName)[0][0], agentUrl)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'acquired')
self.jsonSender.post('request/%s?events_written=10&files_merged=1' % requestName)
self.jsonSender.post('request/%s?events_written=20&files_merged=2&percent_success=99.9' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(len(request['RequestUpdates']), 2)
self.assertEqual(request['RequestUpdates'][0]['files_merged'], 1)
self.assertEqual(request['RequestUpdates'][1]['events_written'], 20)
self.assertEqual(request['RequestUpdates'][1]['percent_success'], 99.9)
message = "The sheriff is near"
jsonMessage = json.dumps(message)
self.jsonSender.put('message/%s' % requestName, message)
messages = self.jsonSender.get('message/%s' % requestName)
#self.assertEqual(messages[0][0][0], message)
for status in ['running-open', 'running-closed', 'completed']:
self.jsonSender.put('request/%s?status=%s' % (requestName, status))
# campaign
self.jsonSender.put('campaign/%s' % 'TestCampaign')
campaigns = self.jsonSender.get('campaign')[0]
self.assertTrue('TestCampaign' in campaigns.keys())
self.jsonSender.put('campaign/%s/%s' % ('TestCampaign', requestName))
requestsInCampaign = self.jsonSender.get('campaign/%s' % 'TestCampaign')[0]
self.assertTrue(requestName in requestsInCampaign.keys())
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['Campaign'], 'TestCampaign')
self.jsonSender.delete('request/%s' % requestName)
@attr("integration")
def testC_404Errors(self):
"""
_404Errors_
Do some things that generate 404 errors. This
should be limited to requests for objects that do not
exist.
"""
badName = 'ThereIsNoWayThisNameShouldExist'
# First, try to find a non-existent request
# This should throw a 404 error.
# The request name should not be in it
self.checkForError(cls = 'request', badName = badName, exitCode = 404,
message = 'Given requestName not found')
# Now look for non-existent user
self.checkForError(cls = 'user', badName = badName, exitCode = 404,
message = 'Cannot find user')
# Now try non-existent campaign
self.checkForError(cls = 'campaign', badName = badName, exitCode = 404,
message = "Cannot find campaign")
# Now try invalid message
# This raises a requestName error because it searches for the request
self.checkForError(cls = 'message', badName = badName, exitCode = 404,
message = "Given requestName not found", testEmpty = False)
# Check for assignments (no teams or requests)
# This raises a team error because it tries to load teams out first
self.checkForError(cls = 'assignment', badName = badName, exitCode = 404,
message = 'Cannot find team')
@attr("integration")
def testD_400Errors(self):
"""
_400Errors_
These are failures created by invalid input, such as sending
args to a request when it doesn't accept any. They should
generatore 400 Errors
"""
badName = 'ThereIsNoWayThisNameShouldExist'
# Attempt to send arguments to a function that doesn't accept them.
self.checkForError(cls = 'team', badName = badName, exitCode = 400,
message = "Invalid input: Arguments added where none allowed")
# Recheck for versions
self.checkForError(cls = 'version', badName = badName, exitCode = 400,
message = "Invalid input: Arguments added where none allowed")
# Break the validation
self.checkForError(cls = 'user', badName = '!', exitCode = 400,
message = 'Invalid input: Input data failed validation')
def checkForError(self, cls, badName, exitCode, message, testEmpty = True):
"""
_checkForError_
Generic function for checking for errors in JSON commands
Does a basic check on type cls searching for name badName which hopefull
does not exist.
Checks to make sure that it exits with code exitCode, and that
the error contains the string message.
Also checks to make sure that name badName is NOT in the output
testEmpty for those that don't handle calls to the main (i.e., who require
an argument)
"""
raises = False
# First assert that the test to be tested is empty
if testEmpty:
result = self.jsonSender.get(cls)
self.assertTrue(type(result[0]) in [type([]), type({})])
# Next, test
try:
result = self.jsonSender.get('%s/%s' % (cls, badName))
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, exitCode)
self.assertTrue(message in ex.result)
self.assertFalse(badName in ex.result)
self.assertTrue(raises)
@attr("integration")
def testE_CheckStatusChanges(self):
"""
_CheckStatusChanges_
Check status changes for a single request. See whether
we can move the request through the proper chain. Figure
out what happens when we fail.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
# There should only be one request in the DB
result = GetRequest.requestID(requestName = requestName)
self.assertEqual(result, 1)
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['Group'], groupName)
self.assertEqual(result[0]['Requestor'], userName)
# Let's see what we can do in terms of setting status
self.changeStatusAndCheck(requestName = requestName,
statusName = 'new')
# Let's try an illegal status change, just for the hell of it
raises = False
try:
self.jsonSender.put('request/%s?status=negotiating' % requestName)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
self.assertTrue(raises)
# Now, let's try a totally bogus status
raises = False
try:
self.jsonSender.put('request/%s?status=bogus' % requestName)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
self.assertTrue(raises)
# We should still be in new
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], 'new')
# Let's go on in a full loop
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing-approved')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'testing')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'tested')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assignment-approved')
# This should fail, as you cannot assign a request without a team
raises = False
try:
self.changeStatusAndCheck(requestName = requestName,
statusName = 'assigned')
except HTTPException as ex:
raises = True
self.assertTrue('Cannot change status without a team' in ex.result)
self.assertTrue(raises)
self.jsonSender.put(urllib.quote('assignment/%s/%s' % (teamName, requestName)))
self.changeStatusAndCheck(requestName = requestName,
statusName = 'negotiating')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'acquired')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'running-open')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'running-closed')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'completed')
self.changeStatusAndCheck(requestName = requestName,
statusName = 'closed-out')
def changeStatusAndCheck(self, requestName, statusName):
"""
_changeStatusAndCheck_
Change the status of a request and make sure that
the request actually did it.
"""
self.jsonSender.put('request/%s?status=%s' % (requestName, statusName))
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['RequestStatus'], statusName)
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
def testF_TestWhitelistBlacklist(self):
"""
_TestWhitelistBlacklist_
Test whether or not we can assign the block/run blacklist/whitelist
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RunWhitelist'] = [1, 2, 3]
schema['RunBlacklist'] = [4, 5, 6]
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema['RunWhitelist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema['RunBlacklist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema['BlockWhitelist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema['BlockBlacklist'])
req = self.jsonSender.get('request/%s' % requestName)
self.assertTrue('Site Blacklist' in req[0])
self.assertTrue('Site Whitelist' in req[0])
schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'}
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue("Error in Workload Validation: Argument BlockBlacklist type is incorrect in schema." in ex.result)
pass
self.assertTrue(raises)
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
schema['RunWhitelist'] = {'1': '/dataset/dataset/dataset#beta'}
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument RunWhitelist type is incorrect in schema." in ex.result)
pass
self.assertTrue(raises)
schema['RunWhitelist'] = ['hello', 'how', 'are', 'you']
try:
raises = True
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument RunWhitelist doesn't pass validation." in ex.result)
pass
self.assertTrue(raises)
def testG_AddDuplicateUser(self):
"""
_AddDuplicateUser_
Test and see if we get a sensible error when adding a duplicate user.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
raises = False
try:
self.jsonSender.put('group/%s/%s' % (groupName, userName))
except HTTPException as ex:
self.assertTrue("User/Group Already Linked in DB" in ex.result)
self.assertEqual(ex.status, 400)
raises = True
self.assertTrue(raises)
def testH_RemoveSoftwareVersion(self):
"""
_RemoveSoftwareVersion_
Remove the software version after submitting the request. See what that does.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
# Delete software versions and make sure they're gone from the DB
SoftwareManagement.removeSoftware(softwareName = schema["CMSSWVersion"],
scramArch = schema["ScramArch"])
versions = myThread.dbi.processData("SELECT * FROM reqmgr_software")[0].fetchall()
self.assertEqual(versions, [])
assocs = myThread.dbi.processData("SELECT * FROM reqmgr_software_dependency")[0].fetchall()
self.assertEqual(assocs, [])
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
def testI_CheckConfigIDs(self):
"""
_CheckConfigIDs_
Check to see if we can pull out the ConfigIDs by request
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
# Set some versions
schema['ProcessingVersion'] = '2012'
schema['AcquisitionEra'] = 'ae2012'
schema["PrimaryDataset"] = "ReallyFake"
schema["RequestNumEvents"] = 100
configID = self.createConfig()
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["ConfigCacheID"] = configID
schema["InputDataset"] = '/MinimumBias/Run2010B-RelValRawSkim-v1/RAW'
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
result = self.jsonSender.get('configIDs?prim=MinimumBias&proc=Run2010B-RelValRawSkim-v1&tier=RAW')[0]
print result
self.assertTrue(requestName in result.keys())
self.assertTrue(configID in result[requestName][0])
def testJ_CheckRequestCloning(self):
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["AcquisitionEra"] = "NewEra"
result = self.jsonSender.put("request", schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
acquisitionEra = result[0]["AcquisitionEra"]
self.assertTrue(schema["AcquisitionEra"], acquisitionEra)
# set some non-default priority
# when cloning a request which had some non default priority,
# the priority values were lost when creating a cloned request, the
# default values were lost. Change it here to specifically catch this case.
priority = 300
result = self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))
self.assertEqual(result[1], 200)
# get the original request from the server, although the variable result
# shall have the same stuff in
response = self.jsonSender.get("request/%s" % requestName)
origRequest = response[0]
self.assertEquals(origRequest["AcquisitionEra"], acquisitionEra)
# test that the priority was correctly set in the brand-new request
self.assertEquals(origRequest["RequestPriority"], priority)
# test cloning not existing request
self.assertRaises(HTTPException, self.jsonSender.put, "clone/%s" % "NotExistingRequestName")
# correct attempt to clone the request
# this is the new request, it'll have different name
result = self.jsonSender.put("clone/%s" % requestName)
# get the cloned request from the server
respose = self.jsonSender.get("request/%s" % result[0]["RequestName"])
clonedRequest = respose[0]
# these request arguments shall differ in the cloned request:
toDiffer = ["RequestName", "RequestStatus"]
for differ in toDiffer:
self.assertNotEqual(origRequest[differ], clonedRequest[differ])
# check the desired status of the cloned request
self.assertEquals(clonedRequest["RequestStatus"], "assignment-approved",
"Cloned request status should be 'assignment-approved', not '%s'." %
clonedRequest["RequestStatus"])
# don't care about these two (they will likely be the same in the unittest
# since the brand new request injection as well as the cloning probably
# happen at roughly the same time)
toDiffer.extend(["RequestDate", "timeStamp", "RequestWorkflow"])
for differ in toDiffer:
del origRequest[differ]
del clonedRequest[differ]
# check the request dictionaries
self.assertEquals(len(origRequest), len(clonedRequest))
for k1, k2 in zip(sorted(origRequest.keys()), sorted(clonedRequest.keys())):
msg = ("Request values: original: %s: %s cloned: %s: %s differ" %
(k1, origRequest[k1], k2, clonedRequest[k2]))
self.assertEqual(origRequest[k1], clonedRequest[k2], msg)
def testK_CheckRequestFailsInjectionForbiddenInputArg(self):
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
from WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel import deprecatedRequestArgs
for deprec in deprecatedRequestArgs:
schema = utils.getSchema(groupName=groupName, userName=userName)
schema[deprec] = "something"
self.assertRaises(HTTPException, self.jsonSender.put, "request", schema)
def setupACDCDatabase(self, collectionName, taskPath,
user, group):
"""
_setupACDCDatabase_
Populate an ACDC database with bogus records
associated to certain collection name, user and task path.
"""
acdcServer = CouchService(url = self.testInit.couchUrl,
database = "%s_acdc" % self.couchDBName)
owner = acdcServer.newOwner(group, user)
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = collectionName)
testCollection.setOwner(owner)
testFileset = CouchFileset(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = taskPath)
testCollection.addFileset(testFileset)
testFiles = []
for _ in range(5):
testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
events = random.randint(1024, 4096))
testFiles.append(testFile)
testFileset.add(testFiles)
def testL_CascadeCloseOutAnnnouncement(self):
"""
_testL_CascadeCloseOutAnnouncement_
Test the cascade closeout REST call, also
check that when announced a request deletes all ACDC records in the system.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
result['Requestor'], result['Group'])
depth = 2
nReq = 3
requests = [originalRequest]
def createChildrenRequest(parentRequest, i, nReq):
createdRequests = []
resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
groupName, userName)
result = self.jsonSender.put("request", resubSchema)[0]
requestName = result['RequestName']
self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
createdRequests.append(requestName)
if i:
for _ in range(nReq):
createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
return createdRequests
requests.extend(createChildrenRequest(originalRequest, depth, nReq))
for request in requests:
self.changeStatusAndCheck(request, 'assignment-approved')
for request in requests:
self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
for status in ['acquired',
'running-open', 'running-closed',
'completed']:
for request in requests:
self.changeStatusAndCheck(request, status)
self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
svc = CouchService(url = self.testInit.couchUrl,
database = "%s_acdc" % self.couchDBName)
owner = svc.newOwner(groupName, userName)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'closed-out')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertNotEqual(len(testCollection["filesets"]), 0)
self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
for request in requests:
result = self.jsonSender.get('request/%s' % request)
self.assertEqual(result[0]['RequestStatus'], 'announced')
testCollection = CouchCollection(database = self.testInit.couchDbName,
url = self.testInit.couchUrl,
name = request)
testCollection.setOwner(owner)
testCollection.populate()
self.assertEqual(len(testCollection["filesets"]), 0)
def testM_PutRequestStats(self):
userName = '******'
groupName = 'Bryant'
teamName = 'Lakers'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
stats = {'total_jobs': 100, 'input_events': 100, 'input_lumis': 100, 'input_num_files': 100}
result = self.reqService.putRequestStats(originalRequest, stats)
self.assertEqual(result['RequestName'], originalRequest)
class ReqMgrTest(RESTBaseUnitTest):
"""
_ReqMgrTest_
Basic test for the ReqMgr services.
Setup is done off-screen in RESTBaseUnitTest - this makes
things confusing
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
tearDown
Tear down everything
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
'process': {
'outputModules_': ['ThisIsAName'],
'ThisIsAName': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
BadTweak = {
'process': {
'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
},
'ThisIsAName2': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username='******')
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
@attr("integration")
def testA_testBasicSetUp(self):
"""
_testBasicSetUp_
Moving the tests that were in the setUp category out of it,
mostly because I want to make sure that they don't fail inside
the setUp statement.
"""
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(
self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = self.jsonSender.get('group/PeopleLikeMe')[0]['users']
self.assertTrue('me' in users)
groups = self.jsonSender.get('user/me')[0]['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(
self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to dealwith
schema = utils.getSchema()
version = "version/" + schema["CMSSWVersion"]
self.assertTrue(self.jsonSender.put(version)[1] == 200)
self.assertTrue(
schema["CMSSWVersion"] in self.jsonSender.get('version')[0])
@attr("integration")
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = utils.getAndSetupSchema(self)
schema['RequestNumEvents'] = 100
schema['RequestEventSize'] = 101
self.doRequest(schema)
def doRequest(self, schema):
"""
_doRequest_
Run all tests on a basic ReReco workflow
"""
requestName = schema['RequestName']
self.assertRaises(HTTPException, self.jsonSender.delete,
'request/%s' % requestName)
result = self.jsonSender.put('request/%s' % (requestName), schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
self.assertEqual(
self.jsonSender.get('request/%s' % requestName)[0]['RequestName'],
requestName)
self.jsonSender.put('request/%s?status=assignment-approved' %
requestName)
me = self.jsonSender.get('user/me')[0]
self.assertTrue(requestName in me['requests'])
self.assertEqual(
self.jsonSender.put('request/%s?priority=5' % requestName)[1], 200)
self.assertEqual(self.jsonSender.post('user/me?priority=6')[1], 200)
self.assertEqual(
self.jsonSender.post('group/PeopleLikeMe?priority=7')[1], 200)
# default priority of group and user of 1
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], 5)
self.assertEqual(request['ReqMgrRequestorBasePriority'], 6)
self.assertEqual(request['ReqMgrGroupBasePriority'], 7)
self.assertEqual(request['RequestPriority'], 5 + 6 + 7)
# Check LFN Bases
self.assertEqual(request['UnmergedLFNBase'], '/store/unmerged')
self.assertEqual(request['MergedLFNBase'], '/store/data')
# Check random other
self.assertEqual(request['CustodialSite'], 'US_T1_FNAL')
# Check Num events
self.assertEqual(request['RequestNumEvents'], 100)
self.assertEqual(request['RequestEventSize'], 101)
# only certain transitions allowed
#self.assertEqual(self.jsonSender.put('request/%s?status=running' % requestName)[1], 400)
self.assertRaises(HTTPException, self.jsonSender.put,
'request/%s?status=running' % requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'assignment-approved')
self.assertTrue(
self.jsonSender.put(
urllib.quote('assignment/White Sox/%s' %
requestName))[1] == 200)
requestsAndSpecs = self.jsonSender.get(
urllib.quote('assignment/White Sox'))[0]
self.assertTrue(requestName in requestsAndSpecs[0])
workloadHelper = WMWorkloadHelper()
workloadHelper.load(requestsAndSpecs[0][1])
self.assertEqual(workloadHelper.getOwner()['Requestor'], "me")
self.assertEqual(
self.jsonSender.get('assignment?request=%s' % requestName)[0],
['White Sox'])
self.assertEqual(
self.jsonSender.get('request/%s' % requestName)[0]['teams'],
['White Sox'])
agentUrl = 'http://cmssrv96.fnal.gov/workqueue'
self.jsonSender.put('workQueue/%s?url=%s' %
(requestName, urllib.quote(agentUrl)))
self.assertEqual(
self.jsonSender.get('workQueue/%s' % requestName)[0][0], agentUrl)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['RequestStatus'], 'acquired')
self.jsonSender.post('request/%s?events_written=10&files_merged=1' %
requestName)
self.jsonSender.post(
'request/%s?events_written=20&files_merged=2&percent_success=99.9'
% requestName)
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(len(request['RequestUpdates']), 2)
self.assertEqual(request['RequestUpdates'][0]['files_merged'], 1)
self.assertEqual(request['RequestUpdates'][1]['events_written'], 20)
self.assertEqual(request['RequestUpdates'][1]['percent_success'], 99.9)
message = "The sheriff is near"
jsonMessage = json.dumps(message)
self.jsonSender.put('message/%s' % requestName, message)
messages = self.jsonSender.get('message/%s' % requestName)
#self.assertEqual(messages[0][0][0], message)
for status in ['running', 'completed']:
self.jsonSender.put('request/%s?status=%s' % (requestName, status))
# campaign
self.jsonSender.put('campaign/%s' % 'TestCampaign')
campaigns = self.jsonSender.get('campaign')[0]
self.assertTrue('TestCampaign' in campaigns.keys())
self.jsonSender.put('campaign/%s/%s' % ('TestCampaign', requestName))
requestsInCampaign = self.jsonSender.get('campaign/%s' %
'TestCampaign')[0]
self.assertTrue(requestName in requestsInCampaign.keys())
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['Campaign'], 'TestCampaign')
self.jsonSender.delete('request/%s' % requestName)
@attr("integration")
def testC_404Errors(self):
"""
_404Errors_
Do some things that generate 404 errors. This
should be limited to requests for objects that do not
exist.
"""
badName = 'ThereIsNoWayThisNameShouldExist'
# First, try to find a non-existant request
# This should throw a 404 error.
# The request name should not be in it
self.checkForError(cls='request',
badName=badName,
exitCode=404,
message='Given requestName not found')
# Now look for non-existant user
self.checkForError(cls='user',
badName=badName,
exitCode=404,
message='Cannot find user')
# Now try non-existant group
self.checkForError(cls='group',
badName=badName,
exitCode=404,
message="Cannot find group/group priority")
# Now try non-existant campaign
self.checkForError(cls='campaign',
badName=badName,
exitCode=404,
message="Cannot find campaign")
# Now try invalid message
# This raises a requestName error becuase it searches for the request
self.checkForError(cls='message',
badName=badName,
exitCode=404,
message="Given requestName not found",
testEmpty=False)
# Check for assignments (no teams or requests)
# This raises a team error because it tries to load teams out first
self.checkForError(cls='assignment',
badName=badName,
exitCode=404,
message='Cannot find team')
@attr("integration")
def testD_400Errors(self):
"""
_400Errors_
These are failures created by invalid input, such as sending
args to a request when it doesn't accept any. They should
generatore 400 Errors
"""
badName = 'ThereIsNoWayThisNameShouldExist'
# Attempt to send arguments to a function that doesn't accept them.
self.checkForError(
cls='team',
badName=badName,
exitCode=400,
message="Invalid input: Arguments added where none allowed")
# Recheck for versions
self.checkForError(
cls='version',
badName=badName,
exitCode=400,
message="Invalid input: Arguments added where none allowed")
# Break the validation
self.checkForError(
cls='user',
badName='!',
exitCode=400,
message='Invalid input: Input data failed validation')
def checkForError(self, cls, badName, exitCode, message, testEmpty=True):
"""
_checkForError_
Generic function for checking for errors in JSON commands
Does a basic check on type cls searching for name badName which hopefull
does not exist.
Checks to make sure that it exits with code exitCode, and that
the error contains the string message.
Also checks to make sure that name badName is NOT in the output
testEmpty for those that don't handle calls to the main (i.e., who require
an argument)
"""
raises = False
# First assert that the test to be tested is empty
if testEmpty:
result = self.jsonSender.get(cls)
self.assertTrue(type(result[0]) in [type([]), type({})])
# Next, test
try:
result = self.jsonSender.get('%s/%s' % (cls, badName))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, exitCode)
self.assertTrue(message in ex.result)
self.assertFalse(badName in ex.result)
self.assertTrue(raises)
class ReqMgrTester(object):
def __init__(self, reqMgrUrl):
self.reqMgrUrl = reqMgrUrl
self.restSender = JSONRequests(reqMgrUrl)
d = dict(endpoint = self.reqMgrUrl)
self.reqMgrService = RequestManager(d)
def queryAllRequests(self):
"""
Returns all requests stored at ReqMgr instance.
"""
logging.info("Querying all requests at ReqMgr instance ...")
r = self.reqMgrService.getRequestNames()
print "Found %s requests:" % len(r)
for req in r:
print req
def queryRequest(self, requestName):
"""
Query a specific request according to the input argument.
"""
urlQuery = "request/%s" % requestName
logging.info("Querying request '%s'" % requestName)
logging.info("Query: '%s':" % urlQuery)
r = self.restSender.get(urlQuery)
print str(r)
def createRequests(self, numRequests):
"""
Inject new numRequests into ReqMgr instance.
(see ReqMgr_t testE how to create a request)
"""
logging.info("Creating %s new requests ..." % numRequests)
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = 'ReReco'
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = '%s' % "zmaxa"
schema['Group'] = '%s' % "DATAOPS"
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
schema['Campaign'] = 'MyTestCampaign'
for i in range(numRequests):
urlQuery = "request/testRequest"
print "Query: '%s':" % urlQuery
print "Schema (request): '%s'" % schema
r = self.restSender.put(urlQuery, schema)
# print "request creating response: ", r
print "created: ", r[0]["RequestName"]
def deleteRequest(self, requestNames):
"""
Delete requests specified in the input, more request names
are comma-separated.
"""
logging.info("Deleting requests ...")
for reqName in requestNames.split(','):
reqName = reqName.strip()
urlQuery = "request/%s" % reqName
logging.info("Deleting request (request_name): '%s'" % reqName)
logging.info("Query: '%s':" % urlQuery)
r = self.restSender.delete(urlQuery)
def requestChangeStates(self, reqName):
"""
Route the request (spec. by the request name) in the input
through a series of possible request states.
"""
logging.info("Changing state of request %s ..." % reqName)
def changeState(requestName, urlQuery):
logging.info("Query: '%s' ..." % urlQuery)
r = self.restSender.put(urlQuery)
r = self.restSender.get("request/%s" % requestName)
#assert r[0]["RequestStatus"] == statusName
logging.info("Querying modified request, new state: %s" % r[0]["RequestStatus"])
# once a request is created, it's in 'new' state
# states transition has to be an allowed one as defined here:
# WMCore/RequestManager/RequestDB/Settings/RequestStatus.py
statesQueries = ["request/%s?status=%s" % (reqName, "testing-approved"),
"request/%s?status=%s" % (reqName, "testing"),
"request/%s?status=%s" % (reqName, "tested"),
"request/%s?status=%s" % (reqName, "assignment-approved"),
# create an assignment now
# need quote because of space in the team name
# (previous name - White Sox)
urllib.quote("assignment/%s/%s" % (TEAM_NAME, reqName))]
for query in statesQueries:
changeState(reqName, query)
def setup(self):
"""
Setup ReqMgr instance for dealing with requests - needs to create
a user, group, SW releases entries, etc.
as done in test/python/WMCore_t/RequestManager_t/ReqMgr_t.py
"""
logging.info("ReqMgr setup ...")
def doQuery(urlQuery):
logging.info("Query: '%s' ..." % urlQuery)
r = None
try:
r = self.restSender.put(urlQuery)
except Exception as ex:
print "exception"
print str(ex)
print "response:", r
queries = ["user/[email protected]",
"group/DATAOPS",
"group/DATAOPS/zmaxa",
urllib.quote("team/" + TEAM_NAME),
"version/%s" % "CMSSW_3_5_8"]
for q in queries:
doQuery(q)
logging.info("ReqMgr setup finished, listing known users ...")
q = "user/"
r = self.restSender.get(q)
print r
class ReqMgrTester(object):
def __init__(self, reqMgrUrl):
self.reqMgrUrl = reqMgrUrl
self.restSender = JSONRequests(reqMgrUrl)
d = dict(endpoint=self.reqMgrUrl)
self.reqMgrService = RequestManager(d)
def queryAllRequests(self):
"""
Returns all requests stored at ReqMgr instance.
"""
logging.info("Querying all requests at ReqMgr instance ...")
r = self.reqMgrService.getRequestNames()
print "Found %s requests:" % len(r)
for req in r:
print req
def queryRequest(self, requestName):
"""
Query a specific request according to the input argument.
"""
urlQuery = "request/%s" % requestName
logging.info("Querying request '%s'" % requestName)
logging.info("Query: '%s':" % urlQuery)
r = self.restSender.get(urlQuery)
print str(r)
def createRequests(self, numRequests):
"""
Inject new numRequests into ReqMgr instance.
(see ReqMgr_t testE how to create a request)
"""
logging.info("Creating %s new requests ..." % numRequests)
schema = ReReco.getTestArguments()
schema['RequestName'] = 'TestReReco'
schema['RequestType'] = 'ReReco'
schema['CmsPath'] = "/uscmst1/prod/sw/cms"
schema['Requestor'] = '%s' % "zmaxa"
schema['Group'] = '%s' % "DATAOPS"
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
schema['Campaign'] = 'MyTestCampaign'
for i in range(numRequests):
urlQuery = "request/testRequest"
print "Query: '%s':" % urlQuery
print "Schema (request): '%s'" % schema
r = self.restSender.put(urlQuery, schema)
# print "request creating response: ", r
print "created: ", r[0]["RequestName"]
def deleteRequest(self, requestNames):
"""
Delete requests specified in the input, more request names
are comma-separated.
"""
logging.info("Deleting requests ...")
for reqName in requestNames.split(','):
reqName = reqName.strip()
urlQuery = "request/%s" % reqName
logging.info("Deleting request (request_name): '%s'" % reqName)
logging.info("Query: '%s':" % urlQuery)
r = self.restSender.delete(urlQuery)
def requestChangeStates(self, reqName):
"""
Route the request (spec. by the request name) in the input
through a series of possible request states.
"""
logging.info("Changing state of request %s ..." % reqName)
def changeState(requestName, urlQuery):
logging.info("Query: '%s' ..." % urlQuery)
r = self.restSender.put(urlQuery)
r = self.restSender.get("request/%s" % requestName)
#assert r[0]["RequestStatus"] == statusName
logging.info("Querying modified request, new state: %s" %
r[0]["RequestStatus"])
# once a request is created, it's in 'new' state
# states transition has to be an allowed one as defined here:
# WMCore/RequestManager/RequestDB/Settings/RequestStatus.py
statesQueries = [
"request/%s?status=%s" % (reqName, "testing-approved"),
"request/%s?status=%s" % (reqName, "testing"),
"request/%s?status=%s" % (reqName, "tested"),
"request/%s?status=%s" % (reqName, "assignment-approved"),
# create an assignment now
# need quote because of space in the team name
# (previous name - White Sox)
urllib.quote("assignment/%s/%s" % (TEAM_NAME, reqName))
]
for query in statesQueries:
changeState(reqName, query)
def setup(self):
"""
Setup ReqMgr instance for dealing with requests - needs to create
a user, group, SW releases entries, etc.
as done in test/python/WMCore_t/RequestManager_t/ReqMgr_t.py
"""
logging.info("ReqMgr setup ...")
def doQuery(urlQuery):
logging.info("Query: '%s' ..." % urlQuery)
r = None
try:
r = self.restSender.put(urlQuery)
except Exception as ex:
print "exception"
print str(ex)
print "response:", r
queries = [
"user/[email protected]", "group/DATAOPS",
"group/DATAOPS/zmaxa",
urllib.quote("team/" + TEAM_NAME),
"version/%s" % "CMSSW_3_5_8"
]
for q in queries:
doQuery(q)
logging.info("ReqMgr setup finished, listing known users ...")
q = "user/"
r = self.restSender.get(q)
print r
