def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "DataProcessing"
schema['ProdScenario'] = 'pp'
result = self.jsonSender.put('request/testRequest', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName=groupName, userName=userName)
schema['RequestType'] = "Resubmission"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Missing required "
"field 'OriginalRequestName' in workload validation!" in
ex.result)
def testK_LHEStepZero(self):
"""
_LHEStepZero_
Test LHEStepZero workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "LHEStepZero"
# Set some versions
schema['ProcessingVersion'] = '2012'
schema['AcquisitionEra'] = 'ae2012'
schema['FirstEvent'] = 1
schema['FirstLumi'] = 1
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Missing required "
"field 'ConfigCacheID' in workload validation!" in ex.result)
def testI_CheckConfigIDs(self):
"""
_CheckConfigIDs_
Check to see if we can pull out the ConfigIDs by request
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
# Set some versions
schema['ProcessingVersion'] = '2012'
schema['AcquisitionEra'] = 'ae2012'
schema["PrimaryDataset"] = "ReallyFake"
schema["RequestNumEvents"] = 100
configID = self.createConfig()
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["ConfigCacheID"] = configID
schema["InputDatasets"] = ['/MinimumBias/Run2010B-RelValRawSkim-v1/RAW']
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
result = self.jsonSender.get('configIDs?prim=MinimumBias&proc=Commissioning10-v4&tier=RAW')[0]
self.assertTrue(requestName in result.keys())
self.assertTrue(configID in result[requestName][0])
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "DataProcessing"
schema['ProdScenario'] = 'pp'
result = self.jsonSender.put('request/testRequest', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['RequestType'] = "Resubmission"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Missing required "
"field 'OriginalRequestName' in workload validation!" in ex.result)
def testC_DataProcessing(self):
"""
_DataProcessing_
Test DataProcessing workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "DataProcessing"
del schema['ProcScenario']
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"No Scenario or Config in Processing Request!" in ex.result)
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxint + 1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
def testG_MonteCarloFromGEN(self):
"""
_MonteCarloFromGEN_
Test MonteCarloFromGEN workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "MonteCarloFromGEN"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue(
"Error in Workload Validation: Argument ConfigCacheID can't be None"
in ex.result)
def testB_Analysis(self):
"""
_Analysis_
Test Analysis workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "Analysis"
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(raises)
# Put the right things in the schema
# And watch it fail, because we can't open a secure connection to CERN
# in a unittest well
schema['RequestorDN'] = 'SomeDN'
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "ReReco"
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['RequestType'] = "Resubmission"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument InitialTaskPath is required." in ex.result)
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxint + 1
self.jsonSender.put('request/%s?priority=%s' %
(requestName, priority))
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Priority must have abs() less then MAXINT!" in ex.result)
def testH_MonteCarlo(self):
"""
_MonteCarlo_
Test MonteCarlo workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "MonteCarlo"
schema['DbsUrl'] = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
# Set some versions
schema['ProcessingVersion'] = '2012'
schema['AcquisitionEra'] = 'ae2012'
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Argument ConfigCacheID can't be None"
in ex.result)
def testD_ReDigi(self):
"""
_ReDigi_
Test ReDigi workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "ReDigi"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Missing required "
"field 'StepOneConfigCacheID' in workload validation!" in
ex.result)
def testE_StoreResults(self):
"""
_StoreResults_
Test StoreResults workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "StoreResults"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue(
"Error in Workload Validation: Argument CmsPath is required."
in ex.result)
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser",
"ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
schema['CouchWorkloadDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request', schema)
try:
self.requestName = r[0]['RequestName']
except:
self.requestName = r[0].values()[0]['RequestName']
except Exception as ex:
msg = traceback.format_exc()
print "Exception during set up, reason: %s" % msg
raise ex
def testA_makeReRecoWorkload(self):
"""
_makeReRecoWorkload_
Check that you can make a basic ReReco workload
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
del schema['GlobalTag']
configID = self.createConfig(bad=False)
schema['ConfigCacheID'] = configID
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Argument GlobalTag is required."
in ex.result)
def testI_CheckConfigIDs(self):
"""
_CheckConfigIDs_
Check to see if we can pull out the ConfigIDs by request
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
# Set some versions
schema['ProcessingVersion'] = '2012'
schema['AcquisitionEra'] = 'ae2012'
schema["PrimaryDataset"] = "ReallyFake"
schema["RequestNumEvents"] = 100
configID = self.createConfig()
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["ConfigCacheID"] = configID
schema["InputDataset"] = '/MinimumBias/Run2010B-RelValRawSkim-v1/RAW'
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
result = self.jsonSender.get('configIDs?prim=MinimumBias&proc=Run2010B-RelValRawSkim-v1&tier=RAW')[0]
print result
self.assertTrue(requestName in result.keys())
self.assertTrue(configID in result[requestName][0])
def testH_MonteCarlo(self):
"""
_MonteCarlo_
Test MonteCarlo workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "MonteCarlo"
schema['DbsUrl'] = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
# Set some versions
schema['ProcessingVersion'] = '2012'
schema['AcquisitionEra'] = 'ae2012'
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument ConfigCacheID can't be None" in ex.result)
def testH_RemoveSoftwareVersion(self):
"""
_RemoveSoftwareVersion_
Remove the software version after submitting the request. See what that does.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
# Delete software versions and make sure they're gone from the DB
SoftwareManagement.removeSoftware(softwareName=schema["CMSSWVersion"],
scramArch=schema["ScramArch"])
versions = myThread.dbi.processData(
"SELECT * FROM reqmgr_software")[0].fetchall()
self.assertEqual(versions, [])
assocs = myThread.dbi.processData(
"SELECT * FROM reqmgr_software_dependency")[0].fetchall()
self.assertEqual(assocs, [])
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
def testB_Analysis(self):
"""
_Analysis_
Test Analysis workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "Analysis"
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(raises)
# Put the right things in the schema
# And watch it fail, because we can't open a secure connection to CERN
# in a unittest well
schema['RequestorDN'] = 'SomeDN'
def testH_RemoveSoftwareVersion(self):
"""
_RemoveSoftwareVersion_
Remove the software version after submitting the request. See what that does.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
# Delete software versions and make sure they're gone from the DB
SoftwareManagement.removeSoftware(softwareName = schema["CMSSWVersion"],
scramArch = schema["ScramArch"])
versions = myThread.dbi.processData("SELECT * FROM reqmgr_software")[0].fetchall()
self.assertEqual(versions, [])
assocs = myThread.dbi.processData("SELECT * FROM reqmgr_software_dependency")[0].fetchall()
self.assertEqual(assocs, [])
req = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "ReReco"
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['RequestType'] = "Resubmission"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument InitialTaskPath is required." in ex.result)
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['ConfigCacheID'] = self.createConfig()
schema['CouchDBName'] = self.couchDBName
schema['CouchWorkloadDBName'] = self.couchDBName
try:
r = self.jsonSender.put('request', schema)
try:
self.requestName = r[0]['RequestName']
except:
self.requestName = r[0].values()[0]['RequestName']
except Exception as ex:
msg = traceback.format_exc()
print("Exception during set up, reason: %s" % msg)
raise ex
def testK_LHEStepZero(self):
"""
_LHEStepZero_
Test LHEStepZero workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "LHEStepZero"
# Set some versions
schema['ProcessingVersion'] = '2012'
schema['AcquisitionEra'] = 'ae2012'
schema['FirstEvent'] = 1
schema['FirstLumi'] = 1
schema['DbsUrl'] = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet'
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Missing required "
"field 'ConfigCacheID' in workload validation!" in ex.result)
def testJ_CheckRequestCloning(self):
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put("request", schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
acquisitionEra = result[0]["AcquisitionEra"]
self.assertTrue(schema["AcquisitionEra"], acquisitionEra)
# set some non-default priority
# when cloning a request which had some non default priority,
# the priority values were lost when creating a cloned request, the
# default values were lost. Change it here to specifically catch this case.
priority = 300
result = self.jsonSender.put("request/%s?priority=%s" % (requestName, priority))
self.assertEqual(result[1], 200)
# get the original request from the server, although the variable result
# shall have the same stuff in
response = self.jsonSender.get("request/%s" % requestName)
origRequest = response[0]
self.assertEquals(origRequest["AcquisitionEra"], acquisitionEra)
# test that the priority was correctly set in the brand-new request
self.assertEquals(origRequest["RequestPriority"], priority)
# test cloning not existing request
self.assertRaises(HTTPException, self.jsonSender.put, "clone/%s" % "NotExistingRequestName")
# correct attempt to clone the request
# this is the new request, it'll have different name
result = self.jsonSender.put("clone/%s" % requestName)
# get the cloned request from the server
respose = self.jsonSender.get("request/%s" % result[0]["RequestName"])
clonedRequest = respose[0]
# these request arguments shall differ in the cloned request:
toDiffer = ["RequestName", "RequestStatus"]
for differ in toDiffer:
self.assertNotEqual(origRequest[differ], clonedRequest[differ])
# check the desired status of the cloned request
self.assertEquals(clonedRequest["RequestStatus"], "assignment-approved",
"Cloned request status should be 'assignment-approved', not '%s'." %
clonedRequest["RequestStatus"])
# don't care about these two (they will likely be the same in the unittest
# since the brand new request injection as well as the cloning probably
# happen at roughly the same time)
toDiffer.extend(["RequestDate", "timeStamp", "RequestWorkflow"])
for differ in toDiffer:
del origRequest[differ]
del clonedRequest[differ]
# check the request dictionaries
self.assertEquals(len(origRequest), len(clonedRequest))
for k1, k2 in zip(sorted(origRequest.keys()), sorted(clonedRequest.keys())):
msg = ("Request values: original: %s: %s cloned: %s: %s differ" %
(k1, origRequest[k1], k2, clonedRequest[k2]))
self.assertEqual(origRequest[k1], clonedRequest[k2], msg)
def testF_TestWhitelistBlacklist(self):
"""
_TestWhitelistBlacklist_
Test whether or not we can assign the block/run blacklist/whitelist
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RunWhitelist'] = [1, 2, 3]
schema['RunBlacklist'] = [4, 5, 6]
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(
workload.data.tasks.DataProcessing.input.dataset.runs.whitelist,
schema['RunWhitelist'])
self.assertEqual(
workload.data.tasks.DataProcessing.input.dataset.runs.blacklist,
schema['RunBlacklist'])
self.assertEqual(
workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist,
schema['BlockWhitelist'])
self.assertEqual(
workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist,
schema['BlockBlacklist'])
req = self.jsonSender.get('request/%s' % requestName)
self.assertTrue(req[0].has_key('Site Blacklist'))
self.assertTrue(req[0].has_key('Site Whitelist'))
schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'}
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue(
"Error in Workload Validation: Argument BlockBlacklist type is incorrect in schema."
in ex.result)
pass
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "ReReco"
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName=groupName, userName=userName)
schema['RequestType'] = "Resubmission"
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Validation failed: InitialTaskPath is mendatory"
in ex.result)
self.assertTrue(raises)
schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName
schema["ACDCServer"] = os.environ.get("COUCHURL")
schema["ACDCDatabase"] = self.couchDBName
schema["CollectionName"] = "SomeOtherName"
# Here we just make sure that real result goes through
result = self.jsonSender.put('request', schema)
resubmitName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % resubmitName)
couchServer = CouchServer(self.testInit.couchUrl)
reqmgrCouch = couchServer.connectDatabase(self.couchDBName)
result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests',
{}, [requestName])['rows']
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['key'], requestName)
self.assertEqual(result[0]['id'], resubmitName)
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = utils.getAndSetupSchema(self)
schema['RequestNumEvents'] = 100
schema['RequestEventSize'] = 101
self.doRequest(schema)
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "ReReco"
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['RequestType'] = "Resubmission"
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Validation failed: InitialTaskPath is mendatory" in ex.result)
self.assertTrue(raises)
schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName
schema["ACDCServer"] = os.environ.get("COUCHURL")
schema["ACDCDatabase"] = self.couchDBName
schema["CollectionName"] = "SomeOtherName"
# Here we just make sure that real result goes through
result = self.jsonSender.put('request', schema)
resubmitName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % resubmitName)
couchServer = CouchServer(self.testInit.couchUrl)
reqmgrCouch = couchServer.connectDatabase(self.couchDBName)
result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests', {}, [requestName])['rows']
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['key'], requestName)
self.assertEqual(result[0]['id'], resubmitName)
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = utils.getAndSetupSchema(self)
schema['RequestNumEvents'] = 100
schema['RequestEventSize'] = 101
self.doRequest(schema)
def testG_MonteCarloFromGEN(self):
"""
_MonteCarloFromGEN_
Test MonteCarloFromGEN workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "MonteCarloFromGEN"
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Failure to load ConfigCache while validating workload"
in ex.result)
self.assertTrue(raises)
schema["ConfigCacheID"] = "fakeID"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Failure to load ConfigCache while validating workload" in
ex.result)
self.assertTrue(raises)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['SizePerEvent'], 512)
self.assertEqual(request['RequestNumEvents'], 0)
self.assertEqual(request['RequestSizeFiles'], 0)
def testC_UserGroupRequestPriority(self):
"""
_UserGroupRequestPriority_
Set the priorities of the user, the group, and the request
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), 0)
# Reset user, group priorities to 0
self.jsonSender.post('user/%s?priority=0' % userName)
self.jsonSender.post('group/%s?priority=0' % groupName)
# Set priority == 5
priority = 5
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority)
self.jsonSender.post('user/%s?priority=6' % userName)
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority + 6)
self.jsonSender.post('group/%s?priority=7' % groupName)
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority + 6 + 7)
self.jsonSender.post('user/%s?priority=0' % userName)
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.priority(), priority + 7)
def testC_UserGroupRequestPriority(self):
"""
_UserGroupRequestPriority_
Set the priorities of the user, the group, and the request
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), 0)
# Reset user, group priorities to 0
self.jsonSender.post('user/%s?priority=0' % userName)
self.jsonSender.post('group/%s?priority=0' % groupName)
# Set priority == 5
priority = 5
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority)
self.jsonSender.post('user/%s?priority=6' % userName)
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority + 6)
self.jsonSender.post('group/%s?priority=7' % groupName)
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority + 6 + 7)
self.jsonSender.post('user/%s?priority=0' % userName)
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
request = self.jsonSender.get('request/%s' % requestName)[0]
self.assertEqual(request['ReqMgrRequestBasePriority'], priority)
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.priority(), priority + 7)
def testM_PutRequestStats(self):
userName = '******'
groupName = 'Bryant'
teamName = 'Lakers'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
stats = {'total_jobs': 100, 'input_events': 100, 'input_lumis': 100, 'input_num_files': 100}
result = self.reqService.putRequestStats(originalRequest, stats)
self.assertEqual(result['RequestName'], originalRequest)
def testF_TestWhitelistBlacklist(self):
"""
_TestWhitelistBlacklist_
Test whether or not we can assign the block/run blacklist/whitelist
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RunWhitelist'] = [1, 2, 3]
schema['RunBlacklist'] = [4, 5, 6]
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(
workload.data.tasks.DataProcessing.input.dataset.runs.whitelist,
schema['RunWhitelist'])
self.assertEqual(
workload.data.tasks.DataProcessing.input.dataset.runs.blacklist,
schema['RunBlacklist'])
self.assertEqual(
workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist,
schema['BlockWhitelist'])
self.assertEqual(
workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist,
schema['BlockBlacklist'])
req = self.jsonSender.get('request/%s' % requestName)
self.assertTrue(req[0].has_key('Site Blacklist'))
self.assertTrue(req[0].has_key('Site Whitelist'))
schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'}
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Bad Run list of type " in ex.result)
pass
def testG_MonteCarloFromGEN(self):
"""
_MonteCarloFromGEN_
Test MonteCarloFromGEN workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "MonteCarloFromGEN"
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Failure to load ConfigCache while validating workload" in ex.result)
self.assertTrue(raises)
schema["ConfigCacheID"] = "fakeID"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Failure to load ConfigCache while validating workload" in ex.result)
self.assertTrue(raises)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['SizePerEvent'], 512)
self.assertEqual(request['RequestNumEvents'], 0)
self.assertEqual(request['RequestSizeFiles'], 0)
def testK_CheckRequestFailsInjectionForbiddenInputArg(self):
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
from WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel import deprecatedRequestArgs
for deprec in deprecatedRequestArgs:
schema = utils.getSchema(groupName=groupName, userName=userName)
schema[deprec] = "something"
self.assertRaises(HTTPException, self.jsonSender.put, "request", schema)
def testK_CheckRequestFailsInjectionForbiddenInputArg(self):
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
from WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel import deprecatedRequestArgs
for deprec in deprecatedRequestArgs:
schema = utils.getSchema(groupName=groupName, userName=userName)
schema[deprec] = "something"
self.assertRaises(HTTPException, self.jsonSender.put, "request", schema)
def testF_TestWhitelistBlacklist(self):
"""
_TestWhitelistBlacklist_
Test whether or not we can assign the block/run blacklist/whitelist
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RunWhitelist'] = [1, 2, 3]
schema['RunBlacklist'] = [4, 5, 6]
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema['RunWhitelist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema['RunBlacklist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema['BlockWhitelist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema['BlockBlacklist'])
req = self.jsonSender.get('request/%s' % requestName)
self.assertTrue(req[0].has_key('Site Blacklist'))
self.assertTrue(req[0].has_key('Site Whitelist'))
schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'}
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue("Error in Workload Validation: Argument BlockBlacklist type is incorrect in schema." in ex.result)
pass
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxsize + 1
self.jsonSender.put('request/%s?priority=%s' %
(requestName, priority))
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
raises = False
try:
priority = -1 - sys.maxsize
self.jsonSender.put('request/%s?priority=%s' %
(requestName, priority))
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
print(ex.result)
self.assertTrue(
"Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = utils.getAndSetupSchema(self)
schema['RequestNumEvents'] = 100
schema['SizePerEvent'] = 101
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
self.doRequest(schema)
def testB_ReReco(self):
"""
_ReReco_
Try a basic ReReco workflow
"""
schema = utils.getAndSetupSchema(self)
schema['RequestNumEvents'] = 100
schema['SizePerEvent'] = 101
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
self.doRequest(schema)
def testE_CheckStatusChanges(self):
"""
_CheckStatusChanges_
Check status changes for a single request. See whether
we can move the request through the proper chain. Figure
out what happens when we fail.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
# There should only be one request in the DB
result = GetRequest.requestID(requestName = requestName)
self.assertEqual(result, 1)
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['Group'], groupName)
self.assertEqual(result[0]['Requestor'], userName)
# Let's see what we can do in terms of setting status
self.changeStatusAndCheck(requestName = requestName,
statusName = 'new')
# Let's try an illegal status change, just for the hell of it
raises = False
try:
self.jsonSender.put('request/%s?status=negotiating' % requestName)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
def testE_CheckStatusChanges(self):
"""
_CheckStatusChanges_
Check status changes for a single request. See whether
we can move the request through the proper chain. Figure
out what happens when we fail.
"""
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
# There should only be one request in the DB
result = GetRequest.requestID(requestName=requestName)
self.assertEqual(result, 1)
result = self.jsonSender.get('request/%s' % requestName)
self.assertEqual(result[0]['Group'], groupName)
self.assertEqual(result[0]['Requestor'], userName)
# Let's see what we can do in terms of setting status
self.changeStatusAndCheck(requestName=requestName, statusName='new')
# Let's try an illegal status change, just for the hell of it
raises = False
try:
self.jsonSender.put('request/%s?status=negotiating' % requestName)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue('Failed to change status' in ex.result)
self.assertFalse(requestName in ex.result)
def testM_PutRequestStats(self):
userName = '******'
groupName = 'Bryant'
teamName = 'Lakers'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
stats = {'total_jobs': 100, 'input_events': 100, 'input_lumis': 100, 'input_num_files': 100}
result = self.reqService.putRequestStats(originalRequest, stats)
self.assertEqual(result['RequestName'], originalRequest)
def testM_PutRequestStats(self):
userName = '******'
groupName = 'Bryant'
teamName = 'Lakers'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put("request", schema)[0]
originalRequest = result['RequestName']
stats = {'total_jobs': 100, 'input_events': 100, 'input_lumis': 100, 'input_num_files': 100}
result = self.reqService.putRequestStats(originalRequest, stats)
self.assertEqual(result['RequestName'], originalRequest)
def testG_AddDuplicateUser(self):
"""
_AddDuplicateUser_
Test and see if we get a sensible error when adding a duplicate user.
"""
userName = "******"
groupName = "Li"
teamName = "Tang"
schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName)
raises = False
try:
self.jsonSender.put("group/%s/%s" % (groupName, userName))
except HTTPException, ex:
self.assertTrue("User/Group Already Linked in DB" in ex.result)
self.assertEqual(ex.status, 400)
raises = True
def testB_InvalidPriority(self):
"""
_InvalidPriority_
Put in a bunch of invalid values for priorities and
see what the code makes of them.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
raises = False
try:
priority = sys.maxint + 1
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
raises = False
try:
priority = -1 - sys.maxint
self.jsonSender.put('request/%s?priority=%s' % (requestName, priority))
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
self.assertTrue(raises)
def testF_TestWhitelistBlacklist(self):
"""
_TestWhitelistBlacklist_
Test whether or not we can assign the block/run blacklist/whitelist
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RunWhitelist'] = [1, 2, 3]
schema['RunBlacklist'] = [4, 5, 6]
schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha']
schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta']
result = self.jsonSender.put('request/testRequest', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
workload = self.loadWorkload(requestName = requestName)
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema['RunWhitelist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema['RunBlacklist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema['BlockWhitelist'])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema['BlockBlacklist'])
req = self.jsonSender.get('request/%s' % requestName)
self.assertTrue(req[0].has_key('Site Blacklist'))
self.assertTrue(req[0].has_key('Site Whitelist'))
schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'}
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Bad Run list of type " in ex.result)
pass
def testJ_CheckRequestCloning(self):
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
result = self.jsonSender.put('request', schema)
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
# AcquisitionEra is returned here, but in fact on server is not stored until assign
self.assertTrue(schema["AcquisitionEra"], result[0]["AcquisitionEra"])
# get the original request (although the variable result shall have the same stuff in)
origRequest = self.jsonSender.get("request/%s" % requestName)
origRequest = origRequest[0]
self.assertEquals(origRequest["AcquisitionEra"], "None") # was not stored
# test cloning not existing request
self.assertRaises(HTTPException, self.jsonSender.put, "clone/%s" % "NotExistingRequestName")
# this is the new request, it'll have different name
result = self.jsonSender.put("clone/%s" % requestName)
cloned = self.jsonSender.get("request/%s" % result[0]["RequestName"])
clonedRequest = cloned[0]
# these request arguments shall differ in the cloned request:
# RequestName, ReqMgrRequestID
# "RequestDate" and "timeStamp" will be the same in the test
toDiffer = ["RequestName", "ReqMgrRequestID", "RequestWorkflow"]
for differ in toDiffer:
self.assertNotEqual(origRequest[differ], clonedRequest[differ])
toDiffer.extend(["RequestDate", "timeStamp"])
for differ in toDiffer:
del origRequest[differ]
del clonedRequest[differ]
# check the request dictionaries
self.assertEquals(len(origRequest), len(clonedRequest))
for k1, k2 in zip(sorted(origRequest.keys()), sorted(clonedRequest.keys())):
msg = ("Request values: original: %s: %s cloned: %s: %s differ" %
(k1, origRequest[k1], k2, clonedRequest[k2]))
self.assertEqual(origRequest[k1], clonedRequest[k2], msg)
def testB_Analysis(self):
"""
_Analysis_
Test Analysis workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "Analysis"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
def testG_AddDuplicateUser(self):
"""
_AddDuplicateUser_
Test and see if we get a sensible error when adding a duplicate user.
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
raises = False
try:
self.jsonSender.put('group/%s/%s' % (groupName, userName))
except HTTPException, ex:
self.assertTrue("User/Group Already Linked in DB" in ex.result)
self.assertEqual(ex.status, 400)
raises = True
def testB_Analysis(self):
"""
_Analysis_
Test Analysis workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "Analysis"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
def testF_TestWhitelistBlacklist(self):
"""
_TestWhitelistBlacklist_
Test whether or not we can assign the block/run blacklist/whitelist
"""
userName = "******"
groupName = "Li"
teamName = "Tang"
schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName)
schema["RunWhitelist"] = [1, 2, 3]
schema["RunBlacklist"] = [4, 5, 6]
schema["BlockWhitelist"] = ["/dataset/dataset/dataset#alpha"]
schema["BlockBlacklist"] = ["/dataset/dataset/dataset#beta"]
result = self.jsonSender.put("request/testRequest", schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
workload = self.loadWorkload(requestName=requestName)
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema["RunWhitelist"])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema["RunBlacklist"])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema["BlockWhitelist"])
self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema["BlockBlacklist"])
req = self.jsonSender.get("request/%s" % requestName)
self.assertTrue(req[0].has_key("Site Blacklist"))
self.assertTrue(req[0].has_key("Site Whitelist"))
schema["BlockBlacklist"] = {"1": "/dataset/dataset/dataset#beta"}
try:
raises = False
result = self.jsonSender.put("request/testRequest", schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Bad Run list of type " in ex.result)
pass
def testE_StoreResults(self):
"""
_StoreResults_
Test StoreResults workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "StoreResults"
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Argument CmsPath is required."
in ex.result)
self.assertTrue(raises)
schema['DbsUrl'] = 'http://fake.dbs.url/dbs'
schema['CmsPath'] = '/fake/tmp/path'
schema['AcquisitionEra'] = 'era'
schema['ProcessingVersion'] = 1
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['DbsUrl'], schema['DbsUrl'])
def testD_ReDigi(self):
"""
_ReDigi_
Test ReDigi workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "ReDigi"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument StepOneConfigCacheID is required." in ex.result)
def testE_CheckStatusChanges(self):
"""
_CheckStatusChanges_
Check status changes for a single request. See whether
we can move the request through the proper chain. Figure
out what happens when we fail.
"""
myThread = threading.currentThread()
userName = "******"
groupName = "Li"
teamName = "Tang"
schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName)
result = self.jsonSender.put("request/testRequest", schema)
self.assertEqual(result[1], 200)
requestName = result[0]["RequestName"]
# There should only be one request in the DB
result = GetRequest.requestID(requestName=requestName)
self.assertEqual(result, 1)
result = self.jsonSender.get("request/%s" % requestName)
self.assertEqual(result[0]["Group"], groupName)
self.assertEqual(result[0]["Requestor"], userName)
# Let's see what we can do in terms of setting status
self.changeStatusAndCheck(requestName=requestName, statusName="new")
# Let's try an illegal status change, just for the hell of it
raises = False
try:
self.jsonSender.put("request/%s?status=negotiating" % requestName)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 403)
self.assertTrue("Failed to change status" in ex.result)
self.assertFalse(requestName in ex.result)
def testE_StoreResults(self):
"""
_StoreResults_
Test StoreResults workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "StoreResults"
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument CmsPath is required." in ex.result)
self.assertTrue(raises)
schema['DbsUrl'] = 'http://fake.dbs.url/dbs'
schema['CmsPath'] = '/fake/tmp/path'
schema['AcquisitionEra'] = 'era'
schema['ProcessingVersion'] = 1
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['DbsUrl'], schema['DbsUrl'])
def testI_RelValMC(self):
"""
_RelValMC_
Test RelValMC workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "RelValMC"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Missing required field "
"'PrimaryDataset' in workload validation!" in ex.result)
def setUp(self):
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
# logging stuff from TestInit is broken, setting myself
l = logging.getLogger()
l.setLevel(logging.DEBUG)
self.params = {}
self.params['endpoint'] = self.config.getServerUrl()
self.reqService = RequestManagerDS(self.params)
self.jsonSender = JSONRequests(self.config.getServerUrl())
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
try:
r = self.jsonSender.put('request/' + schema['RequestName'], schema)
except Exception, ex:
print "Exception during set up, reason: %s" % ex
return
def testI_RelValMC(self):
"""
_RelValMC_
Test RelValMC workflows
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "RelValMC"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Missing required field "
"'PrimaryDataset' in workload validation!" in ex.result)