def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "ReReco"
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['RequestType'] = "Resubmission"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument InitialTaskPath is required." in ex.result)
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "ReReco"
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['RequestType'] = "Resubmission"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument InitialTaskPath is required." in ex.result)
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "DataProcessing"
schema['ProdScenario'] = 'pp'
result = self.jsonSender.put('request/testRequest', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['RequestType'] = "Resubmission"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Missing required "
"field 'OriginalRequestName' in workload validation!" in ex.result)
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "DataProcessing"
schema['ProdScenario'] = 'pp'
result = self.jsonSender.put('request/testRequest', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName=groupName, userName=userName)
schema['RequestType'] = "Resubmission"
try:
raises = False
result = self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Missing required "
"field 'OriginalRequestName' in workload validation!" in
ex.result)
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema['RequestType'] = "ReReco"
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName=groupName, userName=userName)
schema['RequestType'] = "Resubmission"
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Validation failed: InitialTaskPath is mendatory"
in ex.result)
self.assertTrue(raises)
schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName
schema["ACDCServer"] = os.environ.get("COUCHURL")
schema["ACDCDatabase"] = self.couchDBName
schema["CollectionName"] = "SomeOtherName"
# Here we just make sure that real result goes through
result = self.jsonSender.put('request', schema)
resubmitName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % resubmitName)
couchServer = CouchServer(self.testInit.couchUrl)
reqmgrCouch = couchServer.connectDatabase(self.couchDBName)
result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests',
{}, [requestName])['rows']
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['key'], requestName)
self.assertEqual(result[0]['id'], resubmitName)
def testJ_Resubmission(self):
"""
_Resubmission_
Test Resubmission
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema['RequestType'] = "ReReco"
configID = self.createConfig()
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
# user, group schema already set up
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['RequestType'] = "Resubmission"
schema["CouchWorkloadDBName"] = self.couchDBName
try:
raises = False
result = self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Validation failed: InitialTaskPath is mendatory" in ex.result)
self.assertTrue(raises)
schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName
schema["ACDCServer"] = os.environ.get("COUCHURL")
schema["ACDCDatabase"] = self.couchDBName
schema["CollectionName"] = "SomeOtherName"
# Here we just make sure that real result goes through
result = self.jsonSender.put('request', schema)
resubmitName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % resubmitName)
couchServer = CouchServer(self.testInit.couchUrl)
reqmgrCouch = couchServer.connectDatabase(self.couchDBName)
result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests', {}, [requestName])['rows']
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['key'], requestName)
self.assertEqual(result[0]['id'], resubmitName)
def testK_CheckRequestFailsInjectionForbiddenInputArg(self):
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
from WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel import deprecatedRequestArgs
for deprec in deprecatedRequestArgs:
schema = utils.getSchema(groupName=groupName, userName=userName)
schema[deprec] = "something"
self.assertRaises(HTTPException, self.jsonSender.put, "request", schema)
def testK_CheckRequestFailsInjectionForbiddenInputArg(self):
myThread = threading.currentThread()
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
from WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel import deprecatedRequestArgs
for deprec in deprecatedRequestArgs:
schema = utils.getSchema(groupName=groupName, userName=userName)
schema[deprec] = "something"
self.assertRaises(HTTPException, self.jsonSender.put, "request", schema)
def testA_testBasicSetUp(self):
"""
_testBasicSetUp_
Moving the tests that were in the setUp category out of it,
mostly because I want to make sure that they don't fail inside
the setUp statement.
"""
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(
self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = self.jsonSender.get('group/PeopleLikeMe')[0]['users']
self.assertTrue('me' in users)
groups = self.jsonSender.get('user/me')[0]['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(
self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to dealwith
schema = utils.getSchema()
version = "version/" + schema["CMSSWVersion"]
self.assertTrue(self.jsonSender.put(version)[1] == 200)
self.assertTrue(
schema["CMSSWVersion"] in self.jsonSender.get('version')[0])
def testA_testBasicSetUp(self):
"""
_testBasicSetUp_
Moving the tests that were in the setUp category out of it,
mostly because I want to make sure that they don't fail inside
the setUp statement.
"""
if 'me' in self.jsonSender.get('user')[0]:
self.jsonSender.delete('user/me')
self.assertFalse('me' in self.jsonSender.get('user')[0])
self.assertEqual(self.jsonSender.put('user/[email protected]')[1], 200)
self.assertTrue('me' in self.jsonSender.get('user')[0])
if 'PeopleLikeMe' in self.jsonSender.get('group')[0]:
self.jsonSender.delete('group/PeopleLikeMe')
self.assertFalse('PeopleLikeMe' in self.jsonSender.get('group')[0])
self.assertEqual(self.jsonSender.put('group/PeopleLikeMe')[1], 200)
self.assertTrue( 'PeopleLikeMe' in self.jsonSender.get('group')[0])
self.jsonSender.put('group/PeopleLikeMe/me')
users = self.jsonSender.get('group/PeopleLikeMe')[0]['users']
self.assertTrue('me' in users)
groups = self.jsonSender.get('user/me')[0]['groups']
self.assertTrue('PeopleLikeMe' in groups)
groups2 = self.jsonSender.get('group?user=me')[0]
self.assertTrue('PeopleLikeMe' in groups2)
if 'White Sox' in self.jsonSender.get('team')[0]:
self.jsonSender.delete(urllib.quote('team/White Sox'))
self.assertFalse('White Sox' in self.jsonSender.get('team')[0])
self.assertEqual(self.jsonSender.put(urllib.quote('team/White Sox'))[1], 200)
self.assertTrue('White Sox' in self.jsonSender.get('team')[0])
# some foreign key stuff to dealwith
schema = utils.getSchema()
version = "version/" + schema["CMSSWVersion"]
self.assertTrue(self.jsonSender.put(version)[1] == 200)
self.assertTrue(schema["CMSSWVersion"] in self.jsonSender.get('version')[0])
"Error in Workload Validation: Argument GlobalTag is required."
in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName=groupName, userName=userName)
schema['InputDataset'] = '/Nothing'
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue(raises)
schema = utils.getSchema(groupName=groupName, userName=userName)
raises = False
del schema['ConfigCacheID']
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"rror in Workload Validation: Argument ConfigCacheID is required."
in ex.result)
self.assertTrue(raises)
configID = self.createConfig(bad=True)
schema = utils.getSchema(groupName=groupName, userName=userName)
schema["ConfigCacheID"] = configID
def testA_makeReRecoWorkload(self):
"""
_makeReRecoWorkload_
Check that you can make a basic ReReco workload
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
del schema['GlobalTag']
configID = self.createConfig(bad = False)
schema['ConfigCacheID'] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: Argument GlobalTag is required." in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['InputDataset'] = '/Nothing'
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(raises)
schema = utils.getSchema(groupName = groupName, userName = userName)
raises = False
del schema['ConfigCacheID']
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Create request failed, 'ConfigCacheID'" in ex.result)
self.assertTrue(raises)
configID = self.createConfig(bad = True)
schema = utils.getSchema(groupName = groupName, userName = userName)
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("Error in Workload Validation: "
"Duplicate dataTier/filterName combination" in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName = groupName, userName = userName)
configID = self.createConfig(bad = False)
schema['ConfigCacheID'] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
try:
result = self.jsonSender.put('request', schema)
except Exception as ex:
raise
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
def testF_TaskChain(self):
"""
_TaskChain_
Test the monstrous TaskChain workflow
This will be a long one
NOTE: This test is so complicated that all I do is
take code from TaskChain_t and make sure it still
produces and actual request
"""
couchServer = CouchServer(os.environ["COUCHURL"])
configDatabase = couchServer.connectDatabase(self.couchDBName)
generatorDoc = makeGeneratorConfig(configDatabase)
processorDocs = makeProcessingConfigs(configDatabase)
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getSchema(userName=userName)
schema["CouchURL"] = os.environ["COUCHURL"]
schema["CouchDBName"] = self.couchDBName
schema["CouchWorkloadDBName"] = self.couchDBName
schema["SiteWhitelist"] = ["T1_CH_CERN", "T1_US_FNAL"]
schema["TaskChain"] = 5
chains = {
"Task1": {
"TaskName": "GenSim",
"ConfigCacheID": generatorDoc,
"SplittingAlgo": "EventBased",
"EventsPerJob": 250,
"RequestNumEvents": 10000,
"PrimaryDataset": "RelValTTBar"
},
"Task2": {
"TaskName": "DigiHLT",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "FileBased"
},
"Task3": {
"TaskName": "Reco",
"InputTask": "DigiHLT",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgo": "FileBased"
},
"Task4": {
"TaskName": "ALCAReco",
"InputTask": "Reco",
"InputFromOutputModule": "writeALCA",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgo": "FileBased"
},
"Task5": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgo": "FileBased",
"FilesPerJob": 10
}
}
schema.update(chains)
args = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
schema.update(args)
# this is necessary and after all updates to the schema are made,
# otherwise this item will get overwritten
schema['RequestType'] = "TaskChain"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
result = self.jsonSender.put('request', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
workload = self.loadWorkload(requestName)
self.assertEqual(workload.data.request.schema.Task1["EventsPerJob"],
250)
def testA_makeReRecoWorkload(self):
"""
_makeReRecoWorkload_
Check that you can make a basic ReReco workload
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
del schema['GlobalTag']
configID = self.createConfig(bad=False)
schema['ConfigCacheID'] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Argument GlobalTag is required."
in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName=groupName, userName=userName)
schema['InputDataset'] = '/Nothing'
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(raises)
schema = utils.getSchema(groupName=groupName, userName=userName)
raises = False
del schema['ConfigCacheID']
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Create request failed, 'ConfigCacheID'" in ex.result)
self.assertTrue(raises)
configID = self.createConfig(bad=True)
schema = utils.getSchema(groupName=groupName, userName=userName)
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
raises = False
try:
self.jsonSender.put('request', schema)
except HTTPException as ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: "
"Duplicate dataTier/filterName combination" in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName=groupName, userName=userName)
configID = self.createConfig(bad=False)
schema['ConfigCacheID'] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
schema["CouchWorkloadDBName"] = self.couchDBName
try:
result = self.jsonSender.put('request', schema)
except Exception as ex:
raise
self.assertEqual(result[1], 200)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
class ReqMgrWorkloadTest(RESTBaseUnitTest):
"""
Test that sets up and checks the validations of the various main WMSpec.StdSpecs
This is mostly a simple set of tests which can be very repetitive.
"""
def setUp(self):
"""
setUP global values
Database setUp is done in base class
"""
self.couchDBName = "reqmgr_t_0"
RESTBaseUnitTest.setUp(self)
self.testInit.setupCouch("%s" % self.couchDBName, "ConfigCache",
"ReqMgr")
self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats")
reqMgrHost = self.config.getServerUrl()
self.jsonSender = JSONRequests(reqMgrHost)
def initialize(self):
self.config = RequestManagerConfig(
'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel')
self.config.setFormatter('WMCore.WebTools.RESTFormatter')
self.config.setupRequestConfig()
self.config.setupCouchDatabase(dbName=self.couchDBName)
self.config.setPort(12888)
self.schemaModules = ["WMCore.RequestManager.RequestDB"]
def tearDown(self):
"""
_tearDown_
Basic tear down of database
"""
RESTBaseUnitTest.tearDown(self)
self.testInit.tearDownCouch()
def createConfig(self, bad=False):
"""
_createConfig_
Create a config of some sort that we can load out of ConfigCache
"""
PSetTweak = {
'process': {
'outputModules_': ['ThisIsAName'],
'ThisIsAName': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
BadTweak = {
'process': {
'outputModules_': ['ThisIsAName1', 'ThisIsAName2'],
'ThisIsAName1': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
},
'ThisIsAName2': {
'dataset': {
'dataTier': 'RECO',
'filterName': 'Filter'
}
}
}
}
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDBName)
configCache.createUserGroup(groupname="testGroup", username='******')
if bad:
configCache.setPSetTweaks(PSetTweak=BadTweak)
else:
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.save()
return configCache.getCouchID()
def loadWorkload(self, requestName):
"""
_loadWorkload_
Load the workload from couch after we've saved it there.
"""
workload = WMWorkloadHelper()
url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName,
requestName)
workload.load(url)
return workload
@attr('integration')
def testA_makeReRecoWorkload(self):
"""
_makeReRecoWorkload_
Check that you can make a basic ReReco workload
"""
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getAndSetupSchema(self,
userName=userName,
groupName=groupName,
teamName=teamName)
del schema['GlobalTag']
configID = self.createConfig(bad=False)
schema['ConfigCacheID'] = configID
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue(
"Error in Workload Validation: Argument GlobalTag is required."
in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName=groupName, userName=userName)
schema['InputDataset'] = '/Nothing'
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
def testF_TaskChain(self):
"""
_TaskChain_
Test the monstrous TaskChain workflow
This will be a long one
NOTE: This test is so complicated that all I do is
take code from TaskChain_t and make sure it still
produces and actual request
"""
couchServer = CouchServer(os.environ["COUCHURL"])
configDatabase = couchServer.connectDatabase(self.couchDBName)
generatorDoc = makeGeneratorConfig(configDatabase)
processorDocs = makeProcessingConfigs(configDatabase)
userName = '******'
groupName = 'Li'
teamName = 'Tang'
schema = utils.getSchema(userName = userName)
schema["CouchURL"] = os.environ["COUCHURL"]
schema["CouchDBName"] = self.couchDBName
schema["SiteWhitelist"] = ["T1_CH_CERN", "T1_US_FNAL"]
schema["TaskChain"] = 5
chains = {"Task1" : {"TaskName" : "GenSim",
"ConfigCacheID" : generatorDoc,
"SplittingAlgorithm" : "EventBased",
"SplittingArguments" : {"events_per_job" : 250},
"RequestNumEvents" : 10000,
"Seeding" : "Automatic",
"PrimaryDataset" : "RelValTTBar"},
"Task2" : {"TaskName" : "DigiHLT",
"InputTask" : "GenSim",
"InputFromOutputModule" : "writeGENSIM",
"ConfigCacheID" : processorDocs['DigiHLT'],
"SplittingAlgorithm" : "FileBased",
"SplittingArguments" : {"files_per_job" : 1 } },
"Task3" : {"TaskName" : "Reco",
"InputTask" : "DigiHLT",
"InputFromOutputModule" : "writeRAWDIGI",
"ConfigCacheID" : processorDocs['Reco'],
"SplittingAlgorithm" : "FileBased",
"SplittingArguments" : {"files_per_job" : 1 } },
"Task4" : {"TaskName" : "ALCAReco",
"InputTask" : "Reco",
"InputFromOutputModule" : "writeALCA",
"ConfigCacheID" : processorDocs['ALCAReco'],
"SplittingAlgorithm" : "FileBased",
"SplittingArguments" : {"files_per_job" : 1 } },
"Task5" : {"TaskName" : "Skims",
"InputTask" : "Reco",
"InputFromOutputModule" : "writeRECO",
"ConfigCacheID" : processorDocs['Skims'],
"SplittingAlgorithm" : "FileBased",
"SplittingArguments" : {"files_per_job" : 10 } } }
schema.update(chains)
args = utils.getAndSetupSchema(self,
userName = userName,
groupName = groupName,
teamName = teamName)
schema.update(args)
# this is necessary and after all updates to the schema are made,
# otherwise this item will get overwritten
schema['RequestType'] = "TaskChain"
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")
result = self.jsonSender.put('request/testRequest', schema)
requestName = result[0]['RequestName']
result = self.jsonSender.get('request/%s' % requestName)
request = result[0]
self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion'])
self.assertEqual(request['Group'], groupName)
self.assertEqual(request['Requestor'], userName)
self.assertEqual(request['DbsUrl'], None)
workload = self.loadWorkload(requestName)
self.assertEqual(workload.data.request.schema.Task1.SplittingArguments,
{'events_per_job': 250})
self.assertTrue("Error in Workload Validation: Missing required "
"field 'GlobalTag' in workload validation!" in ex.result)
self.assertTrue(raises)
schema = utils.getSchema(groupName = groupName, userName = userName)
schema['InputDataset'] = '/Nothing'
raises = False
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
print ex.result
self.assertTrue(raises)
schema = utils.getSchema(groupName = groupName, userName = userName)
raises = False
del schema['ProcScenario']
try:
self.jsonSender.put('request/testRequest', schema)
except HTTPException, ex:
raises = True
self.assertEqual(ex.status, 400)
self.assertTrue("No Scenario or Config in Processing Request!" in ex.result)
self.assertTrue(raises)
configID = self.createConfig(bad = True)
schema = utils.getSchema(groupName = groupName, userName = userName)
schema["ConfigCacheID"] = configID
schema["CouchDBName"] = self.couchDBName
schema["CouchURL"] = os.environ.get("COUCHURL")