def testJ_Resubmission(self): """ _Resubmission_ Test Resubmission """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "DataProcessing" schema['ProdScenario'] = 'pp' result = self.jsonSender.put('request/testRequest', schema) requestName = result[0]['RequestName'] # user, group schema already set up schema = utils.getSchema(groupName=groupName, userName=userName) schema['RequestType'] = "Resubmission" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Missing required " "field 'OriginalRequestName' in workload validation!" in ex.result)
def testK_LHEStepZero(self): """ _LHEStepZero_ Test LHEStepZero workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "LHEStepZero" # Set some versions schema['ProcessingVersion'] = '2012' schema['AcquisitionEra'] = 'ae2012' schema['FirstEvent'] = 1 schema['FirstLumi'] = 1 try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Missing required " "field 'ConfigCacheID' in workload validation!" in ex.result)
def testI_CheckConfigIDs(self): """ _CheckConfigIDs_ Check to see if we can pull out the ConfigIDs by request """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) # Set some versions schema['ProcessingVersion'] = '2012' schema['AcquisitionEra'] = 'ae2012' schema["PrimaryDataset"] = "ReallyFake" schema["RequestNumEvents"] = 100 configID = self.createConfig() schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["ConfigCacheID"] = configID schema["InputDatasets"] = ['/MinimumBias/Run2010B-RelValRawSkim-v1/RAW'] result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] result = self.jsonSender.get('configIDs?prim=MinimumBias&proc=Commissioning10-v4&tier=RAW')[0] self.assertTrue(requestName in result.keys()) self.assertTrue(configID in result[requestName][0])
def testJ_Resubmission(self): """ _Resubmission_ Test Resubmission """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "DataProcessing" schema['ProdScenario'] = 'pp' result = self.jsonSender.put('request/testRequest', schema) requestName = result[0]['RequestName'] # user, group schema already set up schema = utils.getSchema(groupName = groupName, userName = userName) schema['RequestType'] = "Resubmission" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Missing required " "field 'OriginalRequestName' in workload validation!" in ex.result)
def testC_DataProcessing(self): """ _DataProcessing_ Test DataProcessing workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "DataProcessing" del schema['ProcScenario'] try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "No Scenario or Config in Processing Request!" in ex.result)
def testB_InvalidPriority(self): """ _InvalidPriority_ Put in a bunch of invalid values for priorities and see what the code makes of them. """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] raises = False try: priority = sys.maxint + 1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result)
def testG_MonteCarloFromGEN(self): """ _MonteCarloFromGEN_ Test MonteCarloFromGEN workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "MonteCarloFromGEN" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) print ex.result self.assertTrue( "Error in Workload Validation: Argument ConfigCacheID can't be None" in ex.result)
def testB_Analysis(self): """ _Analysis_ Test Analysis workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "Analysis" try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue(raises) # Put the right things in the schema # And watch it fail, because we can't open a secure connection to CERN # in a unittest well schema['RequestorDN'] = 'SomeDN'
def testJ_Resubmission(self): """ _Resubmission_ Test Resubmission """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "ReReco" configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put('request/testRequest', schema) requestName = result[0]['RequestName'] # user, group schema already set up schema = utils.getSchema(groupName = groupName, userName = userName) schema['RequestType'] = "Resubmission" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Argument InitialTaskPath is required." in ex.result)
def testB_InvalidPriority(self): """ _InvalidPriority_ Put in a bunch of invalid values for priorities and see what the code makes of them. """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] raises = False try: priority = sys.maxint + 1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Priority must have abs() less then MAXINT!" in ex.result)
def testH_MonteCarlo(self): """ _MonteCarlo_ Test MonteCarlo workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "MonteCarlo" schema['DbsUrl'] = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' # Set some versions schema['ProcessingVersion'] = '2012' schema['AcquisitionEra'] = 'ae2012' try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Argument ConfigCacheID can't be None" in ex.result)
def testD_ReDigi(self): """ _ReDigi_ Test ReDigi workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "ReDigi" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Missing required " "field 'StepOneConfigCacheID' in workload validation!" in ex.result)
def testE_StoreResults(self): """ _StoreResults_ Test StoreResults workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "StoreResults" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) print ex.result self.assertTrue( "Error in Workload Validation: Argument CmsPath is required." in ex.result)
def setUp(self): RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") # logging stuff from TestInit is broken, setting myself l = logging.getLogger() l.setLevel(logging.DEBUG) self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['ConfigCacheID'] = self.createConfig() schema['CouchDBName'] = self.couchDBName schema['CouchWorkloadDBName'] = self.couchDBName try: r = self.jsonSender.put('request', schema) try: self.requestName = r[0]['RequestName'] except: self.requestName = r[0].values()[0]['RequestName'] except Exception as ex: msg = traceback.format_exc() print "Exception during set up, reason: %s" % msg raise ex
def testA_makeReRecoWorkload(self): """ _makeReRecoWorkload_ Check that you can make a basic ReReco workload """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) del schema['GlobalTag'] configID = self.createConfig(bad=False) schema['ConfigCacheID'] = configID raises = False try: self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Argument GlobalTag is required." in ex.result)
def testI_CheckConfigIDs(self): """ _CheckConfigIDs_ Check to see if we can pull out the ConfigIDs by request """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) # Set some versions schema['ProcessingVersion'] = '2012' schema['AcquisitionEra'] = 'ae2012' schema["PrimaryDataset"] = "ReallyFake" schema["RequestNumEvents"] = 100 configID = self.createConfig() schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["ConfigCacheID"] = configID schema["InputDataset"] = '/MinimumBias/Run2010B-RelValRawSkim-v1/RAW' result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] result = self.jsonSender.get('configIDs?prim=MinimumBias&proc=Run2010B-RelValRawSkim-v1&tier=RAW')[0] print result self.assertTrue(requestName in result.keys()) self.assertTrue(configID in result[requestName][0])
def testH_MonteCarlo(self): """ _MonteCarlo_ Test MonteCarlo workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "MonteCarlo" schema['DbsUrl'] = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' # Set some versions schema['ProcessingVersion'] = '2012' schema['AcquisitionEra'] = 'ae2012' try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Argument ConfigCacheID can't be None" in ex.result)
def testH_RemoveSoftwareVersion(self): """ _RemoveSoftwareVersion_ Remove the software version after submitting the request. See what that does. """ myThread = threading.currentThread() userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] req = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]]) # Delete software versions and make sure they're gone from the DB SoftwareManagement.removeSoftware(softwareName=schema["CMSSWVersion"], scramArch=schema["ScramArch"]) versions = myThread.dbi.processData( "SELECT * FROM reqmgr_software")[0].fetchall() self.assertEqual(versions, []) assocs = myThread.dbi.processData( "SELECT * FROM reqmgr_software_dependency")[0].fetchall() self.assertEqual(assocs, []) req = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
def testB_Analysis(self): """ _Analysis_ Test Analysis workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "Analysis" try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue(raises) # Put the right things in the schema # And watch it fail, because we can't open a secure connection to CERN # in a unittest well schema['RequestorDN'] = 'SomeDN'
def testH_RemoveSoftwareVersion(self): """ _RemoveSoftwareVersion_ Remove the software version after submitting the request. See what that does. """ myThread = threading.currentThread() userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] req = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]]) # Delete software versions and make sure they're gone from the DB SoftwareManagement.removeSoftware(softwareName = schema["CMSSWVersion"], scramArch = schema["ScramArch"]) versions = myThread.dbi.processData("SELECT * FROM reqmgr_software")[0].fetchall() self.assertEqual(versions, []) assocs = myThread.dbi.processData("SELECT * FROM reqmgr_software_dependency")[0].fetchall() self.assertEqual(assocs, []) req = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(req['SoftwareVersions'], [schema["CMSSWVersion"]])
def setUp(self): RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") # logging stuff from TestInit is broken, setting myself l = logging.getLogger() l.setLevel(logging.DEBUG) self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['ConfigCacheID'] = self.createConfig() schema['CouchDBName'] = self.couchDBName schema['CouchWorkloadDBName'] = self.couchDBName try: r = self.jsonSender.put('request', schema) try: self.requestName = r[0]['RequestName'] except: self.requestName = r[0].values()[0]['RequestName'] except Exception as ex: msg = traceback.format_exc() print("Exception during set up, reason: %s" % msg) raise ex
def testK_LHEStepZero(self): """ _LHEStepZero_ Test LHEStepZero workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "LHEStepZero" # Set some versions schema['ProcessingVersion'] = '2012' schema['AcquisitionEra'] = 'ae2012' schema['FirstEvent'] = 1 schema['FirstLumi'] = 1 schema['DbsUrl'] = 'http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet' try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Missing required " "field 'ConfigCacheID' in workload validation!" in ex.result)
def testJ_CheckRequestCloning(self): myThread = threading.currentThread() userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) result = self.jsonSender.put("request", schema) self.assertEqual(result[1], 200) requestName = result[0]["RequestName"] acquisitionEra = result[0]["AcquisitionEra"] self.assertTrue(schema["AcquisitionEra"], acquisitionEra) # set some non-default priority # when cloning a request which had some non default priority, # the priority values were lost when creating a cloned request, the # default values were lost. Change it here to specifically catch this case. priority = 300 result = self.jsonSender.put("request/%s?priority=%s" % (requestName, priority)) self.assertEqual(result[1], 200) # get the original request from the server, although the variable result # shall have the same stuff in response = self.jsonSender.get("request/%s" % requestName) origRequest = response[0] self.assertEquals(origRequest["AcquisitionEra"], acquisitionEra) # test that the priority was correctly set in the brand-new request self.assertEquals(origRequest["RequestPriority"], priority) # test cloning not existing request self.assertRaises(HTTPException, self.jsonSender.put, "clone/%s" % "NotExistingRequestName") # correct attempt to clone the request # this is the new request, it'll have different name result = self.jsonSender.put("clone/%s" % requestName) # get the cloned request from the server respose = self.jsonSender.get("request/%s" % result[0]["RequestName"]) clonedRequest = respose[0] # these request arguments shall differ in the cloned request: toDiffer = ["RequestName", "RequestStatus"] for differ in toDiffer: self.assertNotEqual(origRequest[differ], clonedRequest[differ]) # check the desired status of the cloned request self.assertEquals(clonedRequest["RequestStatus"], "assignment-approved", "Cloned request status should be 'assignment-approved', not '%s'." % clonedRequest["RequestStatus"]) # don't care about these two (they will likely be the same in the unittest # since the brand new request injection as well as the cloning probably # happen at roughly the same time) toDiffer.extend(["RequestDate", "timeStamp", "RequestWorkflow"]) for differ in toDiffer: del origRequest[differ] del clonedRequest[differ] # check the request dictionaries self.assertEquals(len(origRequest), len(clonedRequest)) for k1, k2 in zip(sorted(origRequest.keys()), sorted(clonedRequest.keys())): msg = ("Request values: original: %s: %s cloned: %s: %s differ" % (k1, origRequest[k1], k2, clonedRequest[k2])) self.assertEqual(origRequest[k1], clonedRequest[k2], msg)
def testF_TestWhitelistBlacklist(self): """ _TestWhitelistBlacklist_ Test whether or not we can assign the block/run blacklist/whitelist """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RunWhitelist'] = [1, 2, 3] schema['RunBlacklist'] = [4, 5, 6] schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha'] schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta'] configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] workload = self.loadWorkload(requestName=requestName) self.assertEqual( workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema['RunWhitelist']) self.assertEqual( workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema['RunBlacklist']) self.assertEqual( workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema['BlockWhitelist']) self.assertEqual( workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema['BlockBlacklist']) req = self.jsonSender.get('request/%s' % requestName) self.assertTrue(req[0].has_key('Site Blacklist')) self.assertTrue(req[0].has_key('Site Whitelist')) schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'} try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) print ex.result self.assertTrue( "Error in Workload Validation: Argument BlockBlacklist type is incorrect in schema." in ex.result) pass
def testJ_Resubmission(self): """ _Resubmission_ Test Resubmission """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "ReReco" configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] # user, group schema already set up schema = utils.getSchema(groupName=groupName, userName=userName) schema['RequestType'] = "Resubmission" schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Validation failed: InitialTaskPath is mendatory" in ex.result) self.assertTrue(raises) schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName schema["ACDCServer"] = os.environ.get("COUCHURL") schema["ACDCDatabase"] = self.couchDBName schema["CollectionName"] = "SomeOtherName" # Here we just make sure that real result goes through result = self.jsonSender.put('request', schema) resubmitName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % resubmitName) couchServer = CouchServer(self.testInit.couchUrl) reqmgrCouch = couchServer.connectDatabase(self.couchDBName) result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests', {}, [requestName])['rows'] self.assertEqual(len(result), 1) self.assertEqual(result[0]['key'], requestName) self.assertEqual(result[0]['id'], resubmitName)
def testB_ReReco(self): """ _ReReco_ Try a basic ReReco workflow """ schema = utils.getAndSetupSchema(self) schema['RequestNumEvents'] = 100 schema['RequestEventSize'] = 101 self.doRequest(schema)
def testJ_Resubmission(self): """ _Resubmission_ Test Resubmission """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "ReReco" configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] # user, group schema already set up schema = utils.getSchema(groupName = groupName, userName = userName) schema['RequestType'] = "Resubmission" schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Validation failed: InitialTaskPath is mendatory" in ex.result) self.assertTrue(raises) schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName schema["ACDCServer"] = os.environ.get("COUCHURL") schema["ACDCDatabase"] = self.couchDBName schema["CollectionName"] = "SomeOtherName" # Here we just make sure that real result goes through result = self.jsonSender.put('request', schema) resubmitName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % resubmitName) couchServer = CouchServer(self.testInit.couchUrl) reqmgrCouch = couchServer.connectDatabase(self.couchDBName) result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests', {}, [requestName])['rows'] self.assertEqual(len(result), 1) self.assertEqual(result[0]['key'], requestName) self.assertEqual(result[0]['id'], resubmitName)
def testG_MonteCarloFromGEN(self): """ _MonteCarloFromGEN_ Test MonteCarloFromGEN workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "MonteCarloFromGEN" try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) schema["ConfigCacheID"] = "fakeID" schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) configID = self.createConfig() schema["ConfigCacheID"] = configID result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['SizePerEvent'], 512) self.assertEqual(request['RequestNumEvents'], 0) self.assertEqual(request['RequestSizeFiles'], 0)
def testC_UserGroupRequestPriority(self): """ _UserGroupRequestPriority_ Set the priorities of the user, the group, and the request """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), 0) # Reset user, group priorities to 0 self.jsonSender.post('user/%s?priority=0' % userName) self.jsonSender.post('group/%s?priority=0' % groupName) # Set priority == 5 priority = 5 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority) self.jsonSender.post('user/%s?priority=6' % userName) self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority + 6) self.jsonSender.post('group/%s?priority=7' % groupName) self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority + 6 + 7) self.jsonSender.post('user/%s?priority=0' % userName) self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.priority(), priority + 7)
def testC_UserGroupRequestPriority(self): """ _UserGroupRequestPriority_ Set the priorities of the user, the group, and the request """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), 0) # Reset user, group priorities to 0 self.jsonSender.post('user/%s?priority=0' % userName) self.jsonSender.post('group/%s?priority=0' % groupName) # Set priority == 5 priority = 5 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority) self.jsonSender.post('user/%s?priority=6' % userName) self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority + 6) self.jsonSender.post('group/%s?priority=7' % groupName) self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority + 6 + 7) self.jsonSender.post('user/%s?priority=0' % userName) self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) request = self.jsonSender.get('request/%s' % requestName)[0] self.assertEqual(request['ReqMgrRequestBasePriority'], priority) workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.priority(), priority + 7)
def testM_PutRequestStats(self): userName = '******' groupName = 'Bryant' teamName = 'Lakers' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) result = self.jsonSender.put("request", schema)[0] originalRequest = result['RequestName'] stats = {'total_jobs': 100, 'input_events': 100, 'input_lumis': 100, 'input_num_files': 100} result = self.reqService.putRequestStats(originalRequest, stats) self.assertEqual(result['RequestName'], originalRequest)
def testF_TestWhitelistBlacklist(self): """ _TestWhitelistBlacklist_ Test whether or not we can assign the block/run blacklist/whitelist """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RunWhitelist'] = [1, 2, 3] schema['RunBlacklist'] = [4, 5, 6] schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha'] schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta'] result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] workload = self.loadWorkload(requestName=requestName) self.assertEqual( workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema['RunWhitelist']) self.assertEqual( workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema['RunBlacklist']) self.assertEqual( workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema['BlockWhitelist']) self.assertEqual( workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema['BlockBlacklist']) req = self.jsonSender.get('request/%s' % requestName) self.assertTrue(req[0].has_key('Site Blacklist')) self.assertTrue(req[0].has_key('Site Whitelist')) schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'} try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Bad Run list of type " in ex.result) pass
def testG_MonteCarloFromGEN(self): """ _MonteCarloFromGEN_ Test MonteCarloFromGEN workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "MonteCarloFromGEN" try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) schema["ConfigCacheID"] = "fakeID" schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Failure to load ConfigCache while validating workload" in ex.result) self.assertTrue(raises) configID = self.createConfig() schema["ConfigCacheID"] = configID result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['SizePerEvent'], 512) self.assertEqual(request['RequestNumEvents'], 0) self.assertEqual(request['RequestSizeFiles'], 0)
def testK_CheckRequestFailsInjectionForbiddenInputArg(self): myThread = threading.currentThread() userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) from WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel import deprecatedRequestArgs for deprec in deprecatedRequestArgs: schema = utils.getSchema(groupName=groupName, userName=userName) schema[deprec] = "something" self.assertRaises(HTTPException, self.jsonSender.put, "request", schema)
def testF_TestWhitelistBlacklist(self): """ _TestWhitelistBlacklist_ Test whether or not we can assign the block/run blacklist/whitelist """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RunWhitelist'] = [1, 2, 3] schema['RunBlacklist'] = [4, 5, 6] schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha'] schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta'] configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema['RunWhitelist']) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema['RunBlacklist']) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema['BlockWhitelist']) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema['BlockBlacklist']) req = self.jsonSender.get('request/%s' % requestName) self.assertTrue(req[0].has_key('Site Blacklist')) self.assertTrue(req[0].has_key('Site Whitelist')) schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'} try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) print ex.result self.assertTrue("Error in Workload Validation: Argument BlockBlacklist type is incorrect in schema." in ex.result) pass
def testB_InvalidPriority(self): """ _InvalidPriority_ Put in a bunch of invalid values for priorities and see what the code makes of them. """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] raises = False try: priority = sys.maxsize + 1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Priority must have abs() less then MAXINT!" in ex.result) self.assertTrue(raises) raises = False try: priority = -1 - sys.maxsize self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) print(ex.result) self.assertTrue( "Priority must have abs() less then MAXINT!" in ex.result) self.assertTrue(raises)
def testB_ReReco(self): """ _ReReco_ Try a basic ReReco workflow """ schema = utils.getAndSetupSchema(self) schema['RequestNumEvents'] = 100 schema['SizePerEvent'] = 101 configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") self.doRequest(schema)
def testE_CheckStatusChanges(self): """ _CheckStatusChanges_ Check status changes for a single request. See whether we can move the request through the proper chain. Figure out what happens when we fail. """ myThread = threading.currentThread() userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] # There should only be one request in the DB result = GetRequest.requestID(requestName = requestName) self.assertEqual(result, 1) result = self.jsonSender.get('request/%s' % requestName) self.assertEqual(result[0]['Group'], groupName) self.assertEqual(result[0]['Requestor'], userName) # Let's see what we can do in terms of setting status self.changeStatusAndCheck(requestName = requestName, statusName = 'new') # Let's try an illegal status change, just for the hell of it raises = False try: self.jsonSender.put('request/%s?status=negotiating' % requestName) except HTTPException, ex: raises = True self.assertEqual(ex.status, 403) self.assertTrue('Failed to change status' in ex.result) self.assertFalse(requestName in ex.result)
def testE_CheckStatusChanges(self): """ _CheckStatusChanges_ Check status changes for a single request. See whether we can move the request through the proper chain. Figure out what happens when we fail. """ myThread = threading.currentThread() userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] # There should only be one request in the DB result = GetRequest.requestID(requestName=requestName) self.assertEqual(result, 1) result = self.jsonSender.get('request/%s' % requestName) self.assertEqual(result[0]['Group'], groupName) self.assertEqual(result[0]['Requestor'], userName) # Let's see what we can do in terms of setting status self.changeStatusAndCheck(requestName=requestName, statusName='new') # Let's try an illegal status change, just for the hell of it raises = False try: self.jsonSender.put('request/%s?status=negotiating' % requestName) except HTTPException, ex: raises = True self.assertEqual(ex.status, 403) self.assertTrue('Failed to change status' in ex.result) self.assertFalse(requestName in ex.result)
def testM_PutRequestStats(self): userName = '******' groupName = 'Bryant' teamName = 'Lakers' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put("request", schema)[0] originalRequest = result['RequestName'] stats = {'total_jobs': 100, 'input_events': 100, 'input_lumis': 100, 'input_num_files': 100} result = self.reqService.putRequestStats(originalRequest, stats) self.assertEqual(result['RequestName'], originalRequest)
def testG_AddDuplicateUser(self): """ _AddDuplicateUser_ Test and see if we get a sensible error when adding a duplicate user. """ userName = "******" groupName = "Li" teamName = "Tang" schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) raises = False try: self.jsonSender.put("group/%s/%s" % (groupName, userName)) except HTTPException, ex: self.assertTrue("User/Group Already Linked in DB" in ex.result) self.assertEqual(ex.status, 400) raises = True
def testB_InvalidPriority(self): """ _InvalidPriority_ Put in a bunch of invalid values for priorities and see what the code makes of them. """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] raises = False try: priority = sys.maxint + 1 self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result) self.assertTrue(raises) raises = False try: priority = -1 - sys.maxint self.jsonSender.put('request/%s?priority=%s' % (requestName, priority)) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) print ex.result self.assertTrue("Priority must have abs() less then MAXINT!" in ex.result) self.assertTrue(raises)
def testF_TestWhitelistBlacklist(self): """ _TestWhitelistBlacklist_ Test whether or not we can assign the block/run blacklist/whitelist """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RunWhitelist'] = [1, 2, 3] schema['RunBlacklist'] = [4, 5, 6] schema['BlockWhitelist'] = ['/dataset/dataset/dataset#alpha'] schema['BlockBlacklist'] = ['/dataset/dataset/dataset#beta'] result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] workload = self.loadWorkload(requestName = requestName) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema['RunWhitelist']) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema['RunBlacklist']) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema['BlockWhitelist']) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema['BlockBlacklist']) req = self.jsonSender.get('request/%s' % requestName) self.assertTrue(req[0].has_key('Site Blacklist')) self.assertTrue(req[0].has_key('Site Whitelist')) schema['BlockBlacklist'] = {'1': '/dataset/dataset/dataset#beta'} try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Bad Run list of type " in ex.result) pass
def testJ_CheckRequestCloning(self): myThread = threading.currentThread() userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) result = self.jsonSender.put('request', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] # AcquisitionEra is returned here, but in fact on server is not stored until assign self.assertTrue(schema["AcquisitionEra"], result[0]["AcquisitionEra"]) # get the original request (although the variable result shall have the same stuff in) origRequest = self.jsonSender.get("request/%s" % requestName) origRequest = origRequest[0] self.assertEquals(origRequest["AcquisitionEra"], "None") # was not stored # test cloning not existing request self.assertRaises(HTTPException, self.jsonSender.put, "clone/%s" % "NotExistingRequestName") # this is the new request, it'll have different name result = self.jsonSender.put("clone/%s" % requestName) cloned = self.jsonSender.get("request/%s" % result[0]["RequestName"]) clonedRequest = cloned[0] # these request arguments shall differ in the cloned request: # RequestName, ReqMgrRequestID # "RequestDate" and "timeStamp" will be the same in the test toDiffer = ["RequestName", "ReqMgrRequestID", "RequestWorkflow"] for differ in toDiffer: self.assertNotEqual(origRequest[differ], clonedRequest[differ]) toDiffer.extend(["RequestDate", "timeStamp"]) for differ in toDiffer: del origRequest[differ] del clonedRequest[differ] # check the request dictionaries self.assertEquals(len(origRequest), len(clonedRequest)) for k1, k2 in zip(sorted(origRequest.keys()), sorted(clonedRequest.keys())): msg = ("Request values: original: %s: %s cloned: %s: %s differ" % (k1, origRequest[k1], k2, clonedRequest[k2])) self.assertEqual(origRequest[k1], clonedRequest[k2], msg)
def testB_Analysis(self): """ _Analysis_ Test Analysis workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "Analysis" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400)
def testG_AddDuplicateUser(self): """ _AddDuplicateUser_ Test and see if we get a sensible error when adding a duplicate user. """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) raises = False try: self.jsonSender.put('group/%s/%s' % (groupName, userName)) except HTTPException, ex: self.assertTrue("User/Group Already Linked in DB" in ex.result) self.assertEqual(ex.status, 400) raises = True
def testB_Analysis(self): """ _Analysis_ Test Analysis workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "Analysis" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400)
def testF_TestWhitelistBlacklist(self): """ _TestWhitelistBlacklist_ Test whether or not we can assign the block/run blacklist/whitelist """ userName = "******" groupName = "Li" teamName = "Tang" schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema["RunWhitelist"] = [1, 2, 3] schema["RunBlacklist"] = [4, 5, 6] schema["BlockWhitelist"] = ["/dataset/dataset/dataset#alpha"] schema["BlockBlacklist"] = ["/dataset/dataset/dataset#beta"] result = self.jsonSender.put("request/testRequest", schema) self.assertEqual(result[1], 200) requestName = result[0]["RequestName"] workload = self.loadWorkload(requestName=requestName) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.whitelist, schema["RunWhitelist"]) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.runs.blacklist, schema["RunBlacklist"]) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.whitelist, schema["BlockWhitelist"]) self.assertEqual(workload.data.tasks.DataProcessing.input.dataset.blocks.blacklist, schema["BlockBlacklist"]) req = self.jsonSender.get("request/%s" % requestName) self.assertTrue(req[0].has_key("Site Blacklist")) self.assertTrue(req[0].has_key("Site Whitelist")) schema["BlockBlacklist"] = {"1": "/dataset/dataset/dataset#beta"} try: raises = False result = self.jsonSender.put("request/testRequest", schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Bad Run list of type " in ex.result) pass
def testE_StoreResults(self): """ _StoreResults_ Test StoreResults workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "StoreResults" schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Argument CmsPath is required." in ex.result) self.assertTrue(raises) schema['DbsUrl'] = 'http://fake.dbs.url/dbs' schema['CmsPath'] = '/fake/tmp/path' schema['AcquisitionEra'] = 'era' schema['ProcessingVersion'] = 1 result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['DbsUrl'], schema['DbsUrl'])
def testD_ReDigi(self): """ _ReDigi_ Test ReDigi workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "ReDigi" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Argument StepOneConfigCacheID is required." in ex.result)
def testE_CheckStatusChanges(self): """ _CheckStatusChanges_ Check status changes for a single request. See whether we can move the request through the proper chain. Figure out what happens when we fail. """ myThread = threading.currentThread() userName = "******" groupName = "Li" teamName = "Tang" schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) result = self.jsonSender.put("request/testRequest", schema) self.assertEqual(result[1], 200) requestName = result[0]["RequestName"] # There should only be one request in the DB result = GetRequest.requestID(requestName=requestName) self.assertEqual(result, 1) result = self.jsonSender.get("request/%s" % requestName) self.assertEqual(result[0]["Group"], groupName) self.assertEqual(result[0]["Requestor"], userName) # Let's see what we can do in terms of setting status self.changeStatusAndCheck(requestName=requestName, statusName="new") # Let's try an illegal status change, just for the hell of it raises = False try: self.jsonSender.put("request/%s?status=negotiating" % requestName) except HTTPException, ex: raises = True self.assertEqual(ex.status, 403) self.assertTrue("Failed to change status" in ex.result) self.assertFalse(requestName in ex.result)
def testE_StoreResults(self): """ _StoreResults_ Test StoreResults workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "StoreResults" schema["CouchWorkloadDBName"] = self.couchDBName try: raises = False result = self.jsonSender.put('request', schema) except HTTPException as ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Argument CmsPath is required." in ex.result) self.assertTrue(raises) schema['DbsUrl'] = 'http://fake.dbs.url/dbs' schema['CmsPath'] = '/fake/tmp/path' schema['AcquisitionEra'] = 'era' schema['ProcessingVersion'] = 1 result = self.jsonSender.put('request', schema) requestName = result[0]['RequestName'] result = self.jsonSender.get('request/%s' % requestName) request = result[0] self.assertEqual(request['CMSSWVersion'], schema['CMSSWVersion']) self.assertEqual(request['Group'], groupName) self.assertEqual(request['Requestor'], userName) self.assertEqual(request['DbsUrl'], schema['DbsUrl'])
def testI_RelValMC(self): """ _RelValMC_ Test RelValMC workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['RequestType'] = "RelValMC" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue("Error in Workload Validation: Missing required field " "'PrimaryDataset' in workload validation!" in ex.result)
def setUp(self): RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") # logging stuff from TestInit is broken, setting myself l = logging.getLogger() l.setLevel(logging.DEBUG) self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) try: r = self.jsonSender.put('request/' + schema['RequestName'], schema) except Exception, ex: print "Exception during set up, reason: %s" % ex return
def testI_RelValMC(self): """ _RelValMC_ Test RelValMC workflows """ userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['RequestType'] = "RelValMC" try: raises = False result = self.jsonSender.put('request/testRequest', schema) except HTTPException, ex: raises = True self.assertEqual(ex.status, 400) self.assertTrue( "Error in Workload Validation: Missing required field " "'PrimaryDataset' in workload validation!" in ex.result)