class WMStatsTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["WMStats"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = self.schema, useDefault = False) self.testInit.setupCouch('wmstats_t', *self.couchApps) self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t'); return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail'); self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"), 'ERROR: request not found - not_exist_schema') self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'), 'ERROR: request not found - not_exist_schema') totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5} self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats), 'ERROR: request not found - not_exist_schema') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2), 'ERROR: request not found - not_exist_schema')
if ds not in request['OutputDatasets']: request['OutputDatasets'].append(ds) # don't want to JSONify the whole workflow del metadata['WorkloadSpec'] workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata) request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl) try: CheckIn.checkIn(request, reqSchema['RequestType']) except CheckIn.RequestCheckInError, ex: msg = ex._message raise HTTPError(400, "Error in Request check-in: %s" % msg) try: wmstatSvc = WMStatsWriter(wmstatUrl) wmstatSvc.insertRequest(request) except Exception as ex: webApi.error("Could not update WMStats, reason: %s" % ex) raise HTTPError(400, "Creating request failed, could not update WMStats.") return request def makeRequest(webApi, reqInputArgs, couchUrl, couchDB, wmstatUrl): """ Handles the submission of requests. """ # make sure no extra spaces snuck in
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False): """ If clone is True, the function is called on a cloned request in which case no modification of the reqSchema shall happen and should be checked in as is. """ try: request = buildWorkloadForRequest(typename = reqSchema["RequestType"], schema = reqSchema) except WMSpecFactoryException as ex: logging.error(traceback.format_exc()) raise HTTPError(400, "Error in Workload Validation: %s" % ex.message()) helper = WMWorkloadHelper(request['WorkloadSpec']) # update request as well for wmstats update # there is a better way to do this (passing helper to request but make sure all the information is there) request["Campaign"] = helper.getCampaign() # Add the output datasets if necessary # for some bizarre reason OutpuDatasets is list of lists, when cloning # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']] # #3743 if not clone: for ds in helper.listOutputDatasets(): if ds not in request['OutputDatasets']: request['OutputDatasets'].append(ds) #TODO: need to update output dataset by Task for task chain requests # can't save Request object directly, because it makes it hard to retrieve the _rev metadata = {} metadata.update(request) # don't want to JSONify the whole workflow del metadata['WorkloadSpec'] workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata) request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl) try: CheckIn.checkIn(request, reqSchema['RequestType']) except CheckIn.RequestCheckInError as ex: raise HTTPError(400, "Error in Request check-in: %s" % str(ex)) # Inconsistent request parameters between Oracle and Couch (#4380, #4388) # metadata above is what is saved into couch to represent a request document. # Number of request arguments on a corresponding couch document # is not set, has default null/None values, update those accordingly now. # It's a mess to have two mutually inconsistent database backends. # Not easy to handle this earlier since couch is stored first and # some parameters are worked out later when storing into Oracle. reqDetails = requestDetails(request["RequestName"]) # couchdb request parameters which are null at the injection time and remain so paramsToUpdate = ["RequestStatus", "RequestSizeFiles", "AcquisitionEra", "RequestWorkflow", "RequestType", "RequestStatus", "RequestPriority", "Requestor", "Group", "SizePerEvent", "PrepID", "RequestNumEvents", "ProcessingString", "ProcessingVersion", ] couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) fields = {} for key in paramsToUpdate: fields[key] = reqDetails[key] couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True) try: wmstatSvc = WMStatsWriter(wmstatUrl) wmstatSvc.insertRequest(request) except Exception as ex: webApi.error("Could not update WMStats, reason: %s" % ex) raise HTTPError(400, "Creating request failed, could not update WMStats.") return request
"Requestor", "Group", "SizePerEvent", "PrepID", "RequestNumEvents", ] couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) fields = {} for key in paramsToUpdate: fields[key] = reqDetails[key] couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields) try: wmstatSvc = WMStatsWriter(wmstatUrl) wmstatSvc.insertRequest(request) except Exception as ex: webApi.error("Could not update WMStats, reason: %s" % ex) raise HTTPError(400, "Creating request failed, could not update WMStats.") return request def makeRequest(webApi, reqInputArgs, couchUrl, couchDB, wmstatUrl): """ Handles the submission of requests. """ # make sure no extra spaces snuck in for k, v in reqInputArgs.iteritems(): if isinstance(v, str):
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False): """ If clone is True, the function is called on a cloned request in which case no modification of the reqSchema shall happen and should be checked in as is. """ try: request = buildWorkloadForRequest(typename=reqSchema["RequestType"], schema=reqSchema) except WMSpecFactoryException as ex: logging.error(traceback.format_exc()) raise HTTPError(400, "Error in Workload Validation: %s" % ex.message()) helper = WMWorkloadHelper(request['WorkloadSpec']) # update request as well for wmstats update # there is a better way to do this (passing helper to request but make sure all the information is there) request["Campaign"] = helper.getCampaign() # Add the output datasets if necessary # for some bizarre reason OutpuDatasets is list of lists, when cloning # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']] # #3743 if not clone: for ds in helper.listOutputDatasets(): if ds not in request['OutputDatasets']: request['OutputDatasets'].append(ds) #TODO: need to update output dataset by Task for task chain requests # can't save Request object directly, because it makes it hard to retrieve the _rev metadata = {} metadata.update(request) # don't want to JSONify the whole workflow del metadata['WorkloadSpec'] workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata) request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl) try: CheckIn.checkIn(request, reqSchema['RequestType']) except CheckIn.RequestCheckInError as ex: raise HTTPError(400, "Error in Request check-in: %s" % str(ex)) # Inconsistent request parameters between Oracle and Couch (#4380, #4388) # metadata above is what is saved into couch to represent a request document. # Number of request arguments on a corresponding couch document # is not set, has default null/None values, update those accordingly now. # It's a mess to have two mutually inconsistent database backends. # Not easy to handle this earlier since couch is stored first and # some parameters are worked out later when storing into Oracle. reqDetails = requestDetails(request["RequestName"]) # couchdb request parameters which are null at the injection time and remain so paramsToUpdate = [ "RequestStatus", "RequestSizeFiles", "AcquisitionEra", "RequestWorkflow", "RequestType", "RequestStatus", "RequestPriority", "Requestor", "Group", "SizePerEvent", "PrepID", "RequestNumEvents", "ProcessingString", "ProcessingVersion", ] couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) fields = {} for key in paramsToUpdate: fields[key] = reqDetails[key] couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True) try: wmstatSvc = WMStatsWriter(wmstatUrl) wmstatSvc.insertRequest(request) except Exception as ex: webApi.error("Could not update WMStats, reason: %s" % ex) raise HTTPError(400, "Creating request failed, could not update WMStats.") return request
class WMStatsTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["WMStats"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) self.testInit.setupCouch('wmstats_t', *self.couchApps) self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t') return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail') self.assertEquals( self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"), 'ERROR: request not found - not_exist_schema') self.assertEquals( self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'), 'ERROR: request not found - not_exist_schema') totalStats = { 'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_file': 5 } self.assertEquals( self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats), 'ERROR: request not found - not_exist_schema') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2), 'ERROR: request not found - not_exist_schema')