class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
self.testInit.setupCouch('wmstats_t', *self.couchApps)
self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t');
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError, ex:
msg = ex._message
raise HTTPError(400, "Error in Request check-in: %s" % msg)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400,
"Creating request failed, could not update WMStats.")
return request
def makeRequest(webApi, reqInputArgs, couchUrl, couchDB, wmstatUrl):
"""
Handles the submission of requests.
"""
# make sure no extra spaces snuck in
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False):
"""
If clone is True, the function is called on a cloned request in which
case no modification of the reqSchema shall happen and should be checked in
as is.
"""
try:
request = buildWorkloadForRequest(typename = reqSchema["RequestType"],
schema = reqSchema)
except WMSpecFactoryException as ex:
logging.error(traceback.format_exc())
raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
helper = WMWorkloadHelper(request['WorkloadSpec'])
# update request as well for wmstats update
# there is a better way to do this (passing helper to request but make sure all the information is there)
request["Campaign"] = helper.getCampaign()
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
#TODO: need to update output dataset by Task for task chain requests
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError as ex:
raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
# Inconsistent request parameters between Oracle and Couch (#4380, #4388)
# metadata above is what is saved into couch to represent a request document.
# Number of request arguments on a corresponding couch document
# is not set, has default null/None values, update those accordingly now.
# It's a mess to have two mutually inconsistent database backends.
# Not easy to handle this earlier since couch is stored first and
# some parameters are worked out later when storing into Oracle.
reqDetails = requestDetails(request["RequestName"])
# couchdb request parameters which are null at the injection time and remain so
paramsToUpdate = ["RequestStatus",
"RequestSizeFiles",
"AcquisitionEra",
"RequestWorkflow",
"RequestType",
"RequestStatus",
"RequestPriority",
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
"ProcessingString",
"ProcessingVersion",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400, "Creating request failed, could not update WMStats.")
return request
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400, "Creating request failed, could not update WMStats.")
return request
def makeRequest(webApi, reqInputArgs, couchUrl, couchDB, wmstatUrl):
"""
Handles the submission of requests.
"""
# make sure no extra spaces snuck in
for k, v in reqInputArgs.iteritems():
if isinstance(v, str):
def buildWorkloadAndCheckIn(webApi,
reqSchema,
couchUrl,
couchDB,
wmstatUrl,
clone=False):
"""
If clone is True, the function is called on a cloned request in which
case no modification of the reqSchema shall happen and should be checked in
as is.
"""
try:
request = buildWorkloadForRequest(typename=reqSchema["RequestType"],
schema=reqSchema)
except WMSpecFactoryException as ex:
logging.error(traceback.format_exc())
raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
helper = WMWorkloadHelper(request['WorkloadSpec'])
# update request as well for wmstats update
# there is a better way to do this (passing helper to request but make sure all the information is there)
request["Campaign"] = helper.getCampaign()
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
#TODO: need to update output dataset by Task for task chain requests
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError as ex:
raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
# Inconsistent request parameters between Oracle and Couch (#4380, #4388)
# metadata above is what is saved into couch to represent a request document.
# Number of request arguments on a corresponding couch document
# is not set, has default null/None values, update those accordingly now.
# It's a mess to have two mutually inconsistent database backends.
# Not easy to handle this earlier since couch is stored first and
# some parameters are worked out later when storing into Oracle.
reqDetails = requestDetails(request["RequestName"])
# couchdb request parameters which are null at the injection time and remain so
paramsToUpdate = [
"RequestStatus",
"RequestSizeFiles",
"AcquisitionEra",
"RequestWorkflow",
"RequestType",
"RequestStatus",
"RequestPriority",
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
"ProcessingString",
"ProcessingVersion",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields=fields,
useBody=True)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400,
"Creating request failed, could not update WMStats.")
return request
class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
self.testInit.setupCouch('wmstats_t', *self.couchApps)
self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t')
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK',
'insert fail')
self.assertEquals(
self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'],
"failed"), 'OK',
'update fail')
self.assertEquals(
self.wmstatsWriter.updateRequestStatus("not_exist_schema",
"assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(
self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'),
'OK', 'update fail')
self.assertEquals(
self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {
'total_jobs': 100,
'input_events': 1000,
'input_lumis': 1234,
'input_num_file': 5
}
self.assertEquals(
self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'],
totalStats), 'OK',
'update fail')
self.assertEquals(
self.wmstatsWriter.insertTotalStats("not_exist_schema",
totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK',
'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
