class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
self.testInit.setupCouch('wmstats_t', *self.couchApps)
self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t');
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
def processInboundWork(self, inbound_work = None, throw = False):
"""Retrieve work from inbox, split and store
If request passed then only process that request
"""
if self.params['LocalQueueFlag']:
self.backend.fixConflicts() # db should be consistent
result = []
if not inbound_work:
inbound_work = self.backend.getElementsForSplitting()
for inbound in inbound_work:
# Check we haven't already split the work
work = self.backend.getElementsForParent(inbound)
try:
if work:
self.logger.info('Request "%s" already split - Resuming' % inbound['RequestName'])
else:
work, totalStats = self._splitWork(inbound['WMSpec'], None, inbound['Inputs'], inbound['Mask'])
self.backend.insertElements(work, parent = inbound) # if this fails, rerunning will pick up here
# save inbound work to signal we have completed queueing
# add the total work on wmstat summary
self.backend.updateInboxElements(inbound.id, Status = 'Acquired')
if not self.params.get('LocalQueueFlag') and self.params.get('WMStatsCouchUrl'):
# only update global stats for global queue
try:
wmstatSvc = WMStatsWriter(self.params.get('WMStatsCouchUrl'))
wmstatSvc.insertTotalStats(inbound['WMSpec'].name(), totalStats)
except Exception, ex:
self.logger.info('Error publishing %s to WMStats: %s' % (inbound['RequestName'], str(ex)))
except TERMINAL_EXCEPTIONS, ex:
self.logger.info('Failing workflow "%s": %s' % (inbound['RequestName'], str(ex)))
self.backend.updateInboxElements(inbound.id, Status = 'Failed')
if throw:
raise
except Exception, ex:
# if request has been failing for too long permanently fail it.
# last update time was when element was assigned to this queue
if (float(inbound.updatetime) + self.params['QueueRetryTime']) < time.time():
self.logger.info('Failing workflow "%s" as not queued in %d secs: %s' % (inbound['RequestName'],
self.params['QueueRetryTime'],
str(ex)))
self.backend.updateInboxElements(inbound.id, Status = 'Failed')
else:
self.logger.info('Exception splitting work for wmspec "%s": %s' % (inbound['RequestName'], str(ex)))
if throw:
raise
continue
class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
self.testInit.setupCouch('wmstats_t', *self.couchApps)
self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t')
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK',
'insert fail')
self.assertEquals(
self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'],
"failed"), 'OK',
'update fail')
self.assertEquals(
self.wmstatsWriter.updateRequestStatus("not_exist_schema",
"assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(
self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'),
'OK', 'update fail')
self.assertEquals(
self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {
'total_jobs': 100,
'input_events': 1000,
'input_lumis': 1234,
'input_num_file': 5
}
self.assertEquals(
self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'],
totalStats), 'OK',
'update fail')
self.assertEquals(
self.wmstatsWriter.insertTotalStats("not_exist_schema",
totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK',
'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
def processInboundWork(self, inbound_work=None, throw=False):
"""Retrieve work from inbox, split and store
If request passed then only process that request
"""
if self.params['LocalQueueFlag']:
self.backend.fixConflicts() # db should be consistent
result = []
if not inbound_work:
inbound_work = self.backend.getElementsForSplitting()
for inbound in inbound_work:
# Check we haven't already split the work
work = self.backend.getElementsForParent(inbound)
try:
if work:
self.logger.info('Request "%s" already split - Resuming' %
inbound['RequestName'])
else:
work, totalStats = self._splitWork(inbound['WMSpec'], None,
inbound['Inputs'],
inbound['Mask'])
self.backend.insertElements(
work, parent=inbound
) # if this fails, rerunning will pick up here
# save inbound work to signal we have completed queueing
# add the total work on wmstat summary
self.backend.updateInboxElements(inbound.id,
Status='Acquired')
if not self.params.get(
'LocalQueueFlag') and self.params.get(
'WMStatsCouchUrl'):
# only update global stats for global queue
try:
wmstatSvc = WMStatsWriter(
self.params.get('WMStatsCouchUrl'))
wmstatSvc.insertTotalStats(
inbound['WMSpec'].name(), totalStats)
except Exception, ex:
self.logger.info(
'Error publishing %s to WMStats: %s' %
(inbound['RequestName'], str(ex)))
except TERMINAL_EXCEPTIONS, ex:
self.logger.info('Failing workflow "%s": %s' %
(inbound['RequestName'], str(ex)))
self.backend.updateInboxElements(inbound.id, Status='Failed')
if throw:
raise
except Exception, ex:
# if request has been failing for too long permanently fail it.
# last update time was when element was assigned to this queue
if (float(inbound.updatetime) +
self.params['QueueRetryTime']) < time.time():
self.logger.info(
'Failing workflow "%s" as not queued in %d secs: %s' %
(inbound['RequestName'], self.params['QueueRetryTime'],
str(ex)))
self.backend.updateInboxElements(inbound.id,
Status='Failed')
else:
self.logger.info(
'Exception splitting work for wmspec "%s": %s' %
(inbound['RequestName'], str(ex)))
if throw:
raise
continue
