class WMStatsTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["WMStats"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = self.schema, useDefault = False) self.testInit.setupCouch('wmstats_t', *self.couchApps) self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t'); return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail'); self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"), 'ERROR: request not found - not_exist_schema') self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'), 'ERROR: request not found - not_exist_schema') totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5} self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats), 'ERROR: request not found - not_exist_schema') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2), 'ERROR: request not found - not_exist_schema')
def processInboundWork(self, inbound_work = None, throw = False): """Retrieve work from inbox, split and store If request passed then only process that request """ if self.params['LocalQueueFlag']: self.backend.fixConflicts() # db should be consistent result = [] if not inbound_work: inbound_work = self.backend.getElementsForSplitting() for inbound in inbound_work: # Check we haven't already split the work work = self.backend.getElementsForParent(inbound) try: if work: self.logger.info('Request "%s" already split - Resuming' % inbound['RequestName']) else: work, totalStats = self._splitWork(inbound['WMSpec'], None, inbound['Inputs'], inbound['Mask']) self.backend.insertElements(work, parent = inbound) # if this fails, rerunning will pick up here # save inbound work to signal we have completed queueing # add the total work on wmstat summary self.backend.updateInboxElements(inbound.id, Status = 'Acquired') if not self.params.get('LocalQueueFlag') and self.params.get('WMStatsCouchUrl'): # only update global stats for global queue try: wmstatSvc = WMStatsWriter(self.params.get('WMStatsCouchUrl')) wmstatSvc.insertTotalStats(inbound['WMSpec'].name(), totalStats) except Exception, ex: self.logger.info('Error publishing %s to WMStats: %s' % (inbound['RequestName'], str(ex))) except TERMINAL_EXCEPTIONS, ex: self.logger.info('Failing workflow "%s": %s' % (inbound['RequestName'], str(ex))) self.backend.updateInboxElements(inbound.id, Status = 'Failed') if throw: raise except Exception, ex: # if request has been failing for too long permanently fail it. # last update time was when element was assigned to this queue if (float(inbound.updatetime) + self.params['QueueRetryTime']) < time.time(): self.logger.info('Failing workflow "%s" as not queued in %d secs: %s' % (inbound['RequestName'], self.params['QueueRetryTime'], str(ex))) self.backend.updateInboxElements(inbound.id, Status = 'Failed') else: self.logger.info('Exception splitting work for wmspec "%s": %s' % (inbound['RequestName'], str(ex))) if throw: raise continue
class WMStatsTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["WMStats"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) self.testInit.setupCouch('wmstats_t', *self.couchApps) self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, 'wmstats_t') return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail') self.assertEquals( self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"), 'ERROR: request not found - not_exist_schema') self.assertEquals( self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'), 'ERROR: request not found - not_exist_schema') totalStats = { 'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_file': 5 } self.assertEquals( self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats), 'ERROR: request not found - not_exist_schema') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2), 'ERROR: request not found - not_exist_schema')
def processInboundWork(self, inbound_work=None, throw=False): """Retrieve work from inbox, split and store If request passed then only process that request """ if self.params['LocalQueueFlag']: self.backend.fixConflicts() # db should be consistent result = [] if not inbound_work: inbound_work = self.backend.getElementsForSplitting() for inbound in inbound_work: # Check we haven't already split the work work = self.backend.getElementsForParent(inbound) try: if work: self.logger.info('Request "%s" already split - Resuming' % inbound['RequestName']) else: work, totalStats = self._splitWork(inbound['WMSpec'], None, inbound['Inputs'], inbound['Mask']) self.backend.insertElements( work, parent=inbound ) # if this fails, rerunning will pick up here # save inbound work to signal we have completed queueing # add the total work on wmstat summary self.backend.updateInboxElements(inbound.id, Status='Acquired') if not self.params.get( 'LocalQueueFlag') and self.params.get( 'WMStatsCouchUrl'): # only update global stats for global queue try: wmstatSvc = WMStatsWriter( self.params.get('WMStatsCouchUrl')) wmstatSvc.insertTotalStats( inbound['WMSpec'].name(), totalStats) except Exception, ex: self.logger.info( 'Error publishing %s to WMStats: %s' % (inbound['RequestName'], str(ex))) except TERMINAL_EXCEPTIONS, ex: self.logger.info('Failing workflow "%s": %s' % (inbound['RequestName'], str(ex))) self.backend.updateInboxElements(inbound.id, Status='Failed') if throw: raise except Exception, ex: # if request has been failing for too long permanently fail it. # last update time was when element was assigned to this queue if (float(inbound.updatetime) + self.params['QueueRetryTime']) < time.time(): self.logger.info( 'Failing workflow "%s" as not queued in %d secs: %s' % (inbound['RequestName'], self.params['QueueRetryTime'], str(ex))) self.backend.updateInboxElements(inbound.id, Status='Failed') else: self.logger.info( 'Exception splitting work for wmspec "%s": %s' % (inbound['RequestName'], str(ex))) if throw: raise continue