def testWorkQueueService(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file", assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True) self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce': True, 'group': True} # This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{'total_jobs': 339, 'request_name': specName}]) # work still available, so no childQueue results = wqApi.getChildQueuesAndStatus() self.assertItemsEqual(set([item['agent_name'] for item in results]), ["AgentNotDefined"]) result = wqApi.getElementsCountAndJobsByWorkflow() self.assertEqual(len(result), 1) self.assertEqual(result[specName]['Available']['Jobs'], 339) results = wqApi.getChildQueuesAndPriority() resultsPrio = set([item['priority'] for item in results if item['agent_name'] == "AgentNotDefined"]) self.assertItemsEqual(resultsPrio, [8000]) self.assertEqual(wqApi.getWMBSUrl(), []) self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
def testWorkQueueService(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl) self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') #overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce': True, 'group': True} #This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{ 'total_jobs': 10, 'request_name': specName }]) self.assertEqual(wqApi.getChildQueues(), []) self.assertEqual(wqApi.getJobStatusByRequest(), [{ 'status': 'Available', 'jobs': 10, 'request_name': specName }]) self.assertEqual(wqApi.getChildQueuesByRequest(), []) self.assertEqual(wqApi.getWMBSUrl(), []) self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
def testCompletedWorkflow(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True) self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce': True, 'group': True} # This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{'total_jobs': 339, 'request_name': specName}]) results = wqApi.getJobsByStatusAndPriority() self.assertEqual(results.keys(), ['Available']) self.assertEqual(results['Available'].keys(), [8000]) self.assertTrue(results['Available'][8000]['sum'], 339) result = wqApi.getElementsCountAndJobsByWorkflow() self.assertEqual(len(result), 1) self.assertEqual(result[specName]['Available']['Jobs'], 339) data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus', {'startkey': [specName], 'endkey': [specName, {}], 'reduce': False}) elements = [x['id'] for x in data.get('rows', [])] wqApi.updateElements(*elements, Status='Canceled') # load this view once again to make sure it will be updated in the next assert.. data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus', {'startkey': [specName], 'endkey': [specName, {}], 'reduce': False}) self.assertEqual(len(wqApi.getCompletedWorkflow(stale=False)), 1) self.assertEqual(wqApi.getJobsByStatusAndPriority().keys(), ['Canceled'])
def testWorkQueueService(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec( specName, "file", assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True) self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce': True, 'group': True} # This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{ 'total_jobs': 339, 'request_name': specName }]) # work still available, so no childQueue self.assertEqual(wqApi.getChildQueuesAndStatus().keys(), [None]) result = wqApi.getElementsCountAndJobsByWorkflow() self.assertEqual(len(result), 1) self.assertEqual(result[specName]['Available']['Jobs'], 339) self.assertEqual(wqApi.getChildQueuesAndPriority()[None].keys(), [8000]) self.assertEqual(wqApi.getWMBSUrl(), []) self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
def testCompletedWorkflow(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec( specName, "file", assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True, **self.queueParams) self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce': True, 'group': True} # This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{ 'total_jobs': 339, 'request_name': specName }]) results = wqApi.getJobsByStatus() self.assertEqual(results['Available']['sum_jobs'], 339) results = wqApi.getJobsByStatusAndPriority() resultsPrio = set( [item['priority'] for item in results.get('Available')]) self.assertItemsEqual(resultsPrio, [8000]) resultsJobs = sum([ item['sum_jobs'] for item in results.get('Available') if item['priority'] == 8000 ]) self.assertTrue(resultsJobs, 339) result = wqApi.getElementsCountAndJobsByWorkflow() self.assertEqual(len(result), 1) self.assertEqual(result[specName]['Available']['Jobs'], 339) data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus', { 'startkey': [specName], 'endkey': [specName, {}], 'reduce': False }) elements = [x['id'] for x in data.get('rows', [])] wqApi.updateElements(*elements, Status='Canceled') # load this view once again to make sure it will be updated in the next assert.. data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus', { 'startkey': [specName], 'endkey': [specName, {}], 'reduce': False }) self.assertEqual(len(wqApi.getCompletedWorkflow(stale=False)), 1) results = wqApi.getJobsByStatusAndPriority() resultsPrio = set( [item['priority'] for item in results.get('Canceled')]) self.assertItemsEqual(resultsPrio, [8000])
def getRequestInfoFromGlobalQueue(serviceURL): """ get the request info from global queue """ url, dbName = splitCouchServiceURL(serviceURL) service = WorkQueue(url, dbName) try: jobInfo = service.getTopLevelJobsByRequest() qInfo = service.getChildQueuesByRequest() siteWhitelists = service.getSiteWhitelistByRequest() childQueueURLs = set() for item in qInfo: childQueueURLs.add(item['local_queue']) except Exception, ex: logging.error("%s: %s" % (serviceURL, str(ex))) return DFormatter.errorFormatter(serviceURL, "GlobalQueue Down")
def testWorkQueueService(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName = 'workqueue_t', QueueURL = self.testInit.couchUrl) self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') #This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{'total_jobs': 2, 'request_name': specName}]) self.assertEqual(wqApi.getChildQueues(), []) self.assertEqual(wqApi.getJobStatusByRequest(), [{'status': 'Available', 'jobs': 2, 'request_name': specName}]) self.assertEqual(wqApi.getChildQueuesByRequest(), []) self.assertEqual(wqApi.getWMBSUrl(), []) self.assertEqual(wqApi.getWMBSUrlByRequest(), [])