def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec( specName, "file", assignKwargs={'SiteWhitelist': ["T2_XX_SiteA"]}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True, **self.queueParams) localQ = localQueue(DbName='local_workqueue_t', QueueURL=self.testInit.couchUrl, CacheDir=self.testInit.testDir, ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName='workqueue_t_inbox', **self.queueParams) # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce': True, 'group': True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) numWorks = localQ.pullWork({'T2_XX_SiteA': 10}) self.assertTrue(numWorks > 0) # replicate from GQ to LQ manually localQ.backend.pullFromParent(continuous=False) # wait until replication is done time.sleep(2) localQ.processInboundWork(continuous=False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') # overwrite default - can't test with stale view localApi.defaultOptions = {'reduce': True, 'group': True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) availableWF = localApi.getAvailableWorkflows() self.assertEqual(availableWF, set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except Exception as ex: self.fail('No exception should be raised.: %s' % str(ex))
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file", assignKwargs={'SiteWhitelist':["T2_XX_SiteA"]}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True) localQ = localQueue(DbName='local_workqueue_t', QueueURL=self.testInit.couchUrl, CacheDir=self.testInit.testDir, ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName='workqueue_t_inbox' ) # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce': True, 'group': True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) numWorks = localQ.pullWork({'T2_XX_SiteA': 10}) self.assertTrue(numWorks > 0) # replicate from GQ to LQ manually localQ.backend.pullFromParent(continuous=False) # wait until replication is done time.sleep(2) localQ.processInboundWork(continuous=False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') # overwrite default - can't test with stale view localApi.defaultOptions = {'reduce': True, 'group': True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) availableWF = localApi.getAvailableWorkflows() self.assertEqual(availableWF, set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except Exception as ex: self.fail('No exception should be raised.: %s' % str(ex))
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl) localQ = localQueue(DbName='local_workqueue_t', QueueURL=self.testInit.couchUrl, CacheDir=self.testInit.testDir, ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName='workqueue_t_inbox') # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') #overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce': True, 'group': True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) self.assertTrue( localQ.pullWork({'T2_XX_SiteA': 10}, continuousReplication=False) > 0) localQ.processInboundWork(continuous=False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') #overwrite default - can't test with stale view localApi.defaultOptions = {'reduce': True, 'group': True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) self.assertEqual(localApi.getAvailableWorkflows(), set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except: self.fail('No exception should be raised.')
def testWorkQueueService(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl) self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') #overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce': True, 'group': True} #This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{ 'total_jobs': 10, 'request_name': specName }]) self.assertEqual(wqApi.getChildQueues(), []) self.assertEqual(wqApi.getJobStatusByRequest(), [{ 'status': 'Available', 'jobs': 10, 'request_name': specName }]) self.assertEqual(wqApi.getChildQueuesByRequest(), []) self.assertEqual(wqApi.getWMBSUrl(), []) self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
def testWorkQueueService(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file", assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True) self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce': True, 'group': True} # This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{'total_jobs': 339, 'request_name': specName}]) # work still available, so no childQueue results = wqApi.getChildQueuesAndStatus() self.assertItemsEqual(set([item['agent_name'] for item in results]), ["AgentNotDefined"]) result = wqApi.getElementsCountAndJobsByWorkflow() self.assertEqual(len(result), 1) self.assertEqual(result[specName]['Available']['Jobs'], 339) results = wqApi.getChildQueuesAndPriority() resultsPrio = set([item['priority'] for item in results if item['agent_name'] == "AgentNotDefined"]) self.assertItemsEqual(resultsPrio, [8000]) self.assertEqual(wqApi.getWMBSUrl(), []) self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
def testCompletedWorkflow(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True) self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce': True, 'group': True} # This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{'total_jobs': 339, 'request_name': specName}]) results = wqApi.getJobsByStatusAndPriority() self.assertEqual(results.keys(), ['Available']) self.assertEqual(results['Available'].keys(), [8000]) self.assertTrue(results['Available'][8000]['sum'], 339) result = wqApi.getElementsCountAndJobsByWorkflow() self.assertEqual(len(result), 1) self.assertEqual(result[specName]['Available']['Jobs'], 339) data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus', {'startkey': [specName], 'endkey': [specName, {}], 'reduce': False}) elements = [x['id'] for x in data.get('rows', [])] wqApi.updateElements(*elements, Status='Canceled') # load this view once again to make sure it will be updated in the next assert.. data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus', {'startkey': [specName], 'endkey': [specName, {}], 'reduce': False}) self.assertEqual(len(wqApi.getCompletedWorkflow(stale=False)), 1) self.assertEqual(wqApi.getJobsByStatusAndPriority().keys(), ['Canceled'])
def testWorkQueueService(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec( specName, "file", assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True) self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce': True, 'group': True} # This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{ 'total_jobs': 339, 'request_name': specName }]) # work still available, so no childQueue self.assertEqual(wqApi.getChildQueuesAndStatus().keys(), [None]) result = wqApi.getElementsCountAndJobsByWorkflow() self.assertEqual(len(result), 1) self.assertEqual(result[specName]['Available']['Jobs'], 339) self.assertEqual(wqApi.getChildQueuesAndPriority()[None].keys(), [8000]) self.assertEqual(wqApi.getWMBSUrl(), []) self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName = 'workqueue_t', QueueURL = self.testInit.couchUrl) localQ = localQueue(DbName = 'local_workqueue_t', QueueURL = self.testInit.couchUrl, CacheDir = self.testInit.testDir, ParentQueueCouchUrl = '%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName = 'workqueue_t_inbox' ) # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') #overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce' : True, 'group' : True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) self.assertTrue(localQ.pullWork({'T2_XX_SiteA' : 10}) > 0) localQ.processInboundWork(continuous = False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') #overwrite default - can't test with stale view localApi.defaultOptions = {'reduce' : True, 'group' : True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) self.assertEqual(localApi.getAvailableWorkflows(), set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except: self.fail('No exception should be raised.')
def testCompletedWorkflow(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec( specName, "file", assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True, **self.queueParams) self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce': True, 'group': True} # This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{ 'total_jobs': 339, 'request_name': specName }]) results = wqApi.getJobsByStatus() self.assertEqual(results['Available']['sum_jobs'], 339) results = wqApi.getJobsByStatusAndPriority() resultsPrio = set( [item['priority'] for item in results.get('Available')]) self.assertItemsEqual(resultsPrio, [8000]) resultsJobs = sum([ item['sum_jobs'] for item in results.get('Available') if item['priority'] == 8000 ]) self.assertTrue(resultsJobs, 339) result = wqApi.getElementsCountAndJobsByWorkflow() self.assertEqual(len(result), 1) self.assertEqual(result[specName]['Available']['Jobs'], 339) data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus', { 'startkey': [specName], 'endkey': [specName, {}], 'reduce': False }) elements = [x['id'] for x in data.get('rows', [])] wqApi.updateElements(*elements, Status='Canceled') # load this view once again to make sure it will be updated in the next assert.. data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus', { 'startkey': [specName], 'endkey': [specName, {}], 'reduce': False }) self.assertEqual(len(wqApi.getCompletedWorkflow(stale=False)), 1) results = wqApi.getJobsByStatusAndPriority() resultsPrio = set( [item['priority'] for item in results.get('Canceled')]) self.assertItemsEqual(resultsPrio, [8000])
def testWorkQueueService(self): # test getWork specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName = 'workqueue_t', QueueURL = self.testInit.couchUrl) self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') #overwrite default - can't test with stale view wqApi.defaultOptions = {'reduce' : True, 'group' : True} #This only checks minimum client call not exactly correctness of return # values. self.assertEqual(wqApi.getTopLevelJobsByRequest(), [{'total_jobs': 10, 'request_name': specName}]) self.assertEqual(wqApi.getChildQueues(), []) self.assertEqual(wqApi.getJobStatusByRequest(), [{'status': 'Available', 'jobs': 10, 'request_name': specName}]) self.assertEqual(wqApi.getChildQueuesByRequest(), []) self.assertEqual(wqApi.getWMBSUrl(), []) self.assertEqual(wqApi.getWMBSUrlByRequest(), [])