def testUpdatePriorityService(self):
"""
_testUpdatePriorityService_
Check that we can update the priority correctly also
check the available workflows feature
"""
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(
specName, "file", assignKwargs={'SiteWhitelist': ["T2_XX_SiteA"]})
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl,
UnittestFlag=True,
**self.queueParams)
localQ = localQueue(DbName='local_workqueue_t',
QueueURL=self.testInit.couchUrl,
CacheDir=self.testInit.testDir,
ParentQueueCouchUrl='%s/workqueue_t' %
self.testInit.couchUrl,
ParentQueueInboxCouchDBName='workqueue_t_inbox',
**self.queueParams)
# Try a full chain of priority update and propagation
self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
# overwrite default - can't test with stale view
globalApi.defaultOptions = {'reduce': True, 'group': True}
globalApi.updatePriority(specName, 100)
self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
storedElements = globalQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
numWorks = localQ.pullWork({'T2_XX_SiteA': 10})
self.assertTrue(numWorks > 0)
# replicate from GQ to LQ manually
localQ.backend.pullFromParent(continuous=False)
# wait until replication is done
time.sleep(2)
localQ.processInboundWork(continuous=False)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
# overwrite default - can't test with stale view
localApi.defaultOptions = {'reduce': True, 'group': True}
localApi.updatePriority(specName, 500)
self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 500)
availableWF = localApi.getAvailableWorkflows()
self.assertEqual(availableWF, set([(specName, 500)]))
# Attempt to update an inexistent workflow in the queue
try:
globalApi.updatePriority('NotExistent', 2)
except Exception as ex:
self.fail('No exception should be raised.: %s' % str(ex))
def testUpdatePriorityService(self):
"""
_testUpdatePriorityService_
Check that we can update the priority correctly also
check the available workflows feature
"""
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(specName, "file",
assignKwargs={'SiteWhitelist':["T2_XX_SiteA"]})
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl,
UnittestFlag=True)
localQ = localQueue(DbName='local_workqueue_t',
QueueURL=self.testInit.couchUrl,
CacheDir=self.testInit.testDir,
ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl,
ParentQueueInboxCouchDBName='workqueue_t_inbox'
)
# Try a full chain of priority update and propagation
self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
# overwrite default - can't test with stale view
globalApi.defaultOptions = {'reduce': True, 'group': True}
globalApi.updatePriority(specName, 100)
self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
storedElements = globalQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
numWorks = localQ.pullWork({'T2_XX_SiteA': 10})
self.assertTrue(numWorks > 0)
# replicate from GQ to LQ manually
localQ.backend.pullFromParent(continuous=False)
# wait until replication is done
time.sleep(2)
localQ.processInboundWork(continuous=False)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
# overwrite default - can't test with stale view
localApi.defaultOptions = {'reduce': True, 'group': True}
localApi.updatePriority(specName, 500)
self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 500)
availableWF = localApi.getAvailableWorkflows()
self.assertEqual(availableWF, set([(specName, 500)]))
# Attempt to update an inexistent workflow in the queue
try:
globalApi.updatePriority('NotExistent', 2)
except Exception as ex:
self.fail('No exception should be raised.: %s' % str(ex))
def testUpdatePriorityService(self):
"""
_testUpdatePriorityService_
Check that we can update the priority correctly also
check the available workflows feature
"""
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(specName, "file")
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl)
localQ = localQueue(DbName='local_workqueue_t',
QueueURL=self.testInit.couchUrl,
CacheDir=self.testInit.testDir,
ParentQueueCouchUrl='%s/workqueue_t' %
self.testInit.couchUrl,
ParentQueueInboxCouchDBName='workqueue_t_inbox')
# Try a full chain of priority update and propagation
self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
#overwrite default - can't test with stale view
globalApi.defaultOptions = {'reduce': True, 'group': True}
globalApi.updatePriority(specName, 100)
self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
storedElements = globalQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
self.assertTrue(
localQ.pullWork({'T2_XX_SiteA': 10}, continuousReplication=False) >
0)
localQ.processInboundWork(continuous=False)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
#overwrite default - can't test with stale view
localApi.defaultOptions = {'reduce': True, 'group': True}
localApi.updatePriority(specName, 500)
self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 500)
self.assertEqual(localApi.getAvailableWorkflows(),
set([(specName, 500)]))
# Attempt to update an inexistent workflow in the queue
try:
globalApi.updatePriority('NotExistent', 2)
except:
self.fail('No exception should be raised.')
def testWorkQueueService(self):
# test getWork
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(specName, "file")
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl)
self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
#overwrite default - can't test with stale view
wqApi.defaultOptions = {'reduce': True, 'group': True}
#This only checks minimum client call not exactly correctness of return
# values.
self.assertEqual(wqApi.getTopLevelJobsByRequest(),
[{
'total_jobs': 10,
'request_name': specName
}])
self.assertEqual(wqApi.getChildQueues(), [])
self.assertEqual(wqApi.getJobStatusByRequest(),
[{
'status': 'Available',
'jobs': 10,
'request_name': specName
}])
self.assertEqual(wqApi.getChildQueuesByRequest(), [])
self.assertEqual(wqApi.getWMBSUrl(), [])
self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
def testWorkQueueService(self):
# test getWork
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(specName, "file",
assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']})
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl,
UnittestFlag=True)
self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)
wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
# overwrite default - can't test with stale view
wqApi.defaultOptions = {'reduce': True, 'group': True}
# This only checks minimum client call not exactly correctness of return
# values.
self.assertEqual(wqApi.getTopLevelJobsByRequest(),
[{'total_jobs': 339, 'request_name': specName}])
# work still available, so no childQueue
results = wqApi.getChildQueuesAndStatus()
self.assertItemsEqual(set([item['agent_name'] for item in results]), ["AgentNotDefined"])
result = wqApi.getElementsCountAndJobsByWorkflow()
self.assertEqual(len(result), 1)
self.assertEqual(result[specName]['Available']['Jobs'], 339)
results = wqApi.getChildQueuesAndPriority()
resultsPrio = set([item['priority'] for item in results if item['agent_name'] == "AgentNotDefined"])
self.assertItemsEqual(resultsPrio, [8000])
self.assertEqual(wqApi.getWMBSUrl(), [])
self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
def testCompletedWorkflow(self):
# test getWork
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(specName, "file")
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl,
UnittestFlag=True)
self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)
wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
# overwrite default - can't test with stale view
wqApi.defaultOptions = {'reduce': True, 'group': True}
# This only checks minimum client call not exactly correctness of return
# values.
self.assertEqual(wqApi.getTopLevelJobsByRequest(),
[{'total_jobs': 339, 'request_name': specName}])
results = wqApi.getJobsByStatusAndPriority()
self.assertEqual(results.keys(), ['Available'])
self.assertEqual(results['Available'].keys(), [8000])
self.assertTrue(results['Available'][8000]['sum'], 339)
result = wqApi.getElementsCountAndJobsByWorkflow()
self.assertEqual(len(result), 1)
self.assertEqual(result[specName]['Available']['Jobs'], 339)
data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus',
{'startkey': [specName], 'endkey': [specName, {}],
'reduce': False})
elements = [x['id'] for x in data.get('rows', [])]
wqApi.updateElements(*elements, Status='Canceled')
# load this view once again to make sure it will be updated in the next assert..
data = wqApi.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus',
{'startkey': [specName], 'endkey': [specName, {}],
'reduce': False})
self.assertEqual(len(wqApi.getCompletedWorkflow(stale=False)), 1)
self.assertEqual(wqApi.getJobsByStatusAndPriority().keys(), ['Canceled'])
def testWorkQueueService(self):
# test getWork
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(
specName, "file", assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']})
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl,
UnittestFlag=True)
self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)
wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
# overwrite default - can't test with stale view
wqApi.defaultOptions = {'reduce': True, 'group': True}
# This only checks minimum client call not exactly correctness of return
# values.
self.assertEqual(wqApi.getTopLevelJobsByRequest(),
[{
'total_jobs': 339,
'request_name': specName
}])
# work still available, so no childQueue
self.assertEqual(wqApi.getChildQueuesAndStatus().keys(), [None])
result = wqApi.getElementsCountAndJobsByWorkflow()
self.assertEqual(len(result), 1)
self.assertEqual(result[specName]['Available']['Jobs'], 339)
self.assertEqual(wqApi.getChildQueuesAndPriority()[None].keys(),
[8000])
self.assertEqual(wqApi.getWMBSUrl(), [])
self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
def testUpdatePriorityService(self):
"""
_testUpdatePriorityService_
Check that we can update the priority correctly also
check the available workflows feature
"""
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(specName, "file")
globalQ = globalQueue(DbName = 'workqueue_t',
QueueURL = self.testInit.couchUrl)
localQ = localQueue(DbName = 'local_workqueue_t',
QueueURL = self.testInit.couchUrl,
CacheDir = self.testInit.testDir,
ParentQueueCouchUrl = '%s/workqueue_t' % self.testInit.couchUrl,
ParentQueueInboxCouchDBName = 'workqueue_t_inbox'
)
# Try a full chain of priority update and propagation
self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
#overwrite default - can't test with stale view
globalApi.defaultOptions = {'reduce' : True, 'group' : True}
globalApi.updatePriority(specName, 100)
self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100)
storedElements = globalQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
self.assertTrue(localQ.pullWork({'T2_XX_SiteA' : 10}) > 0)
localQ.processInboundWork(continuous = False)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 100)
localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t')
#overwrite default - can't test with stale view
localApi.defaultOptions = {'reduce' : True, 'group' : True}
localApi.updatePriority(specName, 500)
self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500)
storedElements = localQ.backend.getElementsForWorkflow(specName)
for element in storedElements:
self.assertEqual(element['Priority'], 500)
self.assertEqual(localApi.getAvailableWorkflows(), set([(specName, 500)]))
# Attempt to update an inexistent workflow in the queue
try:
globalApi.updatePriority('NotExistent', 2)
except:
self.fail('No exception should be raised.')
def testCompletedWorkflow(self):
# test getWork
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(
specName, "file", assignKwargs={'SiteWhitelist': ['T2_XX_SiteA']})
globalQ = globalQueue(DbName='workqueue_t',
QueueURL=self.testInit.couchUrl,
UnittestFlag=True,
**self.queueParams)
self.assertTrue(globalQ.queueWork(specUrl, specName, "teamA") > 0)
wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
# overwrite default - can't test with stale view
wqApi.defaultOptions = {'reduce': True, 'group': True}
# This only checks minimum client call not exactly correctness of return
# values.
self.assertEqual(wqApi.getTopLevelJobsByRequest(),
[{
'total_jobs': 339,
'request_name': specName
}])
results = wqApi.getJobsByStatus()
self.assertEqual(results['Available']['sum_jobs'], 339)
results = wqApi.getJobsByStatusAndPriority()
resultsPrio = set(
[item['priority'] for item in results.get('Available')])
self.assertItemsEqual(resultsPrio, [8000])
resultsJobs = sum([
item['sum_jobs'] for item in results.get('Available')
if item['priority'] == 8000
])
self.assertTrue(resultsJobs, 339)
result = wqApi.getElementsCountAndJobsByWorkflow()
self.assertEqual(len(result), 1)
self.assertEqual(result[specName]['Available']['Jobs'], 339)
data = wqApi.db.loadView('WorkQueue',
'elementsDetailByWorkflowAndStatus', {
'startkey': [specName],
'endkey': [specName, {}],
'reduce': False
})
elements = [x['id'] for x in data.get('rows', [])]
wqApi.updateElements(*elements, Status='Canceled')
# load this view once again to make sure it will be updated in the next assert..
data = wqApi.db.loadView('WorkQueue',
'elementsDetailByWorkflowAndStatus', {
'startkey': [specName],
'endkey': [specName, {}],
'reduce': False
})
self.assertEqual(len(wqApi.getCompletedWorkflow(stale=False)), 1)
results = wqApi.getJobsByStatusAndPriority()
resultsPrio = set(
[item['priority'] for item in results.get('Canceled')])
self.assertItemsEqual(resultsPrio, [8000])
def testWorkQueueService(self):
# test getWork
specName = "RerecoSpec"
specUrl = self.specGenerator.createReRecoSpec(specName, "file")
globalQ = globalQueue(DbName = 'workqueue_t',
QueueURL = self.testInit.couchUrl)
self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0)
wqApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t')
#overwrite default - can't test with stale view
wqApi.defaultOptions = {'reduce' : True, 'group' : True}
#This only checks minimum client call not exactly correctness of return
# values.
self.assertEqual(wqApi.getTopLevelJobsByRequest(),
[{'total_jobs': 10, 'request_name': specName}])
self.assertEqual(wqApi.getChildQueues(), [])
self.assertEqual(wqApi.getJobStatusByRequest(),
[{'status': 'Available', 'jobs': 10, 'request_name': specName}])
self.assertEqual(wqApi.getChildQueuesByRequest(), [])
self.assertEqual(wqApi.getWMBSUrl(), [])
self.assertEqual(wqApi.getWMBSUrlByRequest(), [])
