def test_module_referenceable(self):
"""
Tests a module referenceable
Verifies:
* mapped methods are callable
* non-mapped methods raise Error
* args and kwargs are passed in
"""
foo = CallProxy(None, False)
bar = CallProxy(None, False)
xoo = CallProxy(None, False)
remotes = {'foo':foo, 'bar':bar, 'xoo':xoo}
referenceable = ModuleReferenceable(remotes)
# verify calls work
referenceable.remote_foo()
foo.assertCalled(self)
bar.assertNotCalled(self)
referenceable.remote_bar()
bar.assertCalled(self)
# verify args work
args = (1, 2, 3)
kwargs = {'a':1, 'b':2}
referenceable.remote_xoo(*args, **kwargs)
xoo.assertCalled(self, *args, **kwargs)
def not_mapped():
referenceable.remote_not_mapped()
# verify non-mapped function raises error
self.assertRaises(KeyError, not_mapped)
class ParallelTaskTwistedTest(twisted_unittest.TestCase):
"""
Test ParllelTask functionality that requires twisted to run
"""
def setUp(self):
self.pt = TestParallelTask()
self.worker = WorkerProxy()
self.pt.parent = self.worker
self.callback = CallProxy(None, False)
def test_request_workers(self):
"""
Tests requesting all work units
Verifies:
* work_request called for every work unit
"""
pt = self.pt
pt.request_workers()
self.assertEqual(pt._workunit_count, 10, "Workunit count is not correct")
self.assertEqual(len(pt._data_in_progress), 10, "in progress count is not correct")
self.assertEqual(len(self.worker.request_worker.calls), 10, "request_worker was not called the correct number of times")
def test_progress_stopped(self):
self.assertEquals(self.pt.status(), STATUS_STOPPED)
def verify_parallel_work(self):
"""
helper function for verifying parllel task. must be called from
deferToThread()
Verifies Start:
* status is marked running
* work_units in progress is correct
* work_unit count is correct
* work_request called for every work unit
Verifies Workunit completion:
* _data_in_progress is removed
* custom subtask completion function is run
* worker is released if no more work is pending
Verifies All workunits complete:
* all workers are released
* custom completion function is run
* task is marked complete
* task returns response
* callback is called
"""
# sleep for 0.1 second to allow start thread to finish requesting workes
time.sleep(.1)
pt = self.pt
self.assertEqual(pt.status(), STATUS_RUNNING)
self.assertEqual(pt._workunit_count, 10, "Workunit count is not correct")
self.assertEqual(len(pt._data_in_progress), 10, "in progress count is not correct")
self.assertEqual(len(self.worker.request_worker.calls), 10, "request_worker was not called the correct number of times")
self.callback.assertNotCalled(self)
for i in range(10):
self.assertEqual(pt.status(), STATUS_RUNNING)
pt._work_unit_complete(i, i)
self.assertEqual(pt._workunit_count, 10, "Workunit count is not correct")
self.assertEqual(len(pt._data_in_progress), 9-i, "in progress count is not correct")
self.assert_(pt.complete, 'task is not marked complete via completion code')
self.assertEquals(pt.status(), STATUS_COMPLETE, "Status is not reporting completed")
self.assertEqual(len(self.worker.request_worker_release.calls), 1, "request_worker_release was not called the correct number of times")
self.callback.assertCalled(self)
def test_parallel_work(self):
"""
Tests requesting all work units
"""
pt = self.pt
pt.start(callback=self.callback)
return threads.deferToThread(self.verify_parallel_work)
def test_worker_failed(self):
"""
Work being returned due to worker failure failure:
Verifies work is removed from in progress
"""
pt = self.pt
pt.request_workers()
for i in range(10):
pt._worker_failed(i)
self.assert_(i not in pt._data_in_progress, 'workunit still in progress')
self.assertEquals(len(pt._data_in_progress), 9-i, 'Data in progress is wrong size')
self.assertEquals(len(pt._data_in_progress), 0, 'Data in progress is wrong size')
def verify_batch_complete(self):
"""
helper function for verifying batch completiion of parallel task
subtask. Must be called from deferToThread()
Verifies Start:
* status is marked running
* work_units in progress is correct
* work_unit count is correct
* work_request called for every work unit
Verifies Workunit completion:
* _data_in_progress is removed
* custom subtask completion function is run
* worker is released if no more work is pending
Verifies All workunits complete:
* all workers are released
* custom completion function is run
* task is marked complete
* task returns response
"""
# sleep for 0.1 second to allow start thread to finish requesting workes
time.sleep(.1)
pt = self.pt
self.assertEqual(pt.status(), STATUS_RUNNING)
self.assertEqual(pt._workunit_count, 10, "Workunit count is not correct")
self.assertEqual(len(pt._data_in_progress), 10, "in progress count is not correct")
self.assertEqual(len(self.worker.request_worker.calls), 10, "request_worker was not called the correct number of times")
for i in range(0,10,2):
self.assertEqual(pt.status(), STATUS_RUNNING)
results = ((i, i, 0), (i+1, i+1, 0))
pt._batch_complete(results)
self.assertEqual(pt._workunit_count, 10, "Workunit count is not correct")
self.assertEqual(len(pt._data_in_progress), 8-i, "in progress count is not correct")
self.assert_(pt.complete, 'task is not marked complete via completion code')
self.assertEquals(pt.status(), STATUS_COMPLETE, "Status is not reporting completed")
self.assertEqual(len(self.worker.request_worker_release.calls), 1, "request_worker_release was not called the correct number of times")
def test_batch_complete(self):
"""
Tests completing batched workunits
"""
pt = self.pt
pt.start(callback=self.callback)
return threads.deferToThread(self.verify_batch_complete)