def test_subtask_started(self):
"""
Task receives notice that subtasks have started
Verifies:
* all messages are received by the task
* format is accepted: ((subtask_id, workunit_id)[, ...])
"""
wtc = self.worker_task_controls
batch = (
("foo.bar.fake.subtask", 0),
("foo.bar.fake.subtask", 1),
("foo.bar.fake.subtask", 2),
("foo.bar.fake.subtask", 3),
)
worker_key = "XXX_worker_key_not_needed"
subtask_key = "XXX_need_a_real_subtask"
task = wtc._task_instance
CallProxy.patch(task, "subtask_started")
wtc.subtask_started(batch)
self.assertEqual(task._work_unit_complete.calls, 4)
for pair in zip(batch, task.subtask_started.calls):
result, call = pair
self.assertEqual(result, call)
def test_receive_results(self):
"""
A task receives results from a subtask on another worker
Verifies:
* list of results is iterated
* successful results are passed to task
* failed tasks are not
* expected results format is processed (workunit_id, result, failure)
"""
wtc = self.worker_task_controls
key = self.run_task()
results = ((0, 0, False), (1, 1, True), (2, 2, False), (3, 3, True))
worker_key = "XXX_worker_key_not_needed"
subtask_key = "XXX_need_a_real_subtask"
# get the correct task, this may be a subtask of the root task
task = wtc._task_instance.get_subtask(subtask_key)
CallProxy.patch(task, "_work_unit_complete")
wtc.receive_results("worker_key_not_needed", results, subtask_key)
self.assertEqual(task._work_unit_complete.calls, 2)
for pair in zip(results, task._work_unit_complete.calls):
result, call = pair
self.assertEqual(result, call)
def test_key_exchange(self):
"""
Tests the auth function before the client has paired. This triggers
key exchanging.
Verifies:
* remote command "exchange_keys" is sent
* Avatar response is received and decoded
* response triggers client save_key to be called
* server key is received
* response triggers server save_key to be called
* client key is received
"""
client = RSAClient(self.priv_key, self.pub_key_values)
remote = RemoteProxy()
save_key_server = CallProxy(None, False)
save_key_client = CallProxy(None, False)
client.auth(remote, save_key=save_key_client)
args, kwargs, deferred = remote.assertCalled(self, 'exchange_keys')
avatar = RSAAvatar(self.priv_key, self.pub_key_values, self.pub_key, save_key=save_key_server, key_size=KEY_SIZE)
deferred.callback(avatar.perspective_exchange_keys(*args[1:]))
args, kwargs = save_key_server.assertCalled(self)
key = simplejson.loads(''.join(args[0]))
self.assert_(key==self.pub_key_values, 'keys do not match')
args, kwargs = save_key_client.assertCalled(self)
key = simplejson.loads(''.join(args[0]))
self.assert_(key==self.pub_key_values, 'keys do not match')
def setUp(self):
self.tearDown()
self.scheduler = scheduler.TaskScheduler()
self.scheduler.task_manager = TaskManagerProxy()
self.manager = ModuleManagerProxy()
self.scheduler._register(self.manager)
# intercept _schedule so that its calls can be recorded
CallProxy.patch(self.scheduler, '_schedule')
# hook the threads modules in the scheduling module so we can intercept
# any calls to deferToThread()
self.threads_ = ThreadsProxy(self)
scheduler.threads = self.threads_
def test_receive_results_stopped(self):
"""
A stopped worker receive results
Verifies:
* results are ignored
"""
wtc = self.worker_task_controls
key = self.run_task()
results = ((0, 0, False), (1, 1, True), (2, 2, False), (3, 3, True))
worker_key = "XXX_worker_key_not_needed"
subtask_key = "XXX_need_a_real_subtask"
# get the correct task, this may be a subtask of the root task
task = wtc._task_instance.get_subtask(subtask_key)
CallProxy.patch(task, "_work_unit_complete")
wtc.receive_results("worker_key_not_needed", results, subtask_key)
self.assertEqual(task._work_unit_complete.calls, 0)
def test_return_work(self):
"""
The scheduler returns work to the task because a worker failed
XXX this may not happen anymore since the master queues work requests
Verifies:
* work is returned to Task
"""
wtc = self.worker_task_controls
key = self.run_task()
subtask_key = "XXX_need_a_real_subtask"
workunit_key = "XXX_need_a_real_subtask"
# get the correct task, this may be a subtask of the root task
task = wtc._task_instance.get_subtask(subtask_key).parent
CallProxy.patch(task, "_worker_failed")
wtc.return_work(subtask_key, workunit_key)
self.assertEqual(task._worker_failed.calls, 0)
def test_authenticate(self):
"""
Tests authenticating with the remote server
Verifies:
* remote authentication functions are called
* original function is called
"""
# configure a TwistedWebInterface to proxy directly to by monkey
# patching the methods directly onto the controller. This lets us test
# test interaction with the actual components.
user_id = 'TEST_USER_ID'
interface = TwistedWebInterface(key='./keys')
interface.sessions[user_id] = {'auth':False, 'challenge':None}
auth_proxy = CallProxy(interface.authenticate, user=user_id)
resp_proxy = CallProxy(interface.challenge_response, user=user_id)
self.api.authenticate = auth_proxy
self.api.challenge_response = resp_proxy
response = self.func._authenticate()
auth_proxy.assertCalled(self)
resp_proxy.assertCalled(self)
self.assert_(response)
def setUp(self):
TwistedWebInterfaceTestCaseMixin.setUp(self)
self.return_list = CallProxy(self.return_list)
class FunctionResourceTestCase(unittest.TestCase, TwistedWebInterfaceTestCaseMixin):
def setUp(self):
TwistedWebInterfaceTestCaseMixin.setUp(self)
self.return_list = CallProxy(self.return_list)
def return_list(self, *args, **kwargs):
""" example function that returns a list """
return [1,2,3]
def return_deferred(self):
""" helper function that always returns a deferred. deferred is just a
pass through so a reference to the deferred can be maintained """
return self.deferred
def raise_exception(self):
""" function that always throws an exception, used for testing errors """
raise Exception("I'm failing intentionally")
def test_trivial(self):
"""
Test instantiating FunctionResource class
Verifies:
* instance can be created with or without kwargs
"""
FunctionResource(self.twisted_web_interface, self.return_list)
FunctionResource(self.twisted_web_interface, self.return_list, auth=True, include_user=False)
def test_new_session(self):
"""
Test request from user without session
Verifies:
* session is created
* session structure is correct
"""
api = self.twisted_web_interface
request = HTTPRequestProxy()
resource = FunctionResource(api, self.return_list)
resource.render(request)
self.assert_('SESSION_ID' in api.sessions)
session = api.sessions['SESSION_ID']
self.assert_('expire' in session)
self.assert_('auth' in session)
self.assert_('challenge' in session)
self.assertFalse(session['auth'])
self.assertFalse(session['challenge'])
def test_render_no_auth_required(self):
"""
rendering a resource that does not require authentication
Verifies:
* response returned
"""
api = self.twisted_web_interface
request = HTTPRequestProxy()
resource = FunctionResource(api, self.return_list, auth=False)
response = resource.render(request)
response = simplejson.loads(response)
self.assertEqual([1,2,3], response)
def test_render_with_user(self):
"""
render a resource that requires the session_id as an arg
Verifies:
* response is returned
"""
api = self.twisted_web_interface
request = HTTPRequestProxy()
resource = FunctionResource(api, self.return_list, auth=False, include_user=True)
response = resource.render(request)
response = simplejson.loads(response)
self.assertEqual([1,2,3], response)
self.return_list.assertCalled(self, 'SESSION_ID')
def test_render_unauthorized(self):
"""
render a resource that requires authorization, but user hasn't auth'ed
Verifies:
returns 401
"""
api = self.twisted_web_interface
request = HTTPRequestProxy()
resource = FunctionResource(api, self.return_list, auth=True)
response = resource.render(request)
self.assertEqual(request.response_code, 401)
def test_render_error(self):
"""
renders a resource that throws an error
Verifies:
* returns 500
* response includes exception and stacktrace
"""
api = self.twisted_web_interface
request = HTTPRequestProxy()
resource = FunctionResource(api, self.raise_exception, auth=False)
with MuteStdout():
response = resource.render(request)
self.assertEqual(request.response_code, 500)
self.assert_(isinstance(response,(str,)))
response = simplejson.loads(response)
self.assert_(isinstance(response, (dict,)), response)
self.assert_('exception' in response, response)
self.assert_('traceback' in response, response)
self.assert_(isinstance(response['exception'], (str,)), response)
self.assert_(isinstance(response['traceback'], (str,)), response)
def test_render_with_args(self):
"""
Renders a resource that requires args
"""
api = self.twisted_web_interface
args = [1,2,3]
kwargs = {'a':1, 'b':2, 'c':3}
request = HTTPRequestProxy(args={'args':[simplejson.dumps(args)], \
'kwargs':[simplejson.dumps(kwargs)]})
resource = FunctionResource(api, self.return_list, auth=False)
response = resource.render(request)
response = simplejson.loads(response)
self.assertEqual([1,2,3], response)
self.return_list.assertCalled(self, *args, **kwargs)
def test_render_deferred(self):
"""
Renders a resource that returns a deferred
Verifies:
* server NOT_DONE_YET returned
* deferred callback writes to response
"""
self.deferred = Deferred()
api = self.twisted_web_interface
request = HTTPRequestProxy()
resource = FunctionResource(api, self.return_deferred, auth=False)
response = resource.render(request)
self.assertEqual(server.NOT_DONE_YET, response)
self.deferred.callback([1,2,3])
request.finish.assertCalled(self)
self.assertEqual([1,2,3], simplejson.loads(request.writer.getvalue()))
def test_module_referenceable(self):
"""
Tests a module referenceable
Verifies:
* mapped methods are callable
* non-mapped methods raise Error
* args and kwargs are passed in
"""
foo = CallProxy(None, False)
bar = CallProxy(None, False)
xoo = CallProxy(None, False)
remotes = {'foo':foo, 'bar':bar, 'xoo':xoo}
referenceable = ModuleReferenceable(remotes)
# verify calls work
referenceable.remote_foo()
foo.assertCalled(self)
bar.assertNotCalled(self)
referenceable.remote_bar()
bar.assertCalled(self)
# verify args work
args = (1, 2, 3)
kwargs = {'a':1, 'b':2}
referenceable.remote_xoo(*args, **kwargs)
xoo.assertCalled(self, *args, **kwargs)
def not_mapped():
referenceable.remote_not_mapped()
# verify non-mapped function raises error
self.assertRaises(KeyError, not_mapped)
def setUp(self):
self.pt = TestParallelTask()
self.worker = WorkerProxy()
self.pt.parent = self.worker
self.callback = CallProxy(None, False)
class ParallelTaskTwistedTest(twisted_unittest.TestCase):
"""
Test ParllelTask functionality that requires twisted to run
"""
def setUp(self):
self.pt = TestParallelTask()
self.worker = WorkerProxy()
self.pt.parent = self.worker
self.callback = CallProxy(None, False)
def test_request_workers(self):
"""
Tests requesting all work units
Verifies:
* work_request called for every work unit
"""
pt = self.pt
pt.request_workers()
self.assertEqual(pt._workunit_count, 10, "Workunit count is not correct")
self.assertEqual(len(pt._data_in_progress), 10, "in progress count is not correct")
self.assertEqual(len(self.worker.request_worker.calls), 10, "request_worker was not called the correct number of times")
def test_progress_stopped(self):
self.assertEquals(self.pt.status(), STATUS_STOPPED)
def verify_parallel_work(self):
"""
helper function for verifying parllel task. must be called from
deferToThread()
Verifies Start:
* status is marked running
* work_units in progress is correct
* work_unit count is correct
* work_request called for every work unit
Verifies Workunit completion:
* _data_in_progress is removed
* custom subtask completion function is run
* worker is released if no more work is pending
Verifies All workunits complete:
* all workers are released
* custom completion function is run
* task is marked complete
* task returns response
* callback is called
"""
# sleep for 0.1 second to allow start thread to finish requesting workes
time.sleep(.1)
pt = self.pt
self.assertEqual(pt.status(), STATUS_RUNNING)
self.assertEqual(pt._workunit_count, 10, "Workunit count is not correct")
self.assertEqual(len(pt._data_in_progress), 10, "in progress count is not correct")
self.assertEqual(len(self.worker.request_worker.calls), 10, "request_worker was not called the correct number of times")
self.callback.assertNotCalled(self)
for i in range(10):
self.assertEqual(pt.status(), STATUS_RUNNING)
pt._work_unit_complete(i, i)
self.assertEqual(pt._workunit_count, 10, "Workunit count is not correct")
self.assertEqual(len(pt._data_in_progress), 9-i, "in progress count is not correct")
self.assert_(pt.complete, 'task is not marked complete via completion code')
self.assertEquals(pt.status(), STATUS_COMPLETE, "Status is not reporting completed")
self.assertEqual(len(self.worker.request_worker_release.calls), 1, "request_worker_release was not called the correct number of times")
self.callback.assertCalled(self)
def test_parallel_work(self):
"""
Tests requesting all work units
"""
pt = self.pt
pt.start(callback=self.callback)
return threads.deferToThread(self.verify_parallel_work)
def test_worker_failed(self):
"""
Work being returned due to worker failure failure:
Verifies work is removed from in progress
"""
pt = self.pt
pt.request_workers()
for i in range(10):
pt._worker_failed(i)
self.assert_(i not in pt._data_in_progress, 'workunit still in progress')
self.assertEquals(len(pt._data_in_progress), 9-i, 'Data in progress is wrong size')
self.assertEquals(len(pt._data_in_progress), 0, 'Data in progress is wrong size')
def verify_batch_complete(self):
"""
helper function for verifying batch completiion of parallel task
subtask. Must be called from deferToThread()
Verifies Start:
* status is marked running
* work_units in progress is correct
* work_unit count is correct
* work_request called for every work unit
Verifies Workunit completion:
* _data_in_progress is removed
* custom subtask completion function is run
* worker is released if no more work is pending
Verifies All workunits complete:
* all workers are released
* custom completion function is run
* task is marked complete
* task returns response
"""
# sleep for 0.1 second to allow start thread to finish requesting workes
time.sleep(.1)
pt = self.pt
self.assertEqual(pt.status(), STATUS_RUNNING)
self.assertEqual(pt._workunit_count, 10, "Workunit count is not correct")
self.assertEqual(len(pt._data_in_progress), 10, "in progress count is not correct")
self.assertEqual(len(self.worker.request_worker.calls), 10, "request_worker was not called the correct number of times")
for i in range(0,10,2):
self.assertEqual(pt.status(), STATUS_RUNNING)
results = ((i, i, 0), (i+1, i+1, 0))
pt._batch_complete(results)
self.assertEqual(pt._workunit_count, 10, "Workunit count is not correct")
self.assertEqual(len(pt._data_in_progress), 8-i, "in progress count is not correct")
self.assert_(pt.complete, 'task is not marked complete via completion code')
self.assertEquals(pt.status(), STATUS_COMPLETE, "Status is not reporting completed")
self.assertEqual(len(self.worker.request_worker_release.calls), 1, "request_worker_release was not called the correct number of times")
def test_batch_complete(self):
"""
Tests completing batched workunits
"""
pt = self.pt
pt.start(callback=self.callback)
return threads.deferToThread(self.verify_batch_complete)
