def test_calls_error_callback(self):
"""Ensure run_job catches any exceptions raised by the job, then calls
the error callback.
"""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
call_count = []
handle_count = []
def handle_success():
call_count.append(1)
def handle_errors():
handle_count.append(1)
work = Async(target=dir, args=[1, 2],
callbacks={'success': handle_success,
'error': handle_errors})
with _ExecutionContext(work):
run_job()
self.assertEqual(1, len(handle_count),
"Error handler called wrong number of times.")
self.assertEqual(0, len(call_count),
"Success handler unexpectedly called.")
def test_success_and_combiner_called(self, queue_add_mock,
success_mock,
combiner_mock):
"""Ensure context success and internal vertex combiner
is called when all the context's tasks are processed.
"""
from furious.context import Context
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
from furious.job_utils import encode_callbacks
with Context(callbacks=encode_callbacks(
{'internal_vertex_combiner':
'furious.extras.combiners.lines_combiner',
'success':
'furious.extras.callbacks.small_aggregated'
'_results_success_callback'})) as ctx:
job = ctx.add(pass_args_function, args=[1, 2])
with _ExecutionContext(job):
run_job()
queue_add_mock.assert_called_once()
combiner_mock.assert_called_once()
success_mock.assert_called_once()
def test_context_works(self):
"""Ensure using a _ExecutionContext as a context manager works."""
from furious. async import Async
from furious.context._execution import _ExecutionContext
with _ExecutionContext(Async(target=dir)):
pass
def test_Abort(self, dir_mock, mock_start):
"""Ensures that when Abort is raised, the Async immediately stops."""
import logging
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.errors import Abort
from furious.processors import run_job
class AbortLogHandler(logging.Handler):
def emit(self, record):
if record.levelno >= logging.ERROR:
raise Exception('An Error level log should not be output')
logging.getLogger().addHandler(AbortLogHandler())
dir_mock.side_effect = Abort
mock_success = Mock()
mock_error = Mock()
work = Async(target='dir',
callbacks={'success': mock_success,
'error': mock_error})
with _ExecutionContext(work):
run_job()
self.assertFalse(mock_success.called)
self.assertFalse(mock_error.called)
self.assertFalse(mock_start.called)
logging.getLogger().removeHandler(AbortLogHandler())
def test_context_works(self):
"""Ensure using a _ExecutionContext as a context manager works."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
with _ExecutionContext(Async(target=dir)):
pass
def test_job_removed_from_end_of_local_context(self):
"""Ensure entering the context removes the job from the end of the job
context stack.
"""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.context._local import get_local_context
job_outer = Async(target=dir)
job_inner = Async(target=dir)
with _ExecutionContext(job_outer):
with _ExecutionContext(job_inner):
pass
self.assertEqual(1, len(get_local_context()._executing_async))
self.assertEqual(job_outer,
get_local_context()._executing_async[-1])
def test_job_removed_from_end_of_local_context(self):
"""Ensure entering the context removes the job from the end of the job
context stack.
"""
from furious. async import Async
from furious.context._execution import _ExecutionContext
from furious.context._local import get_local_context
job_outer = Async(target=dir)
job_inner = Async(target=dir)
with _ExecutionContext(job_outer):
with _ExecutionContext(job_inner):
pass
self.assertEqual(1, len(get_local_context()._executing_async))
self.assertEqual(job_outer,
get_local_context()._executing_async[-1])
def test_job_added_to_local_context(self):
"""Ensure entering the context adds the job to the context stack."""
from furious. async import Async
from furious.context._execution import _ExecutionContext
from furious.context._local import get_local_context
job = Async(target=dir)
with _ExecutionContext(job):
self.assertIn(job, get_local_context()._executing_async)
def test_job_added_to_local_context(self):
"""Ensure entering the context adds the job to the context stack."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.context._local import get_local_context
job = Async(target=dir)
with _ExecutionContext(job):
self.assertIn(job, get_local_context()._executing_async)
def test_async_is_preserved(self):
"""Ensure _ExecutionContext exposes the async as a property."""
from furious. async import Async
from furious.context._execution import _ExecutionContext
job = Async(target=dir)
context = _ExecutionContext(job)
self.assertIs(job, context. async)
def test_async_is_preserved(self):
"""Ensure _ExecutionContext exposes the async as a property."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
job = Async(target=dir)
context = _ExecutionContext(job)
self.assertIs(job, context.async)
def test_job_removed_from_local_context(self):
"""Ensure exiting the context removes the job from the context stack.
"""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.context._local import get_local_context
job = Async(target=dir)
with _ExecutionContext(job):
pass
self.assertNotIn(job, get_local_context()._executing_async)
def test_runs_with_none_arg(self, dir_mock):
"""Ensure run_job calls with None arg."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
async = Async("dir", [None])
with _ExecutionContext(async):
run_job()
dir_mock.assert_called_once_with(None)
def test_job_removed_from_local_context(self):
"""Ensure exiting the context removes the job from the context stack.
"""
from furious. async import Async
from furious.context._execution import _ExecutionContext
from furious.context._local import get_local_context
job = Async(target=dir)
with _ExecutionContext(job):
pass
self.assertNotIn(job, get_local_context()._executing_async)
def test_runs_with_non_arg_and_kwarg(self, dir_mock):
"""Ensure run_job calls with a None arg and kwarg=None."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
work = Async("dir", [None], {'something': None})
with _ExecutionContext(work):
run_job()
dir_mock.assert_called_once_with(None, something=None)
def test_runs_with_none_kwarg(self, dir_mock):
"""Ensure run_job calls with a kwarg=None."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
work = Async("dir", kwargs={"something": None})
with _ExecutionContext(work):
run_job()
dir_mock.assert_called_once_with(something=None)
def test_dies_if_no_job(self):
"""Ensure run_job raises if there is no job in the Async."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
work = Async("dir", kwargs={'something': None})
work._options.pop('job')
assert 'job' not in work._options
with _ExecutionContext(work):
self.assertRaises(Exception, run_job)
def test_async_is_not_settable(self):
"""Ensure _ExecutionContext async can not be set."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
job = Async(target=dir)
context = _ExecutionContext(job)
def set_job():
context.async = None
self.assertRaises(AttributeError, set_job)
def test_async_is_not_settable(self):
"""Ensure _ExecutionContext async can not be set."""
from furious. async import Async
from furious.context._execution import _ExecutionContext
job = Async(target=dir)
context = _ExecutionContext(job)
def set_job():
context. async = None
self.assertRaises(AttributeError, set_job)
def test_corrupt_context(self):
"""Ensure wrong context is not popped from execution context stack."""
from furious. async import Async
from furious.context._execution import _ExecutionContext
from furious.context._local import get_local_context
from furious.errors import CorruptContextError
with self.assertRaises(CorruptContextError) as cm:
job_outer = Async(target=dir)
with _ExecutionContext(job_outer):
local_context = get_local_context()
local_context._executing_async.append(object())
self.assertEqual((None, None, None), cm.exception.exc_info)
def test_corrupt_context(self):
"""Ensure wrong context is not popped from execution context stack."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.context._local import get_local_context
from furious.errors import CorruptContextError
with self.assertRaises(CorruptContextError) as cm:
job_outer = Async(target=dir)
with _ExecutionContext(job_outer):
local_context = get_local_context()
local_context._executing_async.append(object())
self.assertEqual((None, None, None), cm.exception.exc_info)
def test_starts_returned_async(self):
"""Ensure run_job calls returned Async objects."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
returned_async = Mock(spec=Async)
work = Async(target=_fake_async_returning_target, args=[returned_async])
with _ExecutionContext(work):
run_job()
returned_async.start.assert_called_once_with()
def test_calls_async_callbacks(self):
"""Ensure run_job will call start on an Async object callback."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
callback = Mock(spec=Async)
work = Async(target=dir, args=[1], callbacks={'success': callback})
with _ExecutionContext(work):
run_job()
callback.start.assert_called_once_with()
def test_handles_job_exception(self):
"""Ensure run_job catches and encodes exceptions to the async result,
then raises them.
"""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
from furious.processors import AsyncException
work = Async(target=dir, args=[1, 2])
with _ExecutionContext(work):
self.assertRaises(TypeError, run_job)
self.assertIsInstance(work.result.payload, AsyncException)
def test_starts_returned_async(self):
"""Ensure run_job calls returned Async objects."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
returned_async = Mock(spec=Async)
work = Async(target=_fake_async_returning_target,
args=[returned_async])
with _ExecutionContext(work):
run_job()
returned_async.start.assert_called_once_with()
def test_calls_success_callback(self):
"""Ensure run_job calls the success callback after a successful run."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
call_count = []
def do_things():
call_count.append(1)
work = Async(target=dir, args=[1], callbacks={"success": do_things})
with _ExecutionContext(work):
run_job()
self.assertEqual(1, len(call_count))
def test_calls_success_callback(self):
"""Ensure run_job calls the success callback after a successful run."""
from furious. async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
call_count = []
def do_things():
call_count.append(1)
work = Async(target=dir, args=[1], callbacks={'success': do_things})
with _ExecutionContext(work):
run_job()
self.assertEqual(1, len(call_count))
def test_internal_vertex_combiner_called(self, queue_add_mock):
"""Call lines_combiner and small_aggregated_results_success_callback
"""
from furious.context import Context
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
from furious.extras.combiners import lines_combiner
from furious.extras.callbacks import small_aggregated_results_success_callback
with Context(callbacks={'internal_vertex_combiner': lines_combiner,
'success': small_aggregated_results_success_callback}) as ctx:
job = ctx.add(pass_args_function, args=[1, 2])
with _ExecutionContext(job):
run_job()
queue_add_mock.assert_called_once()
def test_handle_async_done(self, queue_add_mock, update_done_mock):
"""Ensure handle_async_done is called when Context is given a
callback.
"""
from furious.context import Context
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
update_done_mock.return_value = True
with Context(callbacks={'success': example_callback_success}) as ctx:
job = ctx.add(example_function, args=[1, 2])
with _ExecutionContext(job):
run_job()
update_done_mock.assert_called_once()
queue_add_mock.assert_called_once()
def test_starts_context_result(self):
"""Ensure run_job will call start on an Context object, with
tasks, as a result of the Async target function."""
from furious.async import Async
from furious.context import Context
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
returned_context = Mock(spec=Context, _tasks=[
Async(target=dir, args=[1])])
work = Async(target=_fake_context_returning_target,
args=[returned_context])
with _ExecutionContext(work):
run_job()
returned_context.start.assert_called_once_with()
def test_AbortAndRestart(self, dir_mock):
"""Ensures when AbortAndRestart is raised the Async restarts."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.errors import AbortAndRestart
from furious.processors import run_job
dir_mock.side_effect = AbortAndRestart
mock_success = Mock()
mock_error = Mock()
work = Async(target='dir',
callbacks={'success': mock_success,
'error': mock_error})
with _ExecutionContext(work):
self.assertRaises(AbortAndRestart, run_job)
self.assertFalse(mock_success.called)
self.assertFalse(mock_error.called)
def test_skip_empty_context_result(self, mock_handle_async_done):
"""Ensure run_job will not call start on an Context object, with
no tasks, as a result of the Async target function."""
from furious.async import Async
from furious.context import Context
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
returned_context = Mock(spec=Context, _tasks=[])
work = Async(target=_fake_context_returning_target,
args=[returned_context])
with _ExecutionContext(work):
with patch("furious.marker_tree.async_utils.handle_async_done",
autospec=True) as mock2_handle_async_done:
run_job()
mock2_handle_async_done.assert_called_once_with(work)
self.assertEqual(returned_context._handle_tasks.called, False)
def test_BaseException(self, dir_mock):
"""Ensure exceptions inheriting from BaseException, such as
DeadlineExceededError, trigger the error handler.
"""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
from google.appengine.runtime import DeadlineExceededError
dir_mock.side_effect = DeadlineExceededError
mock_success = Mock()
mock_error = Mock()
work = Async(target='dir',
callbacks={'success': mock_success,
'error': mock_error})
with _ExecutionContext(work):
run_job()
self.assertFalse(mock_success.called)
self.assertEqual(mock_error.call_count, 1)
