def test_Abort(self, dir_mock, mock_start):
"""Ensures that when Abort is raised, the Async immediately stops."""
import logging
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.errors import Abort
from furious.processors import run_job
class AbortLogHandler(logging.Handler):
def emit(self, record):
if record.levelno >= logging.ERROR:
raise Exception('An Error level log should not be output')
logging.getLogger().addHandler(AbortLogHandler())
dir_mock.side_effect = Abort
mock_success = Mock()
mock_error = Mock()
work = Async(target='dir',
callbacks={'success': mock_success,
'error': mock_error})
with _ExecutionContext(work):
run_job()
self.assertFalse(mock_success.called)
self.assertFalse(mock_error.called)
self.assertFalse(mock_start.called)
logging.getLogger().removeHandler(AbortLogHandler())
def test_calls_error_callback(self):
"""Ensure run_job catches any exceptions raised by the job, then calls
the error callback.
"""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
call_count = []
handle_count = []
def handle_success():
call_count.append(1)
def handle_errors():
handle_count.append(1)
work = Async(target=dir, args=[1, 2],
callbacks={'success': handle_success,
'error': handle_errors})
with _ExecutionContext(work):
run_job()
self.assertEqual(1, len(handle_count),
"Error handler called wrong number of times.")
self.assertEqual(0, len(call_count),
"Success handler unexpectedly called.")
def test_success_and_combiner_called(self, queue_add_mock,
success_mock,
combiner_mock):
"""Ensure context success and internal vertex combiner
is called when all the context's tasks are processed.
"""
from furious.context import Context
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
from furious.job_utils import encode_callbacks
with Context(callbacks=encode_callbacks(
{'internal_vertex_combiner':
'furious.extras.combiners.lines_combiner',
'success':
'furious.extras.callbacks.small_aggregated'
'_results_success_callback'})) as ctx:
job = ctx.add(pass_args_function, args=[1, 2])
with _ExecutionContext(job):
run_job()
queue_add_mock.assert_called_once()
combiner_mock.assert_called_once()
success_mock.assert_called_once()
def test_runs_with_none_arg(self, dir_mock):
"""Ensure run_job calls with None arg."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
async = Async("dir", [None])
with _ExecutionContext(async):
run_job()
def process_async_task(headers, request_body):
"""Process an Async task and execute the requested function."""
async_options = json.loads(request_body)
async = async_from_options(async_options)
_log_task_info(headers)
logging.info(async ._function_path)
with context.execution_context_from_async(async):
run_job()
def process_async_task(headers, request_body):
"""Process an Async task and execute the requested function."""
async_options = json.loads(request_body)
async = async_from_options(async_options)
_log_task_info(headers)
logging.info(async._function_path)
with context.execution_context_from_async(async):
run_job()
return 200, async._function_path
def test_runs_with_none_kwarg(self, dir_mock):
"""Ensure run_job calls with a kwarg=None."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
work = Async("dir", kwargs={"something": None})
with _ExecutionContext(work):
run_job()
dir_mock.assert_called_once_with(something=None)
def test_runs_with_non_arg_and_kwarg(self, dir_mock):
"""Ensure run_job calls with a None arg and kwarg=None."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
work = Async("dir", [None], {'something': None})
with _ExecutionContext(work):
run_job()
dir_mock.assert_called_once_with(None, something=None)
def test_runs_with_none_arg(self, dir_mock):
"""Ensure run_job calls with None arg."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
async = Async("dir", [None])
with _ExecutionContext(async):
run_job()
dir_mock.assert_called_once_with(None)
def test_starts_returned_async(self):
"""Ensure run_job calls returned Async objects."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
returned_async = Mock(spec=Async)
work = Async(target=_fake_async_returning_target, args=[returned_async])
with _ExecutionContext(work):
run_job()
returned_async.start.assert_called_once_with()
def test_calls_async_callbacks(self):
"""Ensure run_job will call start on an Async object callback."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
callback = Mock(spec=Async)
work = Async(target=dir, args=[1], callbacks={'success': callback})
with _ExecutionContext(work):
run_job()
callback.start.assert_called_once_with()
def test_starts_returned_async(self):
"""Ensure run_job calls returned Async objects."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
returned_async = Mock(spec=Async)
work = Async(target=_fake_async_returning_target,
args=[returned_async])
with _ExecutionContext(work):
run_job()
returned_async.start.assert_called_once_with()
def test_calls_success_callback(self):
"""Ensure run_job calls the success callback after a successful run."""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
call_count = []
def do_things():
call_count.append(1)
work = Async(target=dir, args=[1], callbacks={"success": do_things})
with _ExecutionContext(work):
run_job()
self.assertEqual(1, len(call_count))
def test_calls_success_callback(self):
"""Ensure run_job calls the success callback after a successful run."""
from furious. async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
call_count = []
def do_things():
call_count.append(1)
work = Async(target=dir, args=[1], callbacks={'success': do_things})
with _ExecutionContext(work):
run_job()
self.assertEqual(1, len(call_count))
def test_internal_vertex_combiner_called(self, queue_add_mock):
"""Call lines_combiner and small_aggregated_results_success_callback
"""
from furious.context import Context
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
from furious.extras.combiners import lines_combiner
from furious.extras.callbacks import small_aggregated_results_success_callback
with Context(callbacks={'internal_vertex_combiner': lines_combiner,
'success': small_aggregated_results_success_callback}) as ctx:
job = ctx.add(pass_args_function, args=[1, 2])
with _ExecutionContext(job):
run_job()
queue_add_mock.assert_called_once()
def test_handle_async_done(self, queue_add_mock, update_done_mock):
"""Ensure handle_async_done is called when Context is given a
callback.
"""
from furious.context import Context
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
update_done_mock.return_value = True
with Context(callbacks={'success': example_callback_success}) as ctx:
job = ctx.add(example_function, args=[1, 2])
with _ExecutionContext(job):
run_job()
update_done_mock.assert_called_once()
queue_add_mock.assert_called_once()
def test_starts_context_result(self):
"""Ensure run_job will call start on an Context object, with
tasks, as a result of the Async target function."""
from furious.async import Async
from furious.context import Context
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
returned_context = Mock(spec=Context, _tasks=[
Async(target=dir, args=[1])])
work = Async(target=_fake_context_returning_target,
args=[returned_context])
with _ExecutionContext(work):
run_job()
returned_context.start.assert_called_once_with()
def test_skip_empty_context_result(self, mock_handle_async_done):
"""Ensure run_job will not call start on an Context object, with
no tasks, as a result of the Async target function."""
from furious.async import Async
from furious.context import Context
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
returned_context = Mock(spec=Context, _tasks=[])
work = Async(target=_fake_context_returning_target,
args=[returned_context])
with _ExecutionContext(work):
with patch("furious.marker_tree.async_utils.handle_async_done",
autospec=True) as mock2_handle_async_done:
run_job()
mock2_handle_async_done.assert_called_once_with(work)
self.assertEqual(returned_context._handle_tasks.called, False)
def test_BaseException(self, dir_mock):
"""Ensure exceptions inheriting from BaseException, such as
DeadlineExceededError, trigger the error handler.
"""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
from google.appengine.runtime import DeadlineExceededError
dir_mock.side_effect = DeadlineExceededError
mock_success = Mock()
mock_error = Mock()
work = Async(target='dir',
callbacks={'success': mock_success,
'error': mock_error})
with _ExecutionContext(work):
run_job()
self.assertFalse(mock_success.called)
self.assertEqual(mock_error.call_count, 1)
