def test_add_pull_and_push_queue_tasks(self):
"""Ensure that push and pull tasks can be added with add_tasks()."""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import add_tasks
from furious.test_stubs.appengine.queues import get_tasks
from furious.test_stubs.appengine.queues import purge_tasks
# Add tasks the normal way so we can get them and test readding them
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
task_dict = get_tasks(self.queue_service)
# purge current tasks so we can verify how many we will add next.
purge_tasks(self.queue_service)
num_added = add_tasks(self.queue_service, task_dict)
# Purge tasks to check how many tasks are in the queues
num_queued = purge_tasks(self.queue_service)
self.assertEqual(3, num_added)
self.assertEqual(3, num_queued)
def test_purge_tasks_with_queue_names_provided(self, ctime):
"""When a list of queue_names is provided, ensure purge_tasks() clears
the tasks and none are left to execute.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service, ['default'])
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure two tasks from the default queue were cleared.
self.assertEqual(2, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_purge_tasks_with_string_passed_to_queue_names(self, ctime):
"""If a single queue_name is passed to purge_tasks() instead of a list,
ensure that the queue specified is still cleared.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Insert a pull task
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service, 'default')
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure two tasks from the default queue were cleared.
self.assertEqual(2, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_start_hits_transient_error_retry_disabled(self, queue_mock,
sleep_mock):
"""Ensure if transient error retries are disabled, that those errors are
re-raised immediately without any attempt to re-insert.
"""
from google.appengine.api.taskqueue import TransientError
from furious. async import Async
queue_mock.return_value.add.side_effect = TransientError()
async_job = Async("something",
queue='my_queue',
retry_transient_errors=False)
self.assertRaises(TransientError, async_job.start)
self.assertEqual(1, queue_mock.return_value.add.call_count)
# Try again with the option enabled, this should cause a retry after a
# delay, which we have also specified.
queue_mock.reset_mock()
async_job = Async("something",
queue='my_queue',
retry_transient_errors=True,
retry_delay=12)
self.assertRaises(TransientError, async_job.start)
self.assertEqual(2, queue_mock.return_value.add.call_count)
sleep_mock.assert_called_once_with(12)
def test_purge_tasks_with_tasks(self, ctime):
"""After queues are run, ensure no tasks are left to execute.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service)
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure three tasks were cleared, from 'default' and 'default-pull'.
self.assertEqual(3, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_to_dict_with_callbacks(self):
"""Ensure to_dict correctly encodes callbacks."""
from furious. async import Async
options = {
'callbacks': {
'success': self.__class__.test_to_dict_with_callbacks,
'failure': "failure_function",
'exec': Async(target=dir)
}
}
job = Async('nonexistant', **options.copy())
options['job'] = ('nonexistant', None, None)
options['callbacks'] = {
'success': ("furious.tests.test_async."
"TestAsync.test_to_dict_with_callbacks"),
'failure':
"failure_function",
'exec': {
'job': ('dir', None, None),
'_recursion': {
'current': 0,
'max': 100
},
'_type': 'furious.async.Async'
}
}
options['_recursion'] = {'current': 0, 'max': 100}
options['_type'] = 'furious.async.Async'
self.assertEqual(options, job.to_dict())
def test_double_init_raises_error(self):
"""Ensure initing twice raises a ContextExistsError."""
from furious. async import Async
from furious.context._execution import execution_context_from_async
from furious.errors import ContextExistsError
execution_context_from_async(Async(target=dir))
self.assertRaises(ContextExistsError, execution_context_from_async,
Async(target=dir))
def test_Abort(self, dir_mock, mock_start):
"""Ensures that when Abort is raised, the Async immediately stops."""
import logging
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.errors import Abort
from furious.processors import run_job
class AbortLogHandler(logging.Handler):
def emit(self, record):
if record.levelno >= logging.ERROR:
raise Exception('An Error level log should not be output')
logging.getLogger().addHandler(AbortLogHandler())
dir_mock.side_effect = Abort
mock_success = Mock()
mock_error = Mock()
work = Async(target='dir',
callbacks={'success': mock_success,
'error': mock_error})
with _ExecutionContext(work):
run_job()
self.assertFalse(mock_success.called)
self.assertFalse(mock_error.called)
self.assertFalse(mock_start.called)
logging.getLogger().removeHandler(AbortLogHandler())
def test_run(self, ctime):
"""Ensure tasks are run when run_queues is called."""
from furious.async import Async
from furious.test_stubs.appengine.queues import run as run_queues
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Run the tasks in the queue
run_queues(self.taskqueue_service)
self.assertEqual(2, ctime.call_count)
def test_context_works(self):
"""Ensure using a _ExecutionContext as a context manager works."""
from furious. async import Async
from furious.context._execution import _ExecutionContext
with _ExecutionContext(Async(target=dir)):
pass
def test_marker_not_complete_when_start_fails(self, mock_insert,
context_from_id,
check_markers):
"""Ensure if the completion handler fails to start, that the marker
does not get marked as complete.
"""
complete_event = Mock()
context = Context(id="contextid",
callbacks={'complete': complete_event})
context_from_id.return_value = context
check_markers.return_value = True, False
async = Async('foo')
async .update_options(context_id='contextid')
FuriousCompletionMarker(id=async .context_id, complete=False).put()
# Simulate the task failing to start
mock_insert.side_effect = DeadlineExceededError()
self.assertRaises(DeadlineExceededError, _completion_checker,
async .id, async .context_id)
# Marker should not have been marked complete.
current_marker = FuriousCompletionMarker.get_by_id(async .context_id)
self.assertFalse(current_marker.complete)
def test_markers_and_context_complete(self, mark, context_from_id,
check_markers):
"""Ensure if all markers are complete that True is returned and
nothing else is done.
"""
async = Async('foo')
async .update_options(context_id='contextid')
complete_event = Mock()
context = Context(id="contextid",
callbacks={'complete': complete_event})
context_from_id.return_value = context
marker = FuriousCompletionMarker(id="contextid", complete=True)
marker.put()
check_markers.return_value = True, False
mark.return_value = True
result = _completion_checker(async .id, async .context_id)
self.assertTrue(result)
self.assertFalse(complete_event.start.called)
marker.key.delete()
def get(self):
from furious. async import Async
count = int(self.request.get('tasks', 5))
Async(insert_tasks, queue='example', args=[count]).start()
self.response.out.write('Successfully inserted a group of Async jobs.')
def state_machine_success():
"""A positive result! Iterate!"""
from furious. async import Async
from furious.context import get_current_async
result = get_current_async().result
if result == 'ALPHA':
logging.info('Inserting continuation for state %s.', result)
return Async(target=complex_state_generator_alpha, args=[result])
elif result == 'BRAVO':
logging.info('Inserting continuation for state %s.', result)
return Async(target=complex_state_generator_bravo, args=[result])
logging.info('Done working, stop now.')
def add(self, target, args=None, kwargs=None, **options):
"""Add an Async job to this context.
Takes an Async object or the arguments to construct an Async
object as arguments. Returns the newly added Async object.
"""
from furious. async import Async
from furious.batcher import Message
if self._tasks_inserted:
raise errors.ContextAlreadyStartedError(
"This Context has already had its tasks inserted.")
if not isinstance(target, (Async, Message)):
target = Async(target, args, kwargs, **options)
target.update_options(_context_id=self.id)
if self.persist_async_results:
target.update_options(persist_result=True)
self._tasks.append(target)
self._options['_task_ids'].append(target.id)
return target
def test_calls_error_callback(self):
"""Ensure run_job catches any exceptions raised by the job, then calls
the error callback.
"""
from furious.async import Async
from furious.context._execution import _ExecutionContext
from furious.processors import run_job
call_count = []
handle_count = []
def handle_success():
call_count.append(1)
def handle_errors():
handle_count.append(1)
work = Async(target=dir, args=[1, 2],
callbacks={'success': handle_success,
'error': handle_errors})
with _ExecutionContext(work):
run_job()
self.assertEqual(1, len(handle_count),
"Error handler called wrong number of times.")
self.assertEqual(0, len(call_count),
"Success handler unexpectedly called.")
def start(number_of_items):
logging.info("******** Starting the process. ******** ")
job_id = uuid.uuid4().hex
logging.info("******* JOB ID: {}.".format(job_id))
start_time = _get_current_datetime_as_string()
rs = RunState(id=job_id)
rs.count_map = {}
cs = CompleteState(id=job_id)
cs.number_of_items = number_of_items
cs.complete = False
ndb.put_multi([rs, cs])
with context.new() as ctx:
ctx.set_event_handler('complete', Async(
completion_handler, args=[job_id, start_time]))
for i in xrange(number_of_items):
logging.info("###### JOB ITEM: {}.".format(i))
ctx.add(target=run_process,
args=[job_id, i, _get_current_datetime_as_string()])
logging.info("###### JOBS STARTED")
return job_id
def test_get_empty_task_args(self):
"""Ensure get_task_args returns {} if no task_args."""
from furious. async import Async
job = Async('nonexistant')
self.assertEqual({}, job.get_task_args())
def test_get_empty_headers(self):
"""Ensure get_headers returns the job headers."""
from furious. async import Async
job = Async('nonexistant')
self.assertEqual({}, job.get_headers())
def test_get_default_queue(self):
"""Ensure get_queue returns the default queue if non was given."""
from furious. async import Async
job = Async('nonexistant')
self.assertEqual('default', job.get_queue())
def test_init_opts_supersede_decorated_options(self):
"""Ensure options passed to init override decorated options."""
from furious. async import Async
from furious. async import defaults
options = {
'value': 1,
'other': 'zzz',
'nested': {
1: 1
},
'id': 'wrong',
'context_id': None,
'parent_id': 'parentid'
}
@defaults(**options.copy())
def some_function():
pass
job = Async(some_function, value=17, other='abc', id='correct')
options['value'] = 17
options['other'] = 'abc'
options['id'] = 'correct'
options['job'] = ("furious.tests.test_async.some_function", None, None)
options['_recursion'] = {'current': 0, 'max': 100}
self.assertEqual(options, job._options)
def test_update_options_supersede_init_opts(self):
"""Ensure update_options supersedes the options set in init."""
from furious. async import Async
options = {
'value': 1,
'other': 'zzz',
'nested': {
1: 1
},
'id': 'wrong',
'context_id': None,
'parent_id': 'parentid'
}
job = Async("nonexistant", **options.copy())
job.update_options(value=23, other='stuff', id='right')
options['value'] = 23
options['other'] = 'stuff'
options['id'] = 'right'
options['job'] = ("nonexistant", None, None)
options['_recursion'] = {'current': 0, 'max': 100}
self.assertEqual(options, job._options)
def test_decorated_options(self):
"""Ensure the defaults decorator sets Async options."""
from furious. async import Async
from furious. async import defaults
options = {
'value': 1,
'other': 'zzz',
'nested': {
1: 1
},
'id': 'thing',
'context_id': None,
'parent_id': 'parentid'
}
@defaults(**options.copy())
def some_function():
pass
job = Async(some_function)
options['job'] = ("furious.tests.test_async.some_function", None, None)
options['_recursion'] = {'current': 0, 'max': 100}
self.assertEqual(options, job._options)
def test_context_id(self):
"""Ensure context_id returns the context_id."""
from furious. async import Async
job = Async('somehting')
job.update_options(context_id='blarghahahaha')
self.assertEqual(job.context_id, 'blarghahahaha')
def test_deepcopy(self):
"""Make sure you can deepcopy an Async."""
import copy
from furious. async import Async
job = Async(dir)
copy.deepcopy(job)
def test_uses_given_id(self):
"""Ensure an id passed in is used."""
from furious. async import Async
job = Async('somehting', id='superrandom')
self.assertEqual(job.id, 'superrandom')
self.assertEqual(job.get_options()['id'], 'superrandom')
def test_no_args_with_kwargs(self):
"""Ensure no args with kwargs generate a well-formed job tuple."""
from furious. async import Async
job = ("test", None, {'a': 1, 'b': 'c', 'alpha': True})
async_job = Async(*job)
self.assertEqual(job, async_job.job)
def test_args_with_no_kwargs(self):
"""Ensure args and no kwargs generate a well-formed job tuple."""
from furious. async import Async
job = ("test", (1, 2, 3))
async_job = Async(*job)
self.assertEqual(job + (None, ), async_job.job)
def test_no_comletion(self):
"""Ensure does not fail if there's no completion checker."""
from furious.async import Async
from furious.processors import _handle_context_completion_check
async = Async(dir)
_handle_context_completion_check(async)
def test_job_params(self):
"""Ensure good args and kwargs generate a well-formed job tuple."""
from furious. async import Async
job = ("test", [1, 2, 3], {'a': 1, 'b': 2, 'c': 3})
async_job = Async(*job)
self.assertEqual(job, async_job.job)