def test_purge_tasks_with_queue_names_provided(self, ctime):
"""When a list of queue_names is provided, ensure purge_tasks() clears
the tasks and none are left to execute.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service, ['default'])
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure two tasks from the default queue were cleared.
self.assertEqual(2, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_add_pull_and_push_queue_tasks(self):
"""Ensure that push and pull tasks can be added with add_tasks()."""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import add_tasks
from furious.test_stubs.appengine.queues import get_tasks
from furious.test_stubs.appengine.queues import purge_tasks
# Add tasks the normal way so we can get them and test readding them
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
task_dict = get_tasks(self.queue_service)
# purge current tasks so we can verify how many we will add next.
purge_tasks(self.queue_service)
num_added = add_tasks(self.queue_service, task_dict)
# Purge tasks to check how many tasks are in the queues
num_queued = purge_tasks(self.queue_service)
self.assertEqual(3, num_added)
self.assertEqual(3, num_queued)
def test_purge_tasks_with_string_passed_to_queue_names(self, ctime):
"""If a single queue_name is passed to purge_tasks() instead of a list,
ensure that the queue specified is still cleared.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Insert a pull task
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service, 'default')
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure two tasks from the default queue were cleared.
self.assertEqual(2, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_start(self, queue_mock):
"""Ensure the Task is inserted into the specified queue."""
from furious.async import Async
async_job = Async("something", queue='my_queue')
# task = async_job.to_task()
async_job.start()
def test_purge_tasks_with_queue_names_provided(self, ctime):
"""When a list of queue_names is provided, ensure purge_tasks() clears
the tasks and none are left to execute.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service, ['default'])
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure two tasks from the default queue were cleared.
self.assertEqual(2, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_purge_tasks_with_tasks(self, ctime):
"""After queues are run, ensure no tasks are left to execute.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service)
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure three tasks were cleared, from 'default' and 'default-pull'.
self.assertEqual(3, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_purge_tasks_with_string_passed_to_queue_names(self, ctime):
"""If a single queue_name is passed to purge_tasks() instead of a list,
ensure that the queue specified is still cleared.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Insert a pull task
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service, 'default')
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure two tasks from the default queue were cleared.
self.assertEqual(2, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_purge_tasks_with_tasks(self, ctime):
"""After queues are run, ensure no tasks are left to execute.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service)
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure three tasks were cleared, from 'default' and 'default-pull'.
self.assertEqual(3, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_add_pull_and_push_queue_tasks(self):
"""Ensure that push and pull tasks can be added with add_tasks()."""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import add_tasks
from furious.test_stubs.appengine.queues import get_tasks
from furious.test_stubs.appengine.queues import purge_tasks
# Add tasks the normal way so we can get them and test readding them
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
task_dict = get_tasks(self.queue_service)
# purge current tasks so we can verify how many we will add next.
purge_tasks(self.queue_service)
num_added = add_tasks(self.queue_service, task_dict)
# Purge tasks to check how many tasks are in the queues
num_queued = purge_tasks(self.queue_service)
self.assertEqual(3, num_added)
self.assertEqual(3, num_queued)
def test_start_runs_successfully(self, queue_mock):
"""Ensure the Task is inserted into the specified queue."""
from furious. async import Async
async_job = Async("something", queue='my_queue')
async_job.start()
queue_mock.assert_called_once_with(name='my_queue')
self.assertTrue(queue_mock.return_value.add.called)
def test_start_runs_successfully(self, queue_mock):
"""Ensure the Task is inserted into the specified queue."""
from furious.async import Async
async_job = Async("something", queue="my_queue")
async_job.start()
queue_mock.assert_called_once_with(name="my_queue")
self.assertTrue(queue_mock.return_value.add.called)
def test_start_async_no_rpc(self, queue_add_async_mock):
"""Ensure that when the task is called with async=True, that the
add_async method is called with the default rpc=None.
"""
from furious.async import Async
async_job = Async("something")
async_job.start(async=True)
self.assertTrue(queue_add_async_mock.called)
self.assertEqual(None, queue_add_async_mock.call_args[1]["rpc"])
def test_task_non_transactional(self, queue_add_mock):
"""Ensure the task is added transactional when start is
called with transactional."""
from furious.async import Async
async_job = Async("something")
async_job.start(transactional=False)
call_args = queue_add_mock.call_args
call_kwargs = call_args[1]
self.assertIn('transactional', call_kwargs)
self.assertFalse(call_kwargs['transactional'])
def get(self):
from furious.async import Async
# Instantiate an Async
async_task = Async(target=aborting_function)
# Start an Async()
async_task.start()
logging.info('Original Async kicked off.')
self.response.write('Successfully inserted AbortAndRestart example.')
def get(self):
from furious. async import Async
# Instantiate an Async
async_task = Async(target=aborting_function)
# Start an Async()
async_task.start()
logging.info('Original Async kicked off.')
self.response.write('Successfully inserted AbortAndRestart example.')
def test_start_hits_tombstoned_task_error_error(self, queue_mock):
"""Ensure the task returns if a tombstoned task error is hit."""
from google.appengine.api.taskqueue import TombstonedTaskError
from furious.async import Async
queue_mock.return_value.add.side_effect = TombstonedTaskError()
async_job = Async("something", queue="my_queue")
async_job.start()
queue_mock.assert_called_with(name="my_queue")
self.assertEqual(1, queue_mock.return_value.add.call_count)
def test_task_non_transactional(self, queue_add_mock):
"""Ensure the task is added transactional when start is
called with transactional."""
from furious. async import Async
async_job = Async("something")
async_job.start(transactional=False)
call_args = queue_add_mock.call_args
call_kwargs = call_args[1]
self.assertIn('transactional', call_kwargs)
self.assertFalse(call_kwargs['transactional'])
def test_start_hits_tombstoned_task_error_error(self, queue_mock):
"""Ensure the task returns if a tombstoned task error is hit."""
from google.appengine.api.taskqueue import TombstonedTaskError
from furious. async import Async
queue_mock.return_value.add.side_effect = TombstonedTaskError()
async_job = Async("something", queue='my_queue')
async_job.start()
queue_mock.assert_called_with(name='my_queue')
self.assertEqual(1, queue_mock.return_value.add.call_count)
def test_start_hits_task_already_exists_error_error(self, queue_mock):
"""Ensure the task returns if a task already exists error is hit."""
from google.appengine.api.taskqueue import TaskAlreadyExistsError
from furious.async import Async
queue_mock.return_value.add.side_effect = TaskAlreadyExistsError()
async_job = Async("something", queue='my_queue')
async_job.start()
queue_mock.assert_called_with(name='my_queue')
self.assertEqual(1, queue_mock.return_value.add.call_count)
def test_start_async_with_rpc(self, queue_add_async_mock):
"""Ensure that when the task is called with async=True and an rpc is
provided, that the add_async method is called with the correct rpc.
"""
from mock import Mock
from furious.async import Async
rpc = Mock()
async_job = Async("something")
async_job.start(async=True, rpc=rpc)
self.assertTrue(queue_add_async_mock.called)
self.assertEqual(rpc, queue_add_async_mock.call_args[1]["rpc"])
def get(self):
from furious.async import Async
# Instantiate an Async object.
async_task = Async(
target=example_function, args=[1], kwargs={'some': 'value'})
# Insert the task to run the Async object, note that it may begin
# executing immediately or with some delay.
async_task.start()
logging.info('Async job kicked off.')
self.response.out.write('Successfully inserted Async job.')
def task_call_task2():
"""The function task2 will call."""
self.async2_call_count += 1
if self.async2_call_count < 3:
# Fail twice.
raise Exception()
self.async2_retries_env = int(
os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT'))
async3 = Async(target='time.accept2dyear')
async3.start()
def get(self):
from furious.async import Async
# Instantiate an Async object.
async_task = Async(
target=example_function, args=[1], kwargs={'some': 'value'})
# Insert the task to run the Async object, note that it may begin
# executing immediately or with some delay.
async_task.start()
logging.info('Async job kicked off.')
self.response.out.write('Successfully inserted Async job.')
def task_call_task2():
"""The function task2 will call."""
self.async2_call_count += 1
if self.async2_call_count < 3:
# Fail twice.
raise Exception()
self.async2_retries_env = int(
os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT'))
async3 = Async(target='time.accept2dyear')
async3.start()
def get(self):
from furious. async import Async
# Instantiate an Async object, specifying a 'error' callback.
async_task = Async(target=dir,
args=[1, 2, 3],
callbacks={'error': handle_an_error})
# Insert the task to run the Async object. The error callback will be
# executed in the furious task after the job has raised an exception.
async_task.start()
logging.info('Erroneous Async job kicked off.')
self.response.out.write('Successfully inserted Async job.')
def get(self):
from furious. async import Async
# Instantiate an Async object, specifying a 'success' callback.
async_task = Async(target=example_function,
args=[1],
kwargs={'some': 'value'},
callbacks={'success': all_done})
# Insert the task to run the Async object. The success callback will
# be executed in the furious task after the job is executed.
async_task.start()
logging.info('Async job kicked off.')
self.response.out.write('Successfully inserted Async job.')
def test_run(self, ctime):
"""Ensure tasks are run when run_queues is called."""
from furious.async import Async
from furious.test_stubs.appengine.queues import run as run_queues
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Run the tasks in the queue
run_queues(self.taskqueue_service)
self.assertEqual(2, ctime.call_count)
def get(self):
from furious.async import Async
# Instantiate an Async object, specifying a 'error' callback.
async_task = Async(
target=dir, args=[1, 2, 3],
callbacks={'error': handle_an_error}
)
# Insert the task to run the Async object. The error callback will be
# executed in the furious task after the job has raised an exception.
async_task.start()
logging.info('Erroneous Async job kicked off.')
self.response.out.write('Successfully inserted Async job.')
def get(self):
from furious.async import Async
# Instantiate an Async object, specifying a 'success' callback.
async_task = Async(
target=example_function, args=[1], kwargs={'some': 'value'},
callbacks={'success': all_done}
)
# Insert the task to run the Async object. The success callback will
# be executed in the furious task after the job is executed.
async_task.start()
logging.info('Async job kicked off.')
self.response.out.write('Successfully inserted Async job.')
def test_run(self, ctime):
"""Ensure tasks are run when run_queues is called."""
from furious.async import Async
from furious.test_stubs.appengine.queues import run as run_queues
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Run the tasks in the queue
run_queues(self.taskqueue_service)
self.assertEqual(2, ctime.call_count)
def test_start_hits_transient_error(self, queue_mock):
"""Ensure the task retries if a transient error is hit."""
from google.appengine.api.taskqueue import TransientError
from furious.async import Async
def add(task, *args, **kwargs):
def add_second(task, *args, **kwargs):
assert task
queue_mock.return_value.add.side_effect = add_second
raise TransientError()
queue_mock.return_value.add.side_effect = add
async_job = Async("something", queue='my_queue')
async_job.start()
queue_mock.assert_called_with(name='my_queue')
self.assertEqual(2, queue_mock.return_value.add.call_count)
def test_start_hits_transient_error(self, queue_mock):
"""Ensure the task retries if a transient error is hit."""
from google.appengine.api.taskqueue import TransientError
from furious. async import Async
def add(task, *args, **kwargs):
def add_second(task, *args, **kwargs):
assert task
queue_mock.return_value.add.side_effect = add_second
raise TransientError()
queue_mock.return_value.add.side_effect = add
async_job = Async("something", queue='my_queue')
async_job.start()
queue_mock.assert_called_with(name='my_queue')
self.assertEqual(2, queue_mock.return_value.add.call_count)
def test_get_tasks_from_all_queues(self):
"""Ensure all tasks are returned from get_tasks()."""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import get_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Insert a pull task
Message(queue='default-pull').insert()
task_dict = get_tasks(self.queue_service)
num_tasks = sum([len(task_list) for task_list in task_dict.values()])
self.assertEqual(3, num_tasks)
def test_get_tasks_from_all_queues(self):
"""Ensure all tasks are returned from get_tasks()."""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import get_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Insert a pull task
Message(queue='default-pull').insert()
task_dict = get_tasks(self.queue_service)
num_tasks = sum([len(task_list) for task_list in task_dict.values()])
self.assertEqual(3, num_tasks)
def test_get_tasks_when_queue_names_are_specified(self):
"""Ensure queues' tasks are returned from get_tasks() when a list of
queue_names are passed as an argument.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import get_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Insert a pull task
Message(queue='default-pull').insert()
task_dict = get_tasks(self.queue_service, ['default'])
num_tasks = sum([len(task_list) for task_list in task_dict.values()])
self.assertEqual(2, num_tasks)
def test_get_tasks_when_queue_names_are_specified(self):
"""Ensure queues' tasks are returned from get_tasks() when a list of
queue_names are passed as an argument.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import get_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Insert a pull task
Message(queue='default-pull').insert()
task_dict = get_tasks(self.queue_service, ['default'])
num_tasks = sum([len(task_list) for task_list in task_dict.values()])
self.assertEqual(2, num_tasks)
def get(self):
from furious. async import Async
# Instantiate an Async object to act as our success callback.
# NOTE: Your async.result is not directly available from the
# success_callback Async, you will need to persist the result
# and fetch it from the other Async if needed.
success_callback = Async(target=example_function,
kwargs={'it': 'worked'})
# Instantiate an Async object, setting the success_callback to the
# above Async object.
async_task = Async(target=example_function,
kwargs={'trigger': 'job'},
callbacks={'success': success_callback})
# Insert the task to run the Async object.
async_task.start()
logging.info('Async job kicked off.')
self.response.out.write('Successfully inserted Async job.')
def get(self):
from furious.async import Async
# Instantiate an Async object to act as our success callback.
# NOTE: Your async.result is not directly available from the
# success_callback Async, you will need to persist the result
# and fetch it from the other Async if needed.
success_callback = Async(
target=example_function, kwargs={'it': 'worked'}
)
# Instantiate an Async object, setting the success_callback to the
# above Async object.
async_task = Async(
target=example_function, kwargs={'trigger': 'job'},
callbacks={'success': success_callback}
)
# Insert the task to run the Async object.
async_task.start()
logging.info('Async job kicked off.')
self.response.out.write('Successfully inserted Async job.')
def test_run_with_retries_and_retries_reset(self, accept2dyear, asctime,
ctime):
"""
Ensure tasks retry counts are separate between asyncs.
Ensure tasks retry counts are reset once an Async is successful.
"""
from furious.async import Async
from furious.test_stubs.appengine.queues import run as run_queues
# Count the task runs.
self.async1_call_count = 0
self.async2_call_count = 0
self.async3_call_count = 0
self.async1_retries_env = 0
self.async2_retries_env = 0
self.async3_retries_env = 0
def task_call_task1():
"""The function task1 will call."""
int(os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT'))
self.async1_call_count += 1
if self.async1_call_count < 2:
# Fail once.
raise Exception()
self.async1_retries_env = int(
os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT'))
def task_call_task3():
"""The function task3 will call."""
self.async3_call_count += 1
self.async3_retries_env = int(
os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT'))
def task_call_task2():
"""The function task2 will call."""
self.async2_call_count += 1
if self.async2_call_count < 3:
# Fail twice.
raise Exception()
self.async2_retries_env = int(
os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT'))
async3 = Async(target='time.accept2dyear')
async3.start()
ctime.side_effect = task_call_task1
asctime.side_effect = task_call_task2
accept2dyear.side_effect = task_call_task3
# Enqueue our task that will fail.
async1 = Async(target='time.ctime')
async1.start()
async2 = Async(target='time.asctime')
async2.start()
# Run the tasks in the queue
run_queues(self.taskqueue_service, enable_retries=True)
self.assertEqual(self.async1_call_count, 2)
self.assertEqual(self.async2_call_count, 3)
self.assertEqual(self.async3_call_count, 1)
self.assertEqual(self.async1_retries_env, 1)
self.assertEqual(self.async2_retries_env, 2)
self.assertEqual(self.async3_retries_env, 0)
# Clear
del self.async1_call_count
del self.async2_call_count
del self.async3_call_count
del self.async1_retries_env
del self.async2_retries_env
del self.async3_retries_env
def test_run_with_retries_and_retries_reset(self, accept2dyear, asctime,
ctime):
"""
Ensure tasks retry counts are separate between asyncs.
Ensure tasks retry counts are reset once an Async is successful.
"""
from furious. async import Async
from furious.test_stubs.appengine.queues import run as run_queues
# Count the task runs.
self.async1_call_count = 0
self.async2_call_count = 0
self.async3_call_count = 0
self.async1_retries_env = 0
self.async2_retries_env = 0
self.async3_retries_env = 0
def task_call_task1():
"""The function task1 will call."""
int(os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT'))
self.async1_call_count += 1
if self.async1_call_count < 2:
# Fail once.
raise Exception()
self.async1_retries_env = int(
os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT'))
def task_call_task3():
"""The function task3 will call."""
self.async3_call_count += 1
self.async3_retries_env = int(
os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT'))
def task_call_task2():
"""The function task2 will call."""
self.async2_call_count += 1
if self.async2_call_count < 3:
# Fail twice.
raise Exception()
self.async2_retries_env = int(
os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT'))
async3 = Async(target='time.accept2dyear')
async3.start()
ctime.side_effect = task_call_task1
asctime.side_effect = task_call_task2
accept2dyear.side_effect = task_call_task3
# Enqueue our task that will fail.
async1 = Async(target='time.ctime')
async1.start()
async2 = Async(target='time.asctime')
async2.start()
# Run the tasks in the queue
run_queues(self.taskqueue_service, enable_retries=True)
self.assertEqual(self.async1_call_count, 2)
self.assertEqual(self.async2_call_count, 3)
self.assertEqual(self.async3_call_count, 1)
self.assertEqual(self.async1_retries_env, 1)
self.assertEqual(self.async2_retries_env, 2)
self.assertEqual(self.async3_retries_env, 0)
# Clear
del self.async1_call_count
del self.async2_call_count
del self.async3_call_count
del self.async1_retries_env
del self.async2_retries_env
del self.async3_retries_env
