def add(self, target, args=None, kwargs=None, **options): """Add an Async job to this context. Takes an Async object or the arguments to construct an Async object as arguments. Returns the newly added Async object. """ from furious.async import Async from furious.batcher import Message if self._tasks_inserted: raise errors.ContextAlreadyStartedError( "This Context has already had its tasks inserted.") if not isinstance(target, (Async, Message)): target = Async(target, args, kwargs, **options) target.update_options(_context_id=self.id) if self.persist_async_results: target.update_options(persist_result=True) self._tasks.append(target) self._options['_task_ids'].append(target.id) return target
def test_get_default_queue(self): """Ensure get_queue returns the default queue if non was given.""" from furious.async import Async job = Async("nonexistant") self.assertEqual("default", job.get_queue())
def test_to_dict_with_callbacks(self): """Ensure to_dict correctly encodes callbacks.""" from furious.async import Async options = { "id": "anident", "context_id": "contextid", "parent_id": "parentid", "callbacks": { "success": self.__class__.test_to_dict_with_callbacks, "failure": "failure_function", "exec": Async(target=dir, id="subidnet", parent_id="parentid"), }, } job = Async("nonexistant", **options.copy()) options["job"] = ("nonexistant", None, None) options["callbacks"] = { "success": ("furious.tests.test_async." "TestAsync.test_to_dict_with_callbacks"), "failure": "failure_function", "exec": { "job": ("dir", None, None), "id": "subidnet", "context_id": None, "parent_id": "parentid", "_recursion": {"current": 0, "max": 100}, "_type": "furious.async.Async", }, } options["_recursion"] = {"current": 0, "max": 100} options["_type"] = "furious.async.Async" self.assertEqual(options, job.to_dict())
def test_get_empty_task_args(self): """Ensure get_task_args returns {} if no task_args.""" from furious.async import Async job = Async("nonexistant") self.assertEqual({}, job.get_task_args())
def test_update_options_supersede_init_opts(self): """Ensure update_options supersedes the options set in init.""" from furious.async import Async options = { "value": 1, "other": "zzz", "nested": {1: 1}, "id": "wrong", "context_id": None, "parent_id": "parentid", } job = Async("nonexistant", **options.copy()) job.update_options(value=23, other="stuff", id="right") options["value"] = 23 options["other"] = "stuff" options["id"] = "right" options["job"] = ("nonexistant", None, None) options["_recursion"] = {"current": 0, "max": 100} self.assertEqual(options, job._options)
def test_to_dict_with_callbacks(self): """Ensure to_dict correctly encodes callbacks.""" from furious.async import Async options = {'id': 'anident', 'context_id': 'contextid', 'callbacks': { 'success': self.__class__.test_to_dict_with_callbacks, 'failure': "failure_function", 'exec': Async(target=dir, id='subidnet'), }} job = Async('nonexistant', **options.copy()) options['job'] = ('nonexistant', None, None) options['callbacks'] = { 'success': ("furious.tests.test_async." "TestAsync.test_to_dict_with_callbacks"), 'failure': "failure_function", 'exec': {'job': ('dir', None, None), 'id': 'subidnet', 'context_id': None, '_recursion': {'current': 0, 'max': 100}, '_type': 'furious.async.Async'} } options['_recursion'] = {'current': 0, 'max': 100} options['_type'] = 'furious.async.Async' self.assertEqual(options, job.to_dict())
def test_start(self, queue_mock): """Ensure the Task is inserted into the specified queue.""" from furious.async import Async async_job = Async("something", queue='my_queue') # task = async_job.to_task() async_job.start()
def test_context_id(self): """Ensure context_id returns the context_id.""" from furious.async import Async job = Async("somehting") job.update_options(context_id="blarghahahaha") self.assertEqual(job.context_id, "blarghahahaha")
def test_purge_tasks_with_tasks(self, ctime): """After queues are run, ensure no tasks are left to execute. Ensure the number of tasks cleared is correct. """ from furious.async import Async from furious.batcher import Message from furious.test_stubs.appengine.queues import run as run_queues from furious.test_stubs.appengine.queues import purge_tasks # Enqueue a couple of tasks async = Async(target='time.ctime') async.start() async2 = Async(target='time.ctime') async2.start() Message(queue='default-pull').insert() num_cleared = purge_tasks(self.taskqueue_service) # Run the tasks to check if tasks remain run_queues(self.taskqueue_service) # Ensure three tasks were cleared, from 'default' and 'default-pull'. self.assertEqual(3, num_cleared) # Ensure no tasks were run self.assertEqual(0, ctime.call_count)
def test_purge_tasks_with_queue_names_provided(self, ctime): """When a list of queue_names is provided, ensure purge_tasks() clears the tasks and none are left to execute. Ensure the number of tasks cleared is correct. """ from furious.async import Async from furious.batcher import Message from furious.test_stubs.appengine.queues import run as run_queues from furious.test_stubs.appengine.queues import purge_tasks # Enqueue a couple of tasks async = Async(target='time.ctime') async.start() async2 = Async(target='time.ctime') async2.start() Message(queue='default-pull').insert() num_cleared = purge_tasks(self.taskqueue_service, ['default']) # Run the tasks to check if tasks remain run_queues(self.taskqueue_service) # Ensure two tasks from the default queue were cleared. self.assertEqual(2, num_cleared) # Ensure no tasks were run self.assertEqual(0, ctime.call_count)
def test_purge_tasks_with_string_passed_to_queue_names(self, ctime): """If a single queue_name is passed to purge_tasks() instead of a list, ensure that the queue specified is still cleared. Ensure the number of tasks cleared is correct. """ from furious.async import Async from furious.batcher import Message from furious.test_stubs.appengine.queues import run as run_queues from furious.test_stubs.appengine.queues import purge_tasks # Enqueue a couple of tasks async = Async(target='time.ctime') async.start() async2 = Async(target='time.ctime') async2.start() # Insert a pull task Message(queue='default-pull').insert() num_cleared = purge_tasks(self.taskqueue_service, 'default') # Run the tasks to check if tasks remain run_queues(self.taskqueue_service) # Ensure two tasks from the default queue were cleared. self.assertEqual(2, num_cleared) # Ensure no tasks were run self.assertEqual(0, ctime.call_count)
def test_get_empty_headers(self): """Ensure get_headers returns the job headers.""" from furious.async import Async job = Async("nonexistant") self.assertEqual({}, job.get_headers())
def test_add_pull_and_push_queue_tasks(self): """Ensure that push and pull tasks can be added with add_tasks().""" from furious.async import Async from furious.batcher import Message from furious.test_stubs.appengine.queues import add_tasks from furious.test_stubs.appengine.queues import get_tasks from furious.test_stubs.appengine.queues import purge_tasks # Add tasks the normal way so we can get them and test readding them async = Async(target='time.ctime') async.start() async2 = Async(target='time.ctime') async2.start() Message(queue='default-pull').insert() task_dict = get_tasks(self.queue_service) # purge current tasks so we can verify how many we will add next. purge_tasks(self.queue_service) num_added = add_tasks(self.queue_service, task_dict) # Purge tasks to check how many tasks are in the queues num_queued = purge_tasks(self.queue_service) self.assertEqual(3, num_added) self.assertEqual(3, num_queued)
def test_uses_given_id(self): """Ensure an id passed in is used.""" from furious.async import Async job = Async('somehting', id='superrandom') self.assertEqual(job.id, 'superrandom') self.assertEqual(job.get_options()['id'], 'superrandom')
def test_uses_given_id(self): """Ensure an id passed in is used.""" from furious.async import Async job = Async("somehting", id="superrandom") self.assertEqual(job.id, "superrandom") self.assertEqual(job.get_options()["id"], "superrandom")
def test_get_queue(self): """Ensure get_queue returns the job queue.""" from furious.async import Async queue = "test" job = Async("nonexistant", queue=queue) self.assertEqual(queue, job.get_queue())
def test_update_id(self): """Ensure using update options to update an id works.""" from furious.async import Async job = Async("somehting") job.update_options(id="newid") self.assertEqual(job.id, "newid") self.assertEqual(job.get_options()["id"], "newid")
def test_setting_result(self): """Ensure the result can be set if the execute flag is set.""" from furious.async import Async job = Async(target=dir) job.executing = True job.result = 123 self.assertEqual(123, job.result) self.assertTrue(job.executed)
def test_start_runs_successfully(self, queue_mock): """Ensure the Task is inserted into the specified queue.""" from furious.async import Async async_job = Async("something", queue="my_queue") async_job.start() queue_mock.assert_called_once_with(name="my_queue") self.assertTrue(queue_mock.return_value.add.called)
def test_update_id(self): """Ensure using update options to update an id works.""" from furious.async import Async job = Async('somehting') job.update_options(id='newid') self.assertEqual(job.id, 'newid') self.assertEqual(job.get_options()['id'], 'newid')
def test_async_to_marker(self, invalid_group_id): from furious.marker_tree.marker import Marker from furious.async import Async task = Async(target=dir) task.id = "1" marker = Marker.from_async(task) self.assertIsNone(marker.group_id)
def test_get_options(self): """Ensure get_options returns the job options.""" from furious.async import Async options = {"value": 1, "other": "zzz", "nested": {1: 1}} job = Async("nonexistant") job._options = options self.assertEqual(options, job.get_options())
def test_getting_result(self): """Ensure getting the result after executing works.""" from furious.async import Async job = Async(target=dir) job._executing = True job.result = 123456 self.assertEqual(123456, job.result) self.assertTrue(job.executed)
def test_get_task_args(self): """Ensure get_task_args returns the job task_args.""" from furious.async import Async task_args = {"other": "zzz", "nested": 1} options = {"task_args": task_args} job = Async("nonexistant", **options) self.assertEqual(task_args, job.get_task_args())
def test_retry_value_is_decodable(self): """Ensure that from_dict is the inverse of to_dict when retry options are given. """ from furious.async import Async async_job = Async("something", task_args={"retry_options": {"task_retry_limit": 5}}) new_async_job = Async.from_dict(async_job.to_dict()) self.assertEqual(async_job.to_dict(), new_async_job.to_dict())
def test_get_options(self): """Ensure get_options returns the job options.""" from furious.async import Async options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}} job = Async("nonexistant") job._options = options self.assertEqual(options, job.get_options())
def test_get_headers(self): """Ensure get_headers returns the job headers.""" from furious.async import Async headers = {"other": "zzz", "nested": 1} options = {"headers": headers} job = Async("nonexistant", **options) self.assertEqual(headers, job.get_headers())
def test_get_task_args(self): """Ensure get_task_args returns the job task_args.""" from furious.async import Async task_args = {'other': 'zzz', 'nested': 1} options = {'task_args': task_args} job = Async('nonexistant', **options) self.assertEqual(task_args, job.get_task_args())
def test_get_headers(self): """Ensure get_headers returns the job headers.""" from furious.async import Async headers = {'other': 'zzz', 'nested': 1} options = {'headers': headers} job = Async('nonexistant', **options) self.assertEqual(headers, job.get_headers())
def test_retry_custom(self): """Ensure that when a custom retry limit is set, that it's propagated. """ from furious.async import Async async_job = Async("something", task_args={"retry_options": {"task_retry_limit": 5}}) task = async_job.to_task() self.assertEqual(5, task.retry_options.task_retry_limit)
def test_retry_default(self): """Ensure that when no task_retry_limit specified, that the default is set. """ from furious. async import Async from furious. async import MAX_RESTARTS async_job = Async("something") task = async_job.to_task() self.assertEqual(MAX_RESTARTS, task.retry_options.task_retry_limit)
def test_update_recursion_level_defaults(self): """Ensure that defaults (1, MAX_DEPTH) are set correctly.""" from furious. async import Async from furious. async import MAX_DEPTH async_job = Async("something") async_job._increment_recursion_level() options = async_job.get_options()['_recursion'] self.assertEqual(1, options['current']) self.assertEqual(MAX_DEPTH, options['max'])
def test_task_non_transactional(self, queue_add_mock): """Ensure the task is added transactional when start is called with transactional.""" from furious. async import Async async_job = Async("something") async_job.start(transactional=False) call_args = queue_add_mock.call_args call_kwargs = call_args[1] self.assertIn('transactional', call_kwargs) self.assertFalse(call_kwargs['transactional'])
def test_start_hits_tombstoned_task_error_error(self, queue_mock): """Ensure the task returns if a tombstoned task error is hit.""" from google.appengine.api.taskqueue import TombstonedTaskError from furious. async import Async queue_mock.return_value.add.side_effect = TombstonedTaskError() async_job = Async("something", queue='my_queue') async_job.start() queue_mock.assert_called_with(name='my_queue') self.assertEqual(1, queue_mock.return_value.add.call_count)
def get(self): from furious. async import Async # Instantiate an Async async_task = Async(target=aborting_function) # Start an Async() async_task.start() logging.info('Original Async kicked off.') self.response.write('Successfully inserted AbortAndRestart example.')
def test_check_recursion_disabled(self): """Ensure that when recursion max depth is explicitly set to -1, then the recursion check is disabled. There are no explicit asserts in this test because the check_recursion_depth() method would throw an exception if this functionality wasn't working. """ from furious. async import Async async_job = Async("something", _recursion={'current': 101, 'max': -1}) async_job.check_recursion_depth()
def test_retry_value_is_decodable(self): """Ensure that from_dict is the inverse of to_dict when retry options are given. """ from furious. async import Async async_job = Async("something", task_args={'retry_options': { 'task_retry_limit': 5 }}) new_async_job = Async.from_dict(async_job.to_dict()) self.assertEqual(async_job.to_dict(), new_async_job.to_dict())
def test_retry_custom(self): """Ensure that when a custom retry limit is set, that it's propagated. """ from furious. async import Async async_job = Async("something", task_args={'retry_options': { 'task_retry_limit': 5 }}) task = async_job.to_task() self.assertEqual(5, task.retry_options.task_retry_limit)
def test_update_options(self): """Ensure update_options updates the options.""" from furious. async import Async options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}} job = Async("nonexistant") job.update_options(**options.copy()) options['job'] = ("nonexistant", None, None) options['_recursion'] = {'current': 0, 'max': 100} self.assertEqual(options, job._options)
def task_call_task2(): """The function task2 will call.""" self.async2_call_count += 1 if self.async2_call_count < 3: # Fail twice. raise Exception() self.async2_retries_env = int( os.environ.get('HTTP_X_APPENGINE_TASKRETRYCOUNT')) async3 = Async(target='time.accept2dyear') async3.start()
def get(self): from furious.async import Async # Instantiate an Async object. async_task = Async( target=example_function, args=[1], kwargs={'some': 'value'}) # Insert the task to run the Async object, note that it may begin # executing immediately or with some delay. async_task.start() logging.info('Async job kicked off.') self.response.out.write('Successfully inserted Async job.')
def test_to_task(self): """Ensure to_task produces the right task object.""" import datetime import time from google.appengine.ext import testbed from furious. async import Async from furious. async import ASYNC_ENDPOINT testbed = testbed.Testbed() testbed.activate() # This just drops the microseconds. It is a total mess, but is needed # to handle all the rounding crap. eta = datetime.datetime.now() + datetime.timedelta(minutes=43) eta_posix = time.mktime(eta.timetuple()) headers = {'some': 'thing', 'fun': 1} job = ('test', None, None) expected_url = "%s/%s" % (ASYNC_ENDPOINT, 'test') task_args = {'eta': eta_posix} options = {'job': job, 'headers': headers, 'task_args': task_args} task = Async.from_dict(options).to_task() # App Engine sets these headers by default. full_headers = { 'Host': 'testbed.example.com', 'X-AppEngine-Current-Namespace': '' } full_headers.update(headers) self.assertEqual(eta_posix, task.eta_posix) self.assertEqual(expected_url, task.url) self.assertEqual(full_headers, task.headers) options['task_args']['eta'] = datetime.datetime.fromtimestamp( eta_posix) options['_recursion'] = {'current': 1, 'max': 100} options['_type'] = 'furious.async.Async' self.assertEqual( options, Async.from_dict(json.loads(task.payload)).get_options())
def test_no_type(self): """Ensure that if not _type is in options, that it defaults to furious.async.Async. """ from furious. async import Async from furious. async import async_from_options async_job = Async(dir) options = async_job.to_dict() options.pop('_type') result = async_from_options(options) self.assertIsInstance(result, Async)
def test_context_checker_encoded_and_decoded(self): """Ensure the _context_checker is correctly encoded to and decoded from an Async options dict. """ from furious. async import Async async_job = Async("something", _context_checker=dir) encoded_async = async_job.to_dict() self.assertEqual(encoded_async['__context_checker'], 'dir') new_async_job = Async.from_dict(encoded_async) self.assertEqual(new_async_job.get_options()['_context_checker'], dir) self.assertEqual(async_job.to_dict(), new_async_job.to_dict())
def test_to_dict(self): """Ensure to_dict returns a dictionary representation of the Async.""" from furious. async import Async task_args = {'other': 'zzz', 'nested': 1} headers = {'some': 'thing', 'fun': 1} options = {'headers': headers, 'task_args': task_args} job = Async('nonexistant', **options.copy()) options['job'] = ('nonexistant', None, None) options['_recursion'] = {'current': 0, 'max': 100} options['_type'] = 'furious.async.Async' self.assertEqual(options, job.to_dict())
def get(self): from furious. async import Async # Instantiate an Async object, specifying a 'error' callback. async_task = Async(target=dir, args=[1, 2, 3], callbacks={'error': handle_an_error}) # Insert the task to run the Async object. The error callback will be # executed in the furious task after the job has raised an exception. async_task.start() logging.info('Erroneous Async job kicked off.') self.response.out.write('Successfully inserted Async job.')
def test_Abort(self, dir_mock, mock_start): """Ensures that when Abort is raised, the Async immediately stops.""" import logging from furious.async import Async from furious.context._execution import _ExecutionContext from furious.errors import Abort from furious.processors import run_job class AbortLogHandler(logging.Handler): def emit(self, record): if record.levelno >= logging.ERROR: raise Exception('An Error level log should not be output') logging.getLogger().addHandler(AbortLogHandler()) dir_mock.side_effect = Abort mock_success = Mock() mock_error = Mock() work = Async(target='dir', callbacks={'success': mock_success, 'error': mock_error}) with _ExecutionContext(work): run_job() self.assertFalse(mock_success.called) self.assertFalse(mock_error.called) self.assertFalse(mock_start.called) logging.getLogger().removeHandler(AbortLogHandler())
def test_process_results_encoded_and_decoded(self): """Ensure _process_results is correctly encoded to and decoded from an Async options dict. """ from furious. async import Async async_job = Async("something", _process_results=locals) encoded_async = async_job.to_dict() self.assertEqual(encoded_async['__process_results'], 'locals') new_async_job = Async.from_dict(encoded_async) self.assertEqual(new_async_job.get_options()['_process_results'], locals) self.assertEqual(async_job.to_dict(), new_async_job.to_dict())
def test_retry_value_with_to_task(self): """Ensure that calling to_task doesn't affect the options when encoding. """ from furious. async import Async from furious. async import encode_async_options async_job = Async("something", task_args={'retry_options': { 'task_retry_limit': 5 }}) async_job.to_task() options = encode_async_options(async_job) self.assertEqual( 5, options['task_args']['retry_options']['task_retry_limit'])
def state_machine_success(): """A positive result! Iterate!""" from furious. async import Async from furious.context import get_current_async result = get_current_async().result if result == 'ALPHA': logging.info('Inserting continuation for state %s.', result) return Async(target=complex_state_generator_alpha, args=[result]) elif result == 'BRAVO': logging.info('Inserting continuation for state %s.', result) return Async(target=complex_state_generator_bravo, args=[result]) logging.info('Done working, stop now.')
def get(self): from furious. async import Async count = int(self.request.get('tasks', 5)) Async(insert_tasks, queue='example', args=[count]).start() self.response.out.write('Successfully inserted a group of Async jobs.')
def test_markers_and_context_complete(self, mark, context_from_id, check_markers): """Ensure if all markers are complete that True is returned and nothing else is done. """ async = Async('foo') async .update_options(context_id='contextid') complete_event = Mock() context = Context(id="contextid", callbacks={'complete': complete_event}) context_from_id.return_value = context marker = FuriousCompletionMarker(id="contextid", complete=True) marker.put() check_markers.return_value = True, False mark.return_value = True result = _completion_checker(async .id, async .context_id) self.assertTrue(result) self.assertFalse(complete_event.start.called) marker.key.delete()
def test_marker_not_complete_when_start_fails(self, mock_insert, context_from_id, check_markers): """Ensure if the completion handler fails to start, that the marker does not get marked as complete. """ complete_event = Mock() context = Context(id="contextid", callbacks={'complete': complete_event}) context_from_id.return_value = context check_markers.return_value = True, False async = Async('foo') async .update_options(context_id='contextid') FuriousCompletionMarker(id=async .context_id, complete=False).put() # Simulate the task failing to start mock_insert.side_effect = DeadlineExceededError() self.assertRaises(DeadlineExceededError, _completion_checker, async .id, async .context_id) # Marker should not have been marked complete. current_marker = FuriousCompletionMarker.get_by_id(async .context_id) self.assertFalse(current_marker.complete)
def test_context_works(self): """Ensure using a _ExecutionContext as a context manager works.""" from furious. async import Async from furious.context._execution import _ExecutionContext with _ExecutionContext(Async(target=dir)): pass
def test_reconstitution(self): """Ensure to_dict(job.from_dict()) returns the same thing.""" from furious. async import Async headers = {'some': 'thing', 'fun': 1} job = ('test', None, None) task_args = {'other': 'zzz', 'nested': 1} options = { 'job': job, 'id': 'someid', 'headers': headers, 'task_args': task_args, 'persistence_engine': 'furious.extras.appengine.ndb_persistence', '_recursion': { 'current': 1, 'max': 100 }, '_type': 'furious.async.Async', 'context_id': None, 'parent_id': 'parentid' } async_job = Async.from_dict(options) self.assertEqual(options, async_job.to_dict())
def get(self): from furious. async import Async # Instantiate an Async object, specifying a 'success' callback. async_task = Async(target=example_function, args=[1], kwargs={'some': 'value'}, callbacks={'success': all_done}) # Insert the task to run the Async object. The success callback will # be executed in the furious task after the job is executed. async_task.start() logging.info('Async job kicked off.') self.response.out.write('Successfully inserted Async job.')
def test_init_opts_supersede_decorated_options(self): """Ensure options passed to init override decorated options.""" from furious. async import Async from furious. async import defaults options = { 'value': 1, 'other': 'zzz', 'nested': { 1: 1 }, 'id': 'wrong', 'context_id': None, 'parent_id': 'parentid' } @defaults(**options.copy()) def some_function(): pass job = Async(some_function, value=17, other='abc', id='correct') options['value'] = 17 options['other'] = 'abc' options['id'] = 'correct' options['job'] = ("furious.tests.test_async.some_function", None, None) options['_recursion'] = {'current': 0, 'max': 100} self.assertEqual(options, job._options)
def test_calls_error_callback(self): """Ensure run_job catches any exceptions raised by the job, then calls the error callback. """ from furious.async import Async from furious.context._execution import _ExecutionContext from furious.processors import run_job call_count = [] handle_count = [] def handle_success(): call_count.append(1) def handle_errors(): handle_count.append(1) work = Async(target=dir, args=[1, 2], callbacks={'success': handle_success, 'error': handle_errors}) with _ExecutionContext(work): run_job() self.assertEqual(1, len(handle_count), "Error handler called wrong number of times.") self.assertEqual(0, len(call_count), "Success handler unexpectedly called.")
def test_decorated_options(self): """Ensure the defaults decorator sets Async options.""" from furious. async import Async from furious. async import defaults options = { 'value': 1, 'other': 'zzz', 'nested': { 1: 1 }, 'id': 'thing', 'context_id': None, 'parent_id': 'parentid' } @defaults(**options.copy()) def some_function(): pass job = Async(some_function) options['job'] = ("furious.tests.test_async.some_function", None, None) options['_recursion'] = {'current': 0, 'max': 100} self.assertEqual(options, job._options)
def test_run(self, ctime): """Ensure tasks are run when run_queues is called.""" from furious.async import Async from furious.test_stubs.appengine.queues import run as run_queues # Enqueue a couple of tasks async = Async(target='time.ctime') async.start() async2 = Async(target='time.ctime') async2.start() # Run the tasks in the queue run_queues(self.taskqueue_service) self.assertEqual(2, ctime.call_count)