def test_nested_context_works(self, queue_add_mock):
"""Ensure adding a job works."""
from furious. async import Async
from furious.context import Context
with Context() as ctx:
job = ctx.add('test', args=[1, 2])
with Context() as ctx2:
job2 = ctx2.add('test', args=[1, 2])
self.assertIsInstance(job, Async)
self.assertIsInstance(job2, Async)
self.assertEqual(2, queue_add_mock.call_count)
def test_marker_not_complete_when_start_fails(self, mock_insert,
context_from_id,
check_markers):
"""Ensure if the completion handler fails to start, that the marker
does not get marked as complete.
"""
complete_event = Mock()
context = Context(id="contextid",
callbacks={'complete': complete_event})
context_from_id.return_value = context
check_markers.return_value = True, False
async = Async('foo')
async .update_options(context_id='contextid')
FuriousCompletionMarker(id=async .context_id, complete=False).put()
# Simulate the task failing to start
mock_insert.side_effect = DeadlineExceededError()
self.assertRaises(DeadlineExceededError, _completion_checker,
async .id, async .context_id)
# Marker should not have been marked complete.
current_marker = FuriousCompletionMarker.get_by_id(async .context_id)
self.assertFalse(current_marker.complete)
def test_results_with_no_tasks_loaded(self, get_multi_async):
"""Ensure results loads the tasks and yields them out when no tasks are
cached.
"""
marker1 = _build_marker(payload="1", status=1)
marker2 = _build_marker(payload="2", status=1)
marker3 = _build_marker(payload="3", status=1)
future_set_1 = [
_build_future(marker1),
_build_future(marker2),
_build_future(marker3)
]
get_multi_async.return_value = future_set_1
context = Context(_task_ids=["1", "2", "3"])
context_result = ContextResult(context)
results = list(context_result.items())
results = sorted(results)
self.assertEqual(results, [("1", "1"), ("2", "2"), ("3", "3")])
self.assertEqual(context_result._task_cache, {
"1": marker1,
"2": marker2,
"3": marker3
})
def test_markers_and_context_complete(self, mark, context_from_id,
check_markers):
"""Ensure if all markers are complete that True is returned and
nothing else is done.
"""
async = Async('foo')
async .update_options(context_id='contextid')
complete_event = Mock()
context = Context(id="contextid",
callbacks={'complete': complete_event})
context_from_id.return_value = context
marker = FuriousCompletionMarker(id="contextid", complete=True)
marker.put()
check_markers.return_value = True, False
mark.return_value = True
result = _completion_checker(async .id, async .context_id)
self.assertTrue(result)
self.assertFalse(complete_event.start.called)
marker.key.delete()
def test_add_jobs_to_multiple_queues(self):
"""Ensure adding jobs to multiple queues works as expected."""
from google.appengine.api.taskqueue import Queue
from furious.context import Context
queue_registry = {}
class AwesomeQueue(Queue):
def __init__(self, *args, **kwargs):
super(AwesomeQueue, self).__init__(*args, **kwargs)
queue_registry[kwargs.get('name')] = self
self._calls = []
def add(self, *args, **kwargs):
self._calls.append((args, kwargs))
with patch('google.appengine.api.taskqueue.Queue', AwesomeQueue):
with Context() as ctx:
ctx.add('test', args=[1, 2], queue='A')
ctx.add('test', args=[1, 2], queue='A')
ctx.add('test', args=[1, 2], queue='B')
ctx.add('test', args=[1, 2], queue='C')
self.assertEqual(2, len(queue_registry['A']._calls[0][0][0]))
self.assertEqual(1, len(queue_registry['B']._calls[0][0][0]))
self.assertEqual(1, len(queue_registry['C']._calls[0][0][0]))
def test_has_no_marker(self):
"""Ensure returns False when no marker found."""
context_id = 1
context = Context(id=context_id)
context_result = ContextResult(context)
self.assertIsNone(context_result._marker)
self.assertFalse(context_result.has_errors())
def test_added_to_correct_queue(self, queue_mock):
"""Ensure jobs are added to the correct queue."""
from furious.context import Context
with Context() as ctx:
ctx.add('test', args=[1, 2], queue='A')
ctx.add('test', args=[1, 2], queue='A')
queue_mock.assert_called_once_with(name='A')
def test_context_gets_one_id(self):
"""Ensure a new Context gets an id only generated once."""
from furious.context import Context
context = Context()
id1 = context.id
id2 = context.id
self.assertEqual(id1, id2)
self.assertEqual(context.id, id1)
def test_add_multiple_jobs_to_context_works(self, queue_add_mock):
"""Ensure adding multiple jobs works."""
from furious.context import Context
with Context() as ctx:
for _ in range(10):
ctx.add('test', args=[1, 2])
queue_add_mock.assert_called_once()
self.assertEqual(10, len(queue_add_mock.call_args[0][0]))
def test_no_task_ids(self, get_multi_async):
"""Ensure no results are yielded out when there are no task ids on the
passed in context.
"""
get_multi_async.return_value = []
context = Context(_task_ids=[])
results = list(iter_context_results(context))
self.assertEqual(results, [])
def test_add_job_to_context_works(self, queue_add_mock):
"""Ensure adding a job works."""
from furious. async import Async
from furious.context import Context
with Context() as ctx:
job = ctx.add('test', args=[1, 2])
self.assertIsInstance(job, Async)
queue_add_mock.assert_called_once()
def test_add_job_to_context_works(self, queue_add_mock):
"""Ensure adding a job works."""
from furious. async import Async
from furious.context import Context
with Context() as ctx:
job = ctx.add('test', args=[1, 2])
self.assertIsInstance(job, Async)
self.assertEqual(1, ctx.insert_success)
self.assertEqual(1, queue_add_mock.call_count)
def test_id_added_to_options(self, uuid_patch):
"""Ensure random context id gets added to options."""
from furious.context import Context
id = 'random-id'
uuid_patch.return_value.hex = id
context = Context()
self.assertEqual(context.id, id)
self.assertEqual(context._options['id'], id)
def test_to_dict_with_callbacks(self):
"""Ensure to_dict correctly encodes callbacks."""
import copy
from furious. async import Async
from furious.context import Context
options = {
'id': 'someid',
'context_id': 'contextid',
'parent_id': 'parentid',
'persistence_engine': 'persistence_engine',
'callbacks': {
'success':
self.__class__.test_to_dict_with_callbacks,
'failure':
"failure_function",
'exec':
Async(target=dir,
id='blargh',
context_id='contextid',
parent_id='parentid')
}
}
context = Context(**copy.deepcopy(options))
# This stuff gets dumped out by to_dict().
options.update({
'id': 'someid',
'insert_tasks': 'furious.context.context._insert_tasks',
'persistence_engine': 'persistence_engine',
'_tasks_inserted': False,
'_task_ids': [],
'callbacks': {
'success': ("furious.tests.context.test_context."
"TestContext.test_to_dict_with_callbacks"),
'failure':
"failure_function",
'exec': {
'job': ('dir', None, None),
'id': 'blargh',
'context_id': 'contextid',
'parent_id': 'parentid',
'_recursion': {
'current': 0,
'max': 100
},
'_type': 'furious.async.Async'
}
}
})
self.assertEqual(options, context.to_dict())
def test_has_errors_with_marker_not_cached(self):
"""Ensure returns the value from the marker when not cached."""
context_id = 1
FuriousCompletionMarker(id=context_id, has_errors=True).put()
context = Context(id=context_id)
context_result = ContextResult(context)
self.assertIsNone(context_result._marker)
self.assertTrue(context_result.has_errors())
context_result._marker.key.delete()
def test_added_asyncs_get_context_id(self, queue_mock):
"""Ensure Asyncs added to context get context id."""
from furious. async import Async
from furious.context import Context
asyncs = [Async('test', id=i) for i in xrange(100, 110)]
with Context() as ctx:
for async in asyncs:
ctx.add(async)
self.assertEqual(ctx.id, async .get_options()['_context_id'])
self.assertEqual(10, ctx.insert_success)
def test_persist_persists(self):
"""Calling persist with an engine persists the Context."""
from furious.context import Context
persistence_engine = Mock()
persistence_engine.func_name = 'persistence_engine'
persistence_engine.im_class.__name__ = 'engine'
context = Context(persistence_engine=persistence_engine)
context.persist()
persistence_engine.store_context.assert_called_once_with(context)
def test_save_context(self):
"""Ensure the passed in context gets serialized and set on the saved
FuriousContext entity.
"""
_id = "contextid"
context = Context(id=_id)
result = store_context(context)
self.assertEqual(result.id(), _id)
loaded_context = FuriousContext.from_id(result.id())
self.assertEqual(context.to_dict(), loaded_context.to_dict())
def test_keys_with_no_results(self, get_multi_async):
"""Ensure empty results are yielded out when there are no items to
load but task ids are on the passed in context.
"""
future_set_1 = [_build_future(), _build_future(), _build_future()]
get_multi_async.return_value = future_set_1
context = Context(_task_ids=["1", "2", "3"])
results = list(iter_context_results(context))
self.assertEqual(results[0], ("1", None))
self.assertEqual(results[1], ("2", None))
self.assertEqual(results[2], ("3", None))
def test_add_task_fails(self, queue_add_mock):
"""Ensure insert_failed and insert_success are calculated correctly."""
from google.appengine.api.taskqueue import TaskAlreadyExistsError
from furious.context import Context
def queue_add(tasks, transactional=False):
if len(tasks) != 2:
raise TaskAlreadyExistsError()
queue_add_mock.side_effect = queue_add
with Context() as ctx:
ctx.add('test', args=[1, 2], queue='A')
ctx.add('test', args=[1, 2], queue='B')
ctx.add('test', args=[1, 2], queue='B')
self.assertEqual(2, ctx.insert_success)
self.assertEqual(1, ctx.insert_failed)
def test_results_with_tasks_loaded_missing_result(self, get_multi_async):
"""Ensure results uses the cached tasks and yields them out when tasks
are cached and there's no results.
"""
marker1 = FuriousAsyncMarker()
context = Context(_task_ids=["1", "2", "3"])
context_result = ContextResult(context)
context_result._task_cache = {"1": marker1, "2": None, "3": None}
results = list(context_result.items())
results = sorted(results)
self.assertEqual(results, [("1", None), ("2", None), ("3", None)])
self.assertFalse(get_multi_async.called)
def test_more_results_than_batch_size(self, get_multi_async):
"""Ensure all the results are yielded out when more than the batch
size.
"""
marker1 = _build_marker(payload="1", status=1)
marker2 = _build_marker(payload="2", status=1)
marker3 = _build_marker(payload="3", status=1)
future_set_1 = [_build_future(marker1), _build_future(marker2)]
future_set_2 = [_build_future(marker3)]
get_multi_async.side_effect = future_set_1, future_set_2
context = Context(_task_ids=["1", "2", "3"])
results = list(iter_context_results(context, batch_size=2))
self.assertEqual(results[0], ("1", marker1))
self.assertEqual(results[1], ("2", marker2))
self.assertEqual(results[2], ("3", marker3))
def test_results_with_tasks_loaded(self, get_multi_async):
"""Ensure results uses the cached tasks and yields them out when tasks
are cached.
"""
marker1 = _build_marker(payload="1", status=1)
marker2 = _build_marker(payload="2", status=1)
marker3 = _build_marker(payload="3", status=1)
context = Context(_task_ids=["1", "2", "3"])
context_result = ContextResult(context)
context_result._task_cache = {"1": marker1, "2": marker2, "3": marker3}
results = list(context_result.items())
results = sorted(results)
self.assertEqual(results, [("1", "1"), ("2", "2"), ("3", "3")])
self.assertFalse(get_multi_async.called)
def test_markers_complete(self, context_from_id, check_markers):
"""Ensure if all markers are complete that True is returned and the
completion handler and cleanup tasks are triggered.
"""
complete_event = Mock()
context = Context(id="contextid",
callbacks={'complete': complete_event})
context_from_id.return_value = context
check_markers.return_value = True, False
async = Async('foo')
async .update_options(context_id='contextid')
FuriousCompletionMarker(id=async .context_id, complete=False).put()
result = _completion_checker(async .id, async .context_id)
self.assertTrue(result)
complete_event.start.assert_called_once_with(transactional=True)
def test_to_dict(self):
"""Ensure to_dict returns a dictionary representation of the Context.
"""
import copy
from furious.context import Context
options = {
'persistence_engine': 'persistence_engine',
'unkown': True,
}
context = Context(**copy.deepcopy(options))
# This stuff gets dumped out by to_dict().
options.update({
'insert_tasks': 'furious.context.context._insert_tasks',
'_tasks_inserted': False,
'_task_ids': [],
})
self.assertEqual(options, context.to_dict())
def test_markers_not_complete(self, context_from_id, check_markers):
"""Ensure if not all markers are complete that False is returned and
the completion handler and cleanup tasks are not triggered.
"""
complete_event = Mock()
context = Context(id="contextid",
callbacks={'complete': complete_event})
context_from_id.return_value = context
check_markers.return_value = False, False
async = Async('foo')
async .update_options(context_id='contextid')
result = _completion_checker(async .id, async .context_id)
self.assertFalse(result)
self.assertTrue(context_from_id.called)
self.assertFalse(complete_event.start.called)
def test_context_gets_assigned_id(self):
"""Ensure a new Context keeps its assigned id."""
from furious.context import Context
self.assertEqual('test_id_weee', Context(id='test_id_weee').id)
def test_context_gets_id(self):
"""Ensure a new Context gets an id generated."""
from furious.context import Context
self.assertTrue(Context().id)
def test_context_works(self):
"""Ensure using a Context as a context manager works."""
from furious.context import Context
with Context():
pass
def test_persist_with_no_engine(self):
"""Calling persist with no engine should blow up."""
from furious.context import Context
context = Context()
self.assertRaises(RuntimeError, context.persist)
