def test_mget(self):
tb = CacheBackend(backend="memory://")
tb.set("foo", 1)
tb.set("bar", 2)
self.assertDictEqual(tb.mget(["foo", "bar"]),
{"foo": 1, "bar": 2})
def test_forget(self):
tb = CacheBackend(backend="memory://")
tid = gen_unique_id()
tb.mark_as_done(tid, {"foo": "bar"})
x = AsyncResult(tid, backend=tb)
x.forget()
self.assertIsNone(x.result)
def test_forget(self):
tb = CacheBackend(backend="memory://")
tid = gen_unique_id()
tb.mark_as_done(tid, {"foo": "bar"})
x = AsyncResult(tid, backend=tb)
x.forget()
self.assertIsNone(x.result)
def test_mget(self):
tb = CacheBackend(backend="memory://")
tb.set("foo", 1)
tb.set("bar", 2)
self.assertDictEqual(tb.mget(["foo", "bar"]),
{"foo": 1, "bar": 2})
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
tb.apply_chord(result, None)
assert self.app.GroupResult.restore(result.id, backend=tb) == result
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
tb.apply_chord(result, None)
assert self.app.GroupResult.restore(result.id, backend=tb) == result
def test_is_pickled(self):
tb = CacheBackend(backend="memory://")
tid2 = gen_unique_id()
result = {"foo": "baz", "bar": SomeClass(12345)}
tb.mark_as_done(tid2, result)
# is serialized properly.
rindb = tb.get_result(tid2)
self.assertEqual(rindb.get("foo"), "baz")
self.assertEqual(rindb.get("bar").data, 12345)
def test_is_pickled(self):
tb = CacheBackend(backend="memory://")
tid2 = gen_unique_id()
result = {"foo": "baz", "bar": SomeClass(12345)}
tb.mark_as_done(tid2, result)
# is serialized properly.
rindb = tb.get_result(tid2)
self.assertEqual(rindb.get("foo"), "baz")
self.assertEqual(rindb.get("bar").data, 12345)
def test_mark_as_failure(self):
tb = CacheBackend(backend="memory://")
tid3 = uuid()
try:
raise KeyError("foo")
except KeyError, exception:
pass
tb.mark_as_failure(tid3, exception)
self.assertEqual(tb.get_status(tid3), states.FAILURE)
self.assertIsInstance(tb.get_result(tid3), KeyError)
def test_on_chord_part_return(self, setresult):
tb = CacheBackend(backend='memory://')
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
setresult.restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
try:
current_app.tasks['foobarbaz'] = task
task.request.chord = subtask(task)
task.request.group = 'group_id'
tb.on_chord_apply(task.request.group, [])
self.assertFalse(deps.join.called)
tb.on_chord_part_return(task)
self.assertFalse(deps.join.called)
tb.on_chord_part_return(task)
deps.join.assert_called_with(propagate=False)
deps.delete.assert_called_with()
finally:
current_app.tasks.pop('foobarbaz')
def test_mark_as_done(self):
cb = CacheBackend()
tid = gen_unique_id()
self.assertFalse(cb.is_successful(tid))
self.assertEqual(cb.get_status(tid), states.PENDING)
self.assertIsNone(cb.get_result(tid))
cb.mark_as_done(tid, 42)
self.assertTrue(cb.is_successful(tid))
self.assertEqual(cb.get_status(tid), states.SUCCESS)
self.assertEqual(cb.get_result(tid), 42)
self.assertTrue(cb.get_result(tid), 42)
def test_on_chord_part_return(self, setresult):
tb = CacheBackend(backend="memory://")
deps = Mock()
deps.total = 2
setresult.restore.return_value = deps
task = Mock()
task.name = "foobarbaz"
try:
current_app.tasks["foobarbaz"] = task
task.request.chord = subtask(task)
task.request.taskset = "setid"
tb.on_chord_apply(task.request.taskset, [])
self.assertFalse(deps.join.called)
tb.on_chord_part_return(task)
self.assertFalse(deps.join.called)
tb.on_chord_part_return(task)
deps.join.assert_called_with(propagate=False)
deps.delete.assert_called_with()
finally:
current_app.tasks.pop("foobarbaz")
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
task.request.group = result.id
tb.apply_chord(result, None)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
def test_on_chord_part_return(self, setresult):
tb = CacheBackend(backend='memory://')
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
setresult.restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
try:
current_app.tasks['foobarbaz'] = task
task.request.chord = subtask(task)
gid, res = uuid(), [AsyncResult(uuid()) for _ in range(3)]
task.request.group = gid
tb.on_chord_apply(gid, {}, result=res)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task)
deps.join_native.assert_called_with(propagate=True)
deps.delete.assert_called_with()
finally:
current_app.tasks.pop('foobarbaz')
def test_mark_as_done(self):
cb = CacheBackend()
tid = gen_unique_id()
self.assertFalse(cb.is_successful(tid))
self.assertEquals(cb.get_status(tid), "PENDING")
self.assertEquals(cb.get_result(tid), None)
cb.mark_as_done(tid, 42)
self.assertTrue(cb.is_successful(tid))
self.assertEquals(cb.get_status(tid), "SUCCESS")
self.assertEquals(cb.get_result(tid), 42)
self.assertTrue(cb._cache.get(tid))
self.assertTrue(cb.get_result(tid), 42)
def test_deduplicate_successful_tasks__result_not_found(self):
@self.app.task(shared=False)
def add(x, y):
return x + y
backend = CacheBackend(app=self.app, backend='memory')
add.backend = backend
add.store_eager_result = True
add.ignore_result = False
add.acks_late = True
self.app.conf.worker_deduplicate_successful_tasks = True
task_id = str(uuid4())
request = {'id': task_id, 'delivery_info': {'redelivered': True}}
with patch('celery.app.trace.AsyncResult') as async_result_mock:
assert trace(self.app,
add, (1, 1),
task_id=task_id,
request=request) == (2, None)
state_property = PropertyMock(side_effect=BackendGetMetaError)
type(async_result_mock()).state = state_property
assert trace(self.app,
add, (1, 1),
task_id=task_id,
request=request) == (2, None)
self.app.conf.worker_deduplicate_successful_tasks = False
def test_deduplicate_successful_tasks__no_deduplication(self):
@self.app.task(shared=False)
def add(x, y):
return x + y
backend = CacheBackend(app=self.app, backend='memory')
add.backend = backend
add.store_eager_result = True
add.ignore_result = False
add.acks_late = True
self.app.conf.worker_deduplicate_successful_tasks = True
task_id = str(uuid4())
request = {'id': task_id, 'delivery_info': {'redelivered': True}}
with patch('celery.app.trace.AsyncResult') as async_result_mock:
async_result_mock().state.return_value = PENDING
assert trace(self.app,
add, (1, 1),
task_id=task_id,
request=request) == (2, None)
assert trace(self.app,
add, (1, 1),
task_id=task_id,
request=request) == (2, None)
self.app.conf.worker_deduplicate_successful_tasks = False
def test_memory_client_is_shared(self):
"""This test verifies that memory:// backend state is shared over multiple threads"""
from threading import Thread
t = Thread(target=lambda: CacheBackend(
backend='memory://', app=self.app).set('test', 12345))
t.start()
t.join()
assert self.tb.client.get('test') == 12345
def test_mark_as_failure(self):
tb = CacheBackend(backend="memory://")
tid3 = gen_unique_id()
try:
raise KeyError("foo")
except KeyError, exception:
pass
def test_on_chord_part_return(self, setresult):
tb = CacheBackend(backend="memory://")
deps = Mock()
deps.total = 2
setresult.restore.return_value = deps
task = Mock()
task.name = "foobarbaz"
try:
current_app.tasks["foobarbaz"] = task
task.request.chord = subtask(task)
task.request.taskset = "setid"
tb.on_chord_apply(task.request.taskset, [])
self.assertFalse(deps.join.called)
tb.on_chord_part_return(task)
self.assertFalse(deps.join.called)
tb.on_chord_part_return(task)
deps.join.assert_called_with(propagate=False)
deps.delete.assert_called_with()
finally:
current_app.tasks.pop("foobarbaz")
def test_on_chord_part_return(self, setresult):
tb = CacheBackend(backend='memory://')
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
setresult.restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
try:
current_app.tasks['foobarbaz'] = task
task.request.chord = subtask(task)
task.request.group = 'group_id'
tb.on_chord_apply(task.request.group, [])
self.assertFalse(deps.join.called)
tb.on_chord_part_return(task)
self.assertFalse(deps.join.called)
tb.on_chord_part_return(task)
deps.join.assert_called_with(propagate=False)
deps.delete.assert_called_with()
finally:
current_app.tasks.pop('foobarbaz')
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
task.request.group = result.id
tb.apply_chord(result, None)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
def test_on_chord_part_return(self, setresult):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
setresult.restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
try:
self.app.tasks['foobarbaz'] = task
task.request.chord = subtask(task)
gid, res = uuid(), [AsyncResult(uuid()) for _ in range(3)]
task.request.group = gid
tb.on_chord_apply(gid, {}, result=res)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task)
deps.join_native.assert_called_with(propagate=True)
deps.delete.assert_called_with()
finally:
self.app.tasks.pop('foobarbaz')
def test_no_backend(self):
prev, self.app.conf.CELERY_CACHE_BACKEND = (
self.app.conf.CELERY_CACHE_BACKEND, None,
)
try:
with self.assertRaises(ImproperlyConfigured):
CacheBackend(backend=None, app=self.app)
finally:
self.app.conf.CELERY_CACHE_BACKEND = prev
def test_mark_as_done(self):
tb = CacheBackend(backend="memory://")
tid = gen_unique_id()
self.assertEqual(tb.get_status(tid), states.PENDING)
self.assertIsNone(tb.get_result(tid))
tb.mark_as_done(tid, 42)
self.assertEqual(tb.get_status(tid), states.SUCCESS)
self.assertEqual(tb.get_result(tid), 42)
def test_mark_as_done(self):
tb = CacheBackend(backend="memory://")
tid = gen_unique_id()
self.assertEqual(tb.get_status(tid), states.PENDING)
self.assertIsNone(tb.get_result(tid))
tb.mark_as_done(tid, 42)
self.assertEqual(tb.get_status(tid), states.SUCCESS)
self.assertEqual(tb.get_result(tid), 42)
def test_mark_as_failure(self):
tb = CacheBackend(backend="memory://")
tid3 = uuid()
try:
raise KeyError("foo")
except KeyError, exception:
pass
tb.mark_as_failure(tid3, exception)
self.assertEqual(tb.get_status(tid3), states.FAILURE)
self.assertIsInstance(tb.get_result(tid3), KeyError)
def test_deduplicate_successful_tasks__deduplication(self):
@self.app.task(shared=False)
def add(x, y):
return x + y
backend = CacheBackend(app=self.app, backend='memory')
add.backend = backend
add.store_eager_result = True
add.ignore_result = False
add.acks_late = True
self.app.conf.worker_deduplicate_successful_tasks = True
task_id = str(uuid4())
request = {'id': task_id, 'delivery_info': {'redelivered': True}}
assert trace(self.app, add, (1, 1), task_id=task_id,
request=request) == (2, None)
assert trace(self.app, add, (1, 1), task_id=task_id,
request=request) == (None, None)
self.app.conf.worker_deduplicate_successful_tasks = False
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
task.request.group = gid
tb.on_chord_apply(gid, {}, result=res)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task)
deps.join_native.assert_called_with(propagate=True)
deps.delete.assert_called_with()
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
task.request.group = gid
tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task)
deps.join_native.assert_called_with(propagate=True)
deps.delete.assert_called_with()
def test_process_cleanup(self):
tb = CacheBackend(backend="memory://")
tb.process_cleanup()
class test_CacheBackend:
def setup(self):
self.app.conf.result_serializer = 'pickle'
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
self.old_get_best_memcached = backends['memcache']
backends['memcache'] = lambda: (DummyClient, ensure_bytes)
def teardown(self):
backends['memcache'] = self.old_get_best_memcached
def test_no_backend(self):
self.app.conf.cache_backend = None
with pytest.raises(ImproperlyConfigured):
CacheBackend(backend=None, app=self.app)
def test_mark_as_done(self):
assert self.tb.get_state(self.tid) == states.PENDING
assert self.tb.get_result(self.tid) is None
self.tb.mark_as_done(self.tid, 42)
assert self.tb.get_state(self.tid) == states.SUCCESS
assert self.tb.get_result(self.tid) == 42
def test_is_pickled(self):
result = {'foo': 'baz', 'bar': SomeClass(12345)}
self.tb.mark_as_done(self.tid, result)
# is serialized properly.
rindb = self.tb.get_result(self.tid)
assert rindb.get('foo') == 'baz'
assert rindb.get('bar').data == 12345
def test_mark_as_failure(self):
try:
raise KeyError('foo')
except KeyError as exception:
self.tb.mark_as_failure(self.tid, exception)
assert self.tb.get_state(self.tid) == states.FAILURE
assert isinstance(self.tb.get_result(self.tid), KeyError)
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
tb.apply_chord(result, None)
assert self.app.GroupResult.restore(result.id, backend=tb) == result
@patch('celery.result.GroupResult.restore')
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
task.request.group = result.id
tb.apply_chord(result, None)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
def test_mget(self):
self.tb.set('foo', 1)
self.tb.set('bar', 2)
assert self.tb.mget(['foo', 'bar']) == {'foo': 1, 'bar': 2}
def test_forget(self):
self.tb.mark_as_done(self.tid, {'foo': 'bar'})
x = self.app.AsyncResult(self.tid, backend=self.tb)
x.forget()
assert x.result is None
def test_process_cleanup(self):
self.tb.process_cleanup()
def test_expires_as_int(self):
tb = CacheBackend(backend='memory://', expires=10, app=self.app)
assert tb.expires == 10
def test_unknown_backend_raises_ImproperlyConfigured(self):
with pytest.raises(ImproperlyConfigured):
CacheBackend(backend='unknown://', app=self.app)
def test_as_uri_no_servers(self):
assert self.tb.as_uri() == 'memory:///'
def test_as_uri_one_server(self):
backend = 'memcache://127.0.0.1:11211/'
b = CacheBackend(backend=backend, app=self.app)
assert b.as_uri() == backend
def test_as_uri_multiple_servers(self):
backend = 'memcache://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/'
b = CacheBackend(backend=backend, app=self.app)
assert b.as_uri() == backend
@skip.unless_module('memcached', name='python-memcached')
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/'
)
worker = self.app.Worker()
with mock.stdouts():
worker.on_start()
assert worker.startup_info()
def setUp(self):
self.tb = CacheBackend(backend="memory://")
self.tid = uuid()
def test_no_backend(self):
self.app.conf.cache_backend = None
with pytest.raises(ImproperlyConfigured):
CacheBackend(backend=None, app=self.app)
class test_CacheBackend:
def setup(self):
self.app.conf.result_serializer = 'pickle'
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
self.old_get_best_memcached = backends['memcache']
backends['memcache'] = lambda: (DummyClient, ensure_bytes)
def teardown(self):
backends['memcache'] = self.old_get_best_memcached
def test_no_backend(self):
self.app.conf.cache_backend = None
with pytest.raises(ImproperlyConfigured):
CacheBackend(backend=None, app=self.app)
def test_mark_as_done(self):
assert self.tb.get_state(self.tid) == states.PENDING
assert self.tb.get_result(self.tid) is None
self.tb.mark_as_done(self.tid, 42)
assert self.tb.get_state(self.tid) == states.SUCCESS
assert self.tb.get_result(self.tid) == 42
def test_is_pickled(self):
result = {'foo': 'baz', 'bar': SomeClass(12345)}
self.tb.mark_as_done(self.tid, result)
# is serialized properly.
rindb = self.tb.get_result(self.tid)
assert rindb.get('foo') == 'baz'
assert rindb.get('bar').data == 12345
def test_mark_as_failure(self):
try:
raise KeyError('foo')
except KeyError as exception:
self.tb.mark_as_failure(self.tid, exception)
assert self.tb.get_state(self.tid) == states.FAILURE
assert isinstance(self.tb.get_result(self.tid), KeyError)
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
tb.apply_chord(result, None)
assert self.app.GroupResult.restore(result.id, backend=tb) == result
@patch('celery.result.GroupResult.restore')
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
result = self.app.GroupResult(
uuid(),
[self.app.AsyncResult(uuid()) for _ in range(3)],
)
task.request.group = result.id
tb.apply_chord(result, None)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
def test_mget(self):
self.tb._set_with_state('foo', 1, states.SUCCESS)
self.tb._set_with_state('bar', 2, states.SUCCESS)
assert self.tb.mget(['foo', 'bar']) == {'foo': 1, 'bar': 2}
def test_forget(self):
self.tb.mark_as_done(self.tid, {'foo': 'bar'})
x = self.app.AsyncResult(self.tid, backend=self.tb)
x.forget()
assert x.result is None
def test_process_cleanup(self):
self.tb.process_cleanup()
def test_expires_as_int(self):
tb = CacheBackend(backend='memory://', expires=10, app=self.app)
assert tb.expires == 10
def test_unknown_backend_raises_ImproperlyConfigured(self):
with pytest.raises(ImproperlyConfigured):
CacheBackend(backend='unknown://', app=self.app)
def test_as_uri_no_servers(self):
assert self.tb.as_uri() == 'memory:///'
def test_as_uri_one_server(self):
backend = 'memcache://127.0.0.1:11211/'
b = CacheBackend(backend=backend, app=self.app)
assert b.as_uri() == backend
def test_as_uri_multiple_servers(self):
backend = 'memcache://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/'
b = CacheBackend(backend=backend, app=self.app)
assert b.as_uri() == backend
@skip.unless_module('memcached', name='python-memcached')
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/')
worker = self.app.Worker()
with mock.stdouts():
worker.on_start()
assert worker.startup_info()
def test_as_uri_one_server(self):
backend = 'memcache://127.0.0.1:11211/'
b = CacheBackend(backend=backend, app=self.app)
assert b.as_uri() == backend
def test_expires_as_int(self):
tb = CacheBackend(backend='memory://', expires=10, app=self.app)
assert tb.expires == 10
def setup(self):
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
class test_CacheBackend(AppCase):
def setup(self):
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
def test_no_backend(self):
self.app.conf.CELERY_CACHE_BACKEND = None
with self.assertRaises(ImproperlyConfigured):
CacheBackend(backend=None, app=self.app)
def test_mark_as_done(self):
self.assertEqual(self.tb.get_status(self.tid), states.PENDING)
self.assertIsNone(self.tb.get_result(self.tid))
self.tb.mark_as_done(self.tid, 42)
self.assertEqual(self.tb.get_status(self.tid), states.SUCCESS)
self.assertEqual(self.tb.get_result(self.tid), 42)
def test_is_pickled(self):
result = {'foo': 'baz', 'bar': SomeClass(12345)}
self.tb.mark_as_done(self.tid, result)
# is serialized properly.
rindb = self.tb.get_result(self.tid)
self.assertEqual(rindb.get('foo'), 'baz')
self.assertEqual(rindb.get('bar').data, 12345)
def test_mark_as_failure(self):
try:
raise KeyError('foo')
except KeyError as exception:
self.tb.mark_as_failure(self.tid, exception)
self.assertEqual(self.tb.get_status(self.tid), states.FAILURE)
self.assertIsInstance(self.tb.get_result(self.tid), KeyError)
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
@patch('celery.result.GroupResult.restore')
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
task.request.group = gid
tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task, 'SUCCESS', 10)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task, 'SUCCESS', 10)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
def test_mget(self):
self.tb.set('foo', 1)
self.tb.set('bar', 2)
self.assertDictEqual(self.tb.mget(['foo', 'bar']),
{'foo': 1, 'bar': 2})
def test_forget(self):
self.tb.mark_as_done(self.tid, {'foo': 'bar'})
x = self.app.AsyncResult(self.tid, backend=self.tb)
x.forget()
self.assertIsNone(x.result)
def test_process_cleanup(self):
self.tb.process_cleanup()
def test_expires_as_int(self):
tb = CacheBackend(backend='memory://', expires=10, app=self.app)
self.assertEqual(tb.expires, 10)
def test_unknown_backend_raises_ImproperlyConfigured(self):
with self.assertRaises(ImproperlyConfigured):
CacheBackend(backend='unknown://', app=self.app)
def test_process_cleanup(self):
tb = CacheBackend(backend="memory://")
tb.process_cleanup()
def setup(self):
self.app.conf.result_serializer = 'pickle'
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
def test_process_cleanup(self):
cb = CacheBackend()
cb.process_cleanup()
def setup(self):
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
self.old_get_best_memcached = backends['memcache']
backends['memcache'] = lambda: (DummyClient, ensure_bytes)
def test_no_backend(self):
self.app.conf.CELERY_CACHE_BACKEND = None
with self.assertRaises(ImproperlyConfigured):
CacheBackend(backend=None, app=self.app)
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
def test_expires_as_int(self):
tb = CacheBackend(backend='memory://', expires=10, app=self.app)
self.assertEqual(tb.expires, 10)
def test_unknown_backend_raises_ImproperlyConfigured(self):
with pytest.raises(ImproperlyConfigured):
CacheBackend(backend='unknown://', app=self.app)
class test_CacheBackend(AppCase):
def setup(self):
self.app.conf.result_serializer = 'pickle'
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
self.old_get_best_memcached = backends['memcache']
backends['memcache'] = lambda: (DummyClient, ensure_bytes)
def teardown(self):
backends['memcache'] = self.old_get_best_memcached
def test_no_backend(self):
self.app.conf.cache_backend = None
with self.assertRaises(ImproperlyConfigured):
CacheBackend(backend=None, app=self.app)
def test_mark_as_done(self):
self.assertEqual(self.tb.get_state(self.tid), states.PENDING)
self.assertIsNone(self.tb.get_result(self.tid))
self.tb.mark_as_done(self.tid, 42)
self.assertEqual(self.tb.get_state(self.tid), states.SUCCESS)
self.assertEqual(self.tb.get_result(self.tid), 42)
def test_is_pickled(self):
result = {'foo': 'baz', 'bar': SomeClass(12345)}
self.tb.mark_as_done(self.tid, result)
# is serialized properly.
rindb = self.tb.get_result(self.tid)
self.assertEqual(rindb.get('foo'), 'baz')
self.assertEqual(rindb.get('bar').data, 12345)
def test_mark_as_failure(self):
try:
raise KeyError('foo')
except KeyError as exception:
self.tb.mark_as_failure(self.tid, exception)
self.assertEqual(self.tb.get_state(self.tid), states.FAILURE)
self.assertIsInstance(self.tb.get_result(self.tid), KeyError)
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
@patch('celery.result.GroupResult.restore')
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
task.request.group = gid
tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_not_called()
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
def test_mget(self):
self.tb.set('foo', 1)
self.tb.set('bar', 2)
self.assertDictEqual(self.tb.mget(['foo', 'bar']), {
'foo': 1,
'bar': 2
})
def test_forget(self):
self.tb.mark_as_done(self.tid, {'foo': 'bar'})
x = self.app.AsyncResult(self.tid, backend=self.tb)
x.forget()
self.assertIsNone(x.result)
def test_process_cleanup(self):
self.tb.process_cleanup()
def test_expires_as_int(self):
tb = CacheBackend(backend='memory://', expires=10, app=self.app)
self.assertEqual(tb.expires, 10)
def test_unknown_backend_raises_ImproperlyConfigured(self):
with self.assertRaises(ImproperlyConfigured):
CacheBackend(backend='unknown://', app=self.app)
def test_as_uri_no_servers(self):
self.assertEqual(self.tb.as_uri(), 'memory:///')
def test_as_uri_one_server(self):
backend = 'memcache://127.0.0.1:11211/'
b = CacheBackend(backend=backend, app=self.app)
self.assertEqual(b.as_uri(), backend)
def test_as_uri_multiple_servers(self):
backend = 'memcache://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/'
b = CacheBackend(backend=backend, app=self.app)
self.assertEqual(b.as_uri(), backend)
@mock.stdouts
@skip.unless_module('memcached', name='python-memcached')
def test_regression_worker_startup_info(self, stdout, stderr):
self.app.conf.result_backend = (
'cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/')
worker = self.app.Worker()
worker.on_start()
self.assertTrue(worker.startup_info())
def test_as_uri_multiple_servers(self):
backend = 'memcache://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/'
b = CacheBackend(backend=backend, app=self.app)
assert b.as_uri() == backend
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
def setup(self):
self.app.conf.result_serializer = 'pickle'
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
self.old_get_best_memcached = backends['memcache']
backends['memcache'] = lambda: (DummyClient, ensure_bytes)
def test_as_uri_one_server(self):
backend = 'memcache://127.0.0.1:11211/'
b = CacheBackend(backend=backend, app=self.app)
self.assertEqual(b.as_uri(), backend)
def test_on_chord_apply(self):
tb = CacheBackend(backend="memory://")
tb.on_chord_apply("setid", [])
def test_as_uri_multiple_servers(self):
backend = 'memcache://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/'
b = CacheBackend(backend=backend, app=self.app)
self.assertEqual(b.as_uri(), backend)
class test_CacheBackend(Case):
def setUp(self):
self.tb = CacheBackend(backend="memory://")
self.tid = uuid()
def test_mark_as_done(self):
self.assertEqual(self.tb.get_status(self.tid), states.PENDING)
self.assertIsNone(self.tb.get_result(self.tid))
self.tb.mark_as_done(self.tid, 42)
self.assertEqual(self.tb.get_status(self.tid), states.SUCCESS)
self.assertEqual(self.tb.get_result(self.tid), 42)
def test_is_pickled(self):
result = {"foo": "baz", "bar": SomeClass(12345)}
self.tb.mark_as_done(self.tid, result)
# is serialized properly.
rindb = self.tb.get_result(self.tid)
self.assertEqual(rindb.get("foo"), "baz")
self.assertEqual(rindb.get("bar").data, 12345)
def test_mark_as_failure(self):
try:
raise KeyError("foo")
except KeyError, exception:
self.tb.mark_as_failure(self.tid, exception)
self.assertEqual(self.tb.get_status(self.tid), states.FAILURE)
self.assertIsInstance(self.tb.get_result(self.tid), KeyError)
class test_CacheBackend(AppCase):
def setup(self):
self.app.conf.result_serializer = 'pickle'
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
self.old_get_best_memcached = backends['memcache']
backends['memcache'] = lambda: (DummyClient, ensure_bytes)
def teardown(self):
backends['memcache'] = self.old_get_best_memcached
def test_no_backend(self):
self.app.conf.cache_backend = None
with self.assertRaises(ImproperlyConfigured):
CacheBackend(backend=None, app=self.app)
def test_mark_as_done(self):
self.assertEqual(self.tb.get_state(self.tid), states.PENDING)
self.assertIsNone(self.tb.get_result(self.tid))
self.tb.mark_as_done(self.tid, 42)
self.assertEqual(self.tb.get_state(self.tid), states.SUCCESS)
self.assertEqual(self.tb.get_result(self.tid), 42)
def test_is_pickled(self):
result = {'foo': 'baz', 'bar': SomeClass(12345)}
self.tb.mark_as_done(self.tid, result)
# is serialized properly.
rindb = self.tb.get_result(self.tid)
self.assertEqual(rindb.get('foo'), 'baz')
self.assertEqual(rindb.get('bar').data, 12345)
def test_mark_as_failure(self):
try:
raise KeyError('foo')
except KeyError as exception:
self.tb.mark_as_failure(self.tid, exception)
self.assertEqual(self.tb.get_state(self.tid), states.FAILURE)
self.assertIsInstance(self.tb.get_result(self.tid), KeyError)
def test_apply_chord(self):
tb = CacheBackend(backend='memory://', app=self.app)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
@patch('celery.result.GroupResult.restore')
def test_on_chord_part_return(self, restore):
tb = CacheBackend(backend='memory://', app=self.app)
deps = Mock()
deps.__len__ = Mock()
deps.__len__.return_value = 2
restore.return_value = deps
task = Mock()
task.name = 'foobarbaz'
self.app.tasks['foobarbaz'] = task
task.request.chord = signature(task)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
task.request.group = gid
tb.apply_chord(group(app=self.app), (), gid, {}, result=res)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
self.assertFalse(deps.join_native.called)
tb.on_chord_part_return(task.request, 'SUCCESS', 10)
deps.join_native.assert_called_with(propagate=True, timeout=3.0)
deps.delete.assert_called_with()
def test_mget(self):
self.tb.set('foo', 1)
self.tb.set('bar', 2)
self.assertDictEqual(self.tb.mget(['foo', 'bar']),
{'foo': 1, 'bar': 2})
def test_forget(self):
self.tb.mark_as_done(self.tid, {'foo': 'bar'})
x = self.app.AsyncResult(self.tid, backend=self.tb)
x.forget()
self.assertIsNone(x.result)
def test_process_cleanup(self):
self.tb.process_cleanup()
def test_expires_as_int(self):
tb = CacheBackend(backend='memory://', expires=10, app=self.app)
self.assertEqual(tb.expires, 10)
def test_unknown_backend_raises_ImproperlyConfigured(self):
with self.assertRaises(ImproperlyConfigured):
CacheBackend(backend='unknown://', app=self.app)
def test_as_uri_no_servers(self):
self.assertEqual(self.tb.as_uri(), 'memory:///')
def test_as_uri_one_server(self):
backend = 'memcache://127.0.0.1:11211/'
b = CacheBackend(backend=backend, app=self.app)
self.assertEqual(b.as_uri(), backend)
def test_as_uri_multiple_servers(self):
backend = 'memcache://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/'
b = CacheBackend(backend=backend, app=self.app)
self.assertEqual(b.as_uri(), backend)
@disable_stdouts
def test_regression_worker_startup_info(self):
self.app.conf.result_backend = (
'cache+memcached://127.0.0.1:11211;127.0.0.2:11211;127.0.0.3/'
)
worker = self.app.Worker()
worker.on_start()
self.assertTrue(worker.startup_info())
def test_on_chord_apply(self):
tb = CacheBackend(backend='memory://')
tb.on_chord_apply('group_id', [])
def setup(self):
self.app.conf.result_serializer = 'pickle'
self.tb = CacheBackend(backend='memory://', app=self.app)
self.tid = uuid()
self.old_get_best_memcached = backends['memcache']
backends['memcache'] = lambda: (DummyClient, ensure_bytes)
def test_expires_as_int(self):
tb = CacheBackend(backend="memory://", expires=10)
self.assertEqual(tb.expires, 10)
def test_on_chord_apply(self):
tb = CacheBackend(backend='memory://', app=self.app)
gid, res = uuid(), [self.app.AsyncResult(uuid()) for _ in range(3)]
tb.on_chord_apply(gid, {}, result=res)
