def test_reduce(self):
a1 = self.app.AsyncResult('uuid')
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.id, 'uuid')
a2 = self.app.AsyncResult('uuid')
self.assertEqual(pickle.loads(pickle.dumps(a2)).id, 'uuid')
def _mget_to_results(self, values, keys):
if hasattr(values, "items"):
# client returns dict so mapping preserved.
return dict((self._strip_prefix(k), pickle.loads(str(v))) for k, v in values.iteritems() if v is not None)
else:
# client returns list so need to recreate mapping.
return dict((keys[i], pickle.loads(str(value))) for i, value in enumerate(values) if value is not None)
def test_reduce(self):
a1 = self.app.AsyncResult('uuid')
restored = pickle.loads(pickle.dumps(a1))
assert restored.id == 'uuid'
a2 = self.app.AsyncResult('uuid')
assert pickle.loads(pickle.dumps(a2)).id == 'uuid'
def test_reduce(self):
a1 = self.app.AsyncResult('uuid')
restored = pickle.loads(pickle.dumps(a1))
assert restored.id == 'uuid'
a2 = self.app.AsyncResult('uuid')
assert pickle.loads(pickle.dumps(a2)).id == 'uuid'
def test_reduce(self):
a1 = self.app.AsyncResult('uuid')
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.id, 'uuid')
a2 = self.app.AsyncResult('uuid')
self.assertEqual(pickle.loads(pickle.dumps(a2)).id, 'uuid')
def test_reduce(self):
a1 = AsyncResult("uuid", task_name="celery.ping")
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.task_id, "uuid")
self.assertEqual(restored.task_name, "celery.ping")
a2 = AsyncResult("uuid")
self.assertEqual(pickle.loads(pickle.dumps(a2)).task_id, "uuid")
def test_reduce(self):
a1 = AsyncResult("uuid", task_name=mytask.name)
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.task_id, "uuid")
self.assertEqual(restored.task_name, mytask.name)
a2 = AsyncResult("uuid")
self.assertEqual(pickle.loads(pickle.dumps(a2)).task_id, "uuid")
def test_reduce(self):
a1 = AsyncResult('uuid', task_name=mytask.name)
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.id, 'uuid')
self.assertEqual(restored.task_name, mytask.name)
a2 = AsyncResult('uuid')
self.assertEqual(pickle.loads(pickle.dumps(a2)).id, 'uuid')
def test_reduce(self):
a1 = self.app.AsyncResult("uuid", task_name=self.mytask.name)
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.id, "uuid")
self.assertEqual(restored.task_name, self.mytask.name)
a2 = self.app.AsyncResult("uuid")
self.assertEqual(pickle.loads(pickle.dumps(a2)).id, "uuid")
def test_reduce(self):
a1 = self.app.AsyncResult('uuid', task_name=self.mytask.name)
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.id, 'uuid')
self.assertEqual(restored.task_name, self.mytask.name)
a2 = self.app.AsyncResult('uuid')
self.assertEqual(pickle.loads(pickle.dumps(a2)).id, 'uuid')
def test_reduce(self):
a1 = AsyncResult("uuid", task_name="celery.ping")
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.task_id, "uuid")
self.assertEqual(restored.task_name, "celery.ping")
a2 = AsyncResult("uuid")
self.assertEqual(pickle.loads(pickle.dumps(a2)).task_id, "uuid")
def _mget_to_results(self, values, keys):
if hasattr(values, "items"):
# client returns dict so mapping preserved.
return dict((self._strip_prefix(k), pickle.loads(str(v)))
for k, v in values.iteritems() if v is not None)
else:
# client returns list so need to recreate mapping.
return dict((keys[i], pickle.loads(str(value)))
for i, value in enumerate(values) if value is not None)
def _do_get():
cf = self._get_column_family()
try:
obj = cf.get(task_id)
meta = {
"task_id": task_id,
"status": obj["status"],
"result": pickle.loads(str(obj["result"])),
"date_done": obj["date_done"],
"traceback": pickle.loads(str(obj["traceback"])),
}
except (KeyError, pycassa.NotFoundException):
meta = {"status": states.PENDING, "result": None}
return meta
def _do_get():
cf = self._get_column_family()
try:
obj = cf.get(task_id)
meta = {
"task_id": task_id,
"status": obj["status"],
"result": pickle.loads(str(obj["result"])),
"date_done": obj["date_done"],
"traceback": pickle.loads(str(obj["traceback"])),
}
except (KeyError, pycassa.NotFoundException):
meta = {"status": states.PENDING, "result": None}
return meta
def test_memcache_wrapper(self):
try:
from django.core.cache.backends import memcached
from django.core.cache.backends import locmem
except InvalidCacheBackendError:
sys.stderr.write(
"\n* Memcache library is not installed. Skipping test.\n")
return
prev_cache_cls = memcached.CacheClass
memcached.CacheClass = locmem.CacheClass
prev_backend_module = sys.modules.pop("djcelery.backends.cache")
try:
from djcelery.backends.cache import cache, DjangoMemcacheWrapper
self.assertIsInstance(cache, DjangoMemcacheWrapper)
key = "cu.test_memcache_wrapper"
val = "The quick brown fox."
default = "The lazy dog."
self.assertEqual(cache.get(key, default=default), default)
cache.set(key, val)
self.assertEqual(pickle.loads(cache.get(key, default=default)),
val)
finally:
memcached.CacheClass = prev_cache_cls
sys.modules["djcelery.backends.cache"] = prev_backend_module
def test_pickle_app(self):
changes = dict(THE_FOO_BAR="bars", THE_MII_MAR="jars")
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
self.assertLess(len(saved), 2048)
restored = pickle.loads(saved)
self.assertDictContainsSubset(changes, restored.conf)
def test_memcache_wrapper(self):
try:
from django.core.cache.backends import memcached
from django.core.cache.backends import locmem
except InvalidCacheBackendError:
sys.stderr.write(
"\n* Memcache library is not installed. Skipping test.\n")
return
prev_cache_cls = memcached.CacheClass
memcached.CacheClass = locmem.CacheClass
prev_backend_module = sys.modules.pop("djcelery.backends.cache")
try:
from djcelery.backends.cache import cache, DjangoMemcacheWrapper
self.assertIsInstance(cache, DjangoMemcacheWrapper)
key = "cu.test_memcache_wrapper"
val = "The quick brown fox."
default = "The lazy dog."
self.assertEqual(cache.get(key, default=default), default)
cache.set(key, val)
self.assertEqual(pickle.loads(cache.get(key, default=default)),
val)
finally:
memcached.CacheClass = prev_cache_cls
sys.modules["djcelery.backends.cache"] = prev_backend_module
def test_pickle_app(self):
changes = dict(THE_FOO_BAR='bars', THE_MII_MAR='jars')
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
self.assertLess(len(saved), 2048)
restored = pickle.loads(saved)
self.assertDictContainsSubset(changes, restored.conf)
def test_pickle_app(self):
changes = dict(THE_FOO_BAR="bars", THE_MII_MAR="jars")
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
assert len(saved) < 2048
restored = pickle.loads(saved)
for key, value in items(changes):
assert restored.conf[key] == value
def test_pickle_app(self):
changes = dict(THE_FOO_BAR='bars', THE_MII_MAR='jars')
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
assert len(saved) < 2048
restored = pickle.loads(saved)
for key, value in items(changes):
assert restored.conf[key] == value
def test_pickle_app(self):
changes = {'THE_FOO_BAR': 'bars', 'THE_MII_MAR': 'jars'}
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
assert len(saved) < 2048
restored = pickle.loads(saved)
for key, value in changes.items():
assert restored.conf[key] == value
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
db = self._get_database()
taskmeta_collection = db[self.mongodb_taskmeta_collection]
obj = taskmeta_collection.find_one({"_id": task_id})
if not obj:
return {"status": states.PENDING, "result": None}
meta = {
"task_id": obj["_id"],
"status": obj["status"],
"result": pickle.loads(str(obj["result"])),
"date_done": obj["date_done"],
"traceback": pickle.loads(str(obj["traceback"])),
}
return meta
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
db = self._get_database()
taskmeta_collection = db[self.mongodb_taskmeta_collection]
obj = taskmeta_collection.find_one({"_id": task_id})
if not obj:
return {"status": states.PENDING, "result": None}
meta = {
"task_id": obj["_id"],
"status": obj["status"],
"result": pickle.loads(str(obj["result"])),
"date_done": obj["date_done"],
"traceback": pickle.loads(str(obj["traceback"])),
}
return meta
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException('RegularException raised')
except RegularException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ('RegularException raised', ))
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException('RegularException raised')
except RegularException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
assert exception
assert isinstance(exception, RegularException)
assert exception.args == ('RegularException raised',)
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException('RegularException raised')
except RegularException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ('RegularException raised', ))
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException('RegularException raised')
except RegularException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
assert exception
assert isinstance(exception, RegularException)
assert exception.args == ('RegularException raised',)
class TestPickle(unittest.TestCase):
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException("RegularException raised")
except RegularException, exc_:
exc = exc_
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ("RegularException raised", ))
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException(
'ArgOverrideException raised', status_code=100,
)
except ArgOverrideException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
assert exception
assert isinstance(exception, ArgOverrideException)
assert exception.args == ('ArgOverrideException raised', 100)
assert exception.status_code == 100
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException('ArgOverrideException raised',
status_code=100)
except ArgOverrideException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
self.assertTrue(exception)
self.assertIsInstance(exception, ArgOverrideException)
self.assertTupleEqual(exception.args,
('ArgOverrideException raised', 100))
self.assertEqual(exception.status_code, 100)
def test_pending_configuration__setattr(self):
with self.Celery(broker='foo://bar') as app:
app.conf.task_default_delivery_mode = 44
app.conf.worker_agent = 'foo:Bar'
self.assertFalse(app.configured)
self.assertEqual(app.conf.worker_agent, 'foo:Bar')
self.assertEqual(app.conf.broker_url, 'foo://bar')
self.assertEqual(app._preconf['worker_agent'], 'foo:Bar')
self.assertTrue(app.configured)
reapp = pickle.loads(pickle.dumps(app))
self.assertEqual(reapp._preconf['worker_agent'], 'foo:Bar')
self.assertFalse(reapp.configured)
self.assertEqual(reapp.conf.worker_agent, 'foo:Bar')
self.assertTrue(reapp.configured)
self.assertEqual(reapp.conf.broker_url, 'foo://bar')
self.assertEqual(reapp._preconf['worker_agent'], 'foo:Bar')
def test_pending_configuration__setattr(self):
with self.Celery(broker='foo://bar') as app:
app.conf.task_default_delivery_mode = 44
app.conf.worker_agent = 'foo:Bar'
assert not app.configured
assert app.conf.worker_agent == 'foo:Bar'
assert app.conf.broker_url == 'foo://bar'
assert app._preconf['worker_agent'] == 'foo:Bar'
assert app.configured
reapp = pickle.loads(pickle.dumps(app))
assert reapp._preconf['worker_agent'] == 'foo:Bar'
assert not reapp.configured
assert reapp.conf.worker_agent == 'foo:Bar'
assert reapp.configured
assert reapp.conf.broker_url == 'foo://bar'
assert reapp._preconf['worker_agent'] == 'foo:Bar'
def test_pending_configuration__setattr(self):
with self.Celery(broker='foo://bar') as app:
app.conf.task_default_delivery_mode = 44
app.conf.worker_agent = 'foo:Bar'
self.assertFalse(app.configured)
self.assertEqual(app.conf.worker_agent, 'foo:Bar')
self.assertEqual(app.conf.broker_url, 'foo://bar')
self.assertEqual(app._preconf['worker_agent'], 'foo:Bar')
self.assertTrue(app.configured)
reapp = pickle.loads(pickle.dumps(app))
self.assertEqual(reapp._preconf['worker_agent'], 'foo:Bar')
self.assertFalse(reapp.configured)
self.assertEqual(reapp.conf.worker_agent, 'foo:Bar')
self.assertTrue(reapp.configured)
self.assertEqual(reapp.conf.broker_url, 'foo://bar')
self.assertEqual(reapp._preconf['worker_agent'], 'foo:Bar')
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException(
'ArgOverrideException raised', status_code=100,
)
except ArgOverrideException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
assert exception
assert isinstance(exception, ArgOverrideException)
assert exception.args == ('ArgOverrideException raised', 100)
assert exception.status_code == 100
def test_init(self):
from celery import chord
from celery.utils.serialization import pickle
@self.app.task(shared=False)
def addX(x, y):
return x + y
@self.app.task(shared=False)
def sumX(n):
return sum(n)
x = chord(addX.s(i, i) for i in range(10))
# kwargs used to nest and recurse in serialization/deserialization
# (#6810)
assert x.kwargs['kwargs'] == {}
assert pickle.loads(pickle.dumps(x)).kwargs == x.kwargs
def test_pending_configuration__setattr(self):
with self.Celery(broker='foo://bar') as app:
app.conf.task_default_delivery_mode = 44
app.conf.worker_agent = 'foo:Bar'
assert not app.configured
assert app.conf.worker_agent == 'foo:Bar'
assert app.conf.broker_url == 'foo://bar'
assert app._preconf['worker_agent'] == 'foo:Bar'
assert app.configured
reapp = pickle.loads(pickle.dumps(app))
assert reapp._preconf['worker_agent'] == 'foo:Bar'
assert not reapp.configured
assert reapp.conf.worker_agent == 'foo:Bar'
assert reapp.configured
assert reapp.conf.broker_url == 'foo://bar'
assert reapp._preconf['worker_agent'] == 'foo:Bar'
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException(
'ArgOverrideException raised', status_code=100,
)
except ArgOverrideException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
self.assertTrue(exception)
self.assertIsInstance(exception, ArgOverrideException)
self.assertTupleEqual(exception.args, (
'ArgOverrideException raised', 100))
self.assertEqual(exception.status_code, 100)
def test_is_pickleable(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
self.assertEqual(pickle.loads(pickle.dumps(ts)), ts)
ts2 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
self.assertEqual(pickle.loads(pickle.dumps(ts2)), ts2)
def test_reduce(self):
self.assertTrue(pickle.loads(pickle.dumps(self.ts)))
exc = None
try:
raise RegularException("RegularException raised")
except RegularException, exc_:
exc = exc_
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ("RegularException raised", ))
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException("ArgOverrideException raised",
status_code=100)
except ArgOverrideException, exc_:
exc = exc_
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, ArgOverrideException)
self.assertTupleEqual(exception.args, (
"ArgOverrideException raised", 100))
self.assertEqual(exception.status_code, 100)
def test_reduce(self):
self.assertTrue(pickle.loads(pickle.dumps(self.ts)))
def test_pickleable(self):
self.assertTrue(pickle.loads(pickle.dumps(self.ts)))
def _restore_taskset(self, taskset_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_taskset(taskset_id))
if meta:
meta = pickle.loads(str(meta))
return meta
def _restore_taskset(self, taskset_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_taskset(taskset_id))
if meta:
meta = pickle.loads(str(meta))
return meta
def test_is_pickleable(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
assert pickle.loads(pickle.dumps(ts)) == ts
ts2 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
assert pickle.loads(pickle.dumps(ts2)) == ts2
def test_pickleable(self):
assert pickle.loads(pickle.dumps(self.ts))
exc = None
try:
raise RegularException("RegularException raised")
except RegularException, exc_:
exc = exc_
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ("RegularException raised", ))
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException("ArgOverrideException raised",
status_code=100)
except ArgOverrideException, exc_:
exc = exc_
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, ArgOverrideException)
self.assertTupleEqual(exception.args,
("ArgOverrideException raised", 100))
self.assertEqual(exception.status_code, 100)
def decode(value, compress_object=False):
return pickle.loads(maybe_decompress(b64decode(value), compress_object))
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_task(task_id))
if not meta:
return {"status": states.PENDING, "result": None}
return pickle.loads(str(meta))
def decode(value, compress_object=False):
return pickle.loads(maybe_decompress(b64decode(value), compress_object))
def test_is_pickleable(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
assert pickle.loads(pickle.dumps(ts)) == ts
ts2 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
assert pickle.loads(pickle.dumps(ts2)) == ts2
def test_is_pickleable(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
self.assertEqual(pickle.loads(pickle.dumps(ts)), ts)
ts2 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
self.assertEqual(pickle.loads(pickle.dumps(ts2)), ts2)
def test_pickleable(self):
assert pickle.loads(pickle.dumps(self.ts))
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_task(task_id))
if not meta:
return {"status": states.PENDING, "result": None}
return pickle.loads(str(meta))
def test_pickleable(self):
self.assertTrue(pickle.loads(pickle.dumps(self.ts)))
