def to_python(self, value):
"""Convert the database value to a python value."""
if isinstance(value, PickledObject):
# If the value is a definite pickle; and an error is
# raised in de-pickling it should be allowed to propogate.
return pickle.loads(str(value))
else:
try:
return pickle.loads(str(value))
except Exception:
# If an error was raised, just return the plain value
return value
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
cf = self._get_column_family()
try:
obj = cf.get(task_id)
meta = {
"task_id": task_id,
"status": obj["status"],
"result": pickle.loads(str(obj["result"])),
"date_done": obj["date_done"],
"traceback": pickle.loads(str(obj["traceback"])),
}
except (KeyError, pycassa.NotFoundException):
meta = {"status": states.PENDING, "result": None}
return meta
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
cf = self._get_column_family()
try:
obj = cf.get(task_id)
meta = {
"task_id": task_id,
"status": obj["status"],
"result": pickle.loads(str(obj["result"])),
"date_done": obj["date_done"],
"traceback": pickle.loads(str(obj["traceback"])),
}
except (KeyError, pycassa.NotFoundException):
meta = {"status": states.PENDING, "result": None}
return meta
def test_memcache_wrapper(self):
try:
from django.core.cache.backends import memcached
from django.core.cache.backends import locmem
except InvalidCacheBackendError:
sys.stderr.write(
"\n* Memcache library is not installed. Skipping test.\n")
return
prev_cache_cls = memcached.CacheClass
memcached.CacheClass = locmem.CacheClass
prev_backend_module = sys.modules.pop("djcelery.backends.cache")
try:
from djcelery.backends.cache import cache, DjangoMemcacheWrapper
self.assertIsInstance(cache, DjangoMemcacheWrapper)
key = "cu.test_memcache_wrapper"
val = "The quick brown fox."
default = "The lazy dog."
self.assertEqual(cache.get(key, default=default), default)
cache.set(key, val)
self.assertEqual(pickle.loads(cache.get(key, default=default)),
val)
finally:
memcached.CacheClass = prev_cache_cls
sys.modules["djcelery.backends.cache"] = prev_backend_module
def test_memcache_wrapper(self):
try:
from django.core.cache.backends import memcached
from django.core.cache.backends import locmem
except InvalidCacheBackendError:
sys.stderr.write(
"\n* Memcache library is not installed. Skipping test.\n")
return
prev_cache_cls = memcached.CacheClass
memcached.CacheClass = locmem.CacheClass
prev_backend_module = sys.modules.pop("djcelery.backends.cache")
try:
from djcelery.backends.cache import cache, DjangoMemcacheWrapper
self.assertIsInstance(cache, DjangoMemcacheWrapper)
key = "cu.test_memcache_wrapper"
val = "The quick brown fox."
default = "The lazy dog."
self.assertEqual(cache.get(key, default=default), default)
cache.set(key, val)
self.assertEqual(pickle.loads(cache.get(key, default=default)),
val)
finally:
memcached.CacheClass = prev_cache_cls
sys.modules["djcelery.backends.cache"] = prev_backend_module
def run(self, ser_callable, fargs, fkwargs, **kwargs):
"""
:param ser_callable: A pickled function or callable object.
:param fargs: Positional arguments to apply to the function.
:param fkwargs: Keyword arguments to apply to the function.
"""
return pickle.loads(ser_callable)(*fargs, **fkwargs)
def run(self, ser_callable, fargs, fkwargs, **kwargs):
"""
:param ser_callable: A pickled function or callable object.
:param fargs: Positional arguments to apply to the function.
:param fkwargs: Keyword arguments to apply to the function.
"""
return pickle.loads(ser_callable)(*fargs, **fkwargs)
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
db = self._get_database()
taskmeta_collection = db[self.mongodb_taskmeta_collection]
obj = taskmeta_collection.find_one({"_id": task_id})
if not obj:
return {"status": states.PENDING, "result": None}
meta = {
"task_id": obj["_id"],
"status": obj["status"],
"result": pickle.loads(str(obj["result"])),
"date_done": obj["date_done"],
"traceback": pickle.loads(str(obj["traceback"])),
}
return meta
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
db = self._get_database()
taskmeta_collection = db[self.mongodb_taskmeta_collection]
obj = taskmeta_collection.find_one({"_id": task_id})
if not obj:
return {"status": states.PENDING, "result": None}
meta = {
"task_id": obj["_id"],
"status": obj["status"],
"result": pickle.loads(str(obj["result"])),
"date_done": obj["date_done"],
"traceback": pickle.loads(str(obj["traceback"])),
}
return meta
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
if task_id in self._cache:
return self._cache[task_id]
meta = self.get(self.get_cache_key_for_task(task_id))
if not meta:
return {"status": "PENDING", "result": None}
meta = pickle.loads(str(meta))
if meta.get("status") == "DONE":
self._cache[task_id] = meta
return meta
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
if task_id in self._cache:
return self._cache[task_id]
db = self._get_database()
taskmeta_collection = db[self.mongodb_taskmeta_collection]
obj = taskmeta_collection.find_one({"_id": task_id})
if not obj:
return {"status": "PENDING", "result": None}
meta = {
"task_id": obj["_id"],
"status": obj["status"],
"result": pickle.loads(str(obj["result"])),
"date_done": obj["date_done"],
"traceback": pickle.loads(str(obj["traceback"])),
}
if meta["status"] == "DONE":
self._cache[task_id] = meta
return meta
class TestPickle(unittest.TestCase):
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException("RegularException raised")
except RegularException, exc:
pass
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ("RegularException raised", ))
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_task(task_id))
if not meta:
return {"status": states.PENDING, "result": None}
return pickle.loads(str(meta))
exc = None
try:
raise RegularException("RegularException raised")
except RegularException, exc:
pass
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ("RegularException raised", ))
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException("ArgOverrideException raised",
status_code=100)
except ArgOverrideException, exc:
pass
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, ArgOverrideException)
self.assertTupleEqual(exception.args,
("ArgOverrideException raised", 100))
self.assertEqual(exception.status_code, 100)
def _get_task_meta_for(self, task_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_task(task_id))
if not meta:
return {"status": states.PENDING, "result": None}
return pickle.loads(str(meta))
def run(self, serfun, args, timeout=None, **kwargs):
return _dmap(pickle.loads(serfun), args, timeout=timeout)
def run(self, serfunc, args, **kwargs):
"""The method run by ``celeryd``."""
timeout = kwargs.get("timeout")
return TaskSet.map(pickle.loads(serfunc), args, timeout=timeout)
def _restore_taskset(self, taskset_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_taskset(taskset_id))
if meta:
meta = pickle.loads(str(meta))
return meta
e = None
try:
raise RegularException("RegularException raised")
except RegularException, e:
pass
pickled = pickle.dumps({"exception": e})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertTrue(isinstance(exception, RegularException))
self.assertEquals(exception.args, ("RegularException raised", ))
def test_pickle_arg_override_exception(self):
e = None
try:
raise ArgOverrideException("ArgOverrideException raised",
status_code=100)
except ArgOverrideException, e:
pass
pickled = pickle.dumps({"exception": e})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertTrue(isinstance(exception, ArgOverrideException))
self.assertEquals(exception.args, ("ArgOverrideException raised",
100))
self.assertEquals(exception.status_code, 100)
def _restore_taskset(self, taskset_id):
"""Get task metadata for a task by id."""
meta = self.get(self.get_key_for_taskset(taskset_id))
if meta:
meta = pickle.loads(str(meta))
return meta
def run(self, serfun, args, timeout=None, **kwargs):
return _dmap(pickle.loads(serfun), args, timeout=timeout)
