def to_python(self, value): """Convert the database value to a python value.""" if isinstance(value, PickledObject): # If the value is a definite pickle; and an error is # raised in de-pickling it should be allowed to propogate. return pickle.loads(str(value)) else: try: return pickle.loads(str(value)) except Exception: # If an error was raised, just return the plain value return value
def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" cf = self._get_column_family() try: obj = cf.get(task_id) meta = { "task_id": task_id, "status": obj["status"], "result": pickle.loads(str(obj["result"])), "date_done": obj["date_done"], "traceback": pickle.loads(str(obj["traceback"])), } except (KeyError, pycassa.NotFoundException): meta = {"status": states.PENDING, "result": None} return meta
def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" cf = self._get_column_family() try: obj = cf.get(task_id) meta = { "task_id": task_id, "status": obj["status"], "result": pickle.loads(str(obj["result"])), "date_done": obj["date_done"], "traceback": pickle.loads(str(obj["traceback"])), } except (KeyError, pycassa.NotFoundException): meta = {"status": states.PENDING, "result": None} return meta
def test_memcache_wrapper(self): try: from django.core.cache.backends import memcached from django.core.cache.backends import locmem except InvalidCacheBackendError: sys.stderr.write( "\n* Memcache library is not installed. Skipping test.\n") return prev_cache_cls = memcached.CacheClass memcached.CacheClass = locmem.CacheClass prev_backend_module = sys.modules.pop("djcelery.backends.cache") try: from djcelery.backends.cache import cache, DjangoMemcacheWrapper self.assertIsInstance(cache, DjangoMemcacheWrapper) key = "cu.test_memcache_wrapper" val = "The quick brown fox." default = "The lazy dog." self.assertEqual(cache.get(key, default=default), default) cache.set(key, val) self.assertEqual(pickle.loads(cache.get(key, default=default)), val) finally: memcached.CacheClass = prev_cache_cls sys.modules["djcelery.backends.cache"] = prev_backend_module
def test_memcache_wrapper(self): try: from django.core.cache.backends import memcached from django.core.cache.backends import locmem except InvalidCacheBackendError: sys.stderr.write( "\n* Memcache library is not installed. Skipping test.\n") return prev_cache_cls = memcached.CacheClass memcached.CacheClass = locmem.CacheClass prev_backend_module = sys.modules.pop("djcelery.backends.cache") try: from djcelery.backends.cache import cache, DjangoMemcacheWrapper self.assertIsInstance(cache, DjangoMemcacheWrapper) key = "cu.test_memcache_wrapper" val = "The quick brown fox." default = "The lazy dog." self.assertEqual(cache.get(key, default=default), default) cache.set(key, val) self.assertEqual(pickle.loads(cache.get(key, default=default)), val) finally: memcached.CacheClass = prev_cache_cls sys.modules["djcelery.backends.cache"] = prev_backend_module
def run(self, ser_callable, fargs, fkwargs, **kwargs): """ :param ser_callable: A pickled function or callable object. :param fargs: Positional arguments to apply to the function. :param fkwargs: Keyword arguments to apply to the function. """ return pickle.loads(ser_callable)(*fargs, **fkwargs)
def run(self, ser_callable, fargs, fkwargs, **kwargs): """ :param ser_callable: A pickled function or callable object. :param fargs: Positional arguments to apply to the function. :param fkwargs: Keyword arguments to apply to the function. """ return pickle.loads(ser_callable)(*fargs, **fkwargs)
def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" db = self._get_database() taskmeta_collection = db[self.mongodb_taskmeta_collection] obj = taskmeta_collection.find_one({"_id": task_id}) if not obj: return {"status": states.PENDING, "result": None} meta = { "task_id": obj["_id"], "status": obj["status"], "result": pickle.loads(str(obj["result"])), "date_done": obj["date_done"], "traceback": pickle.loads(str(obj["traceback"])), } return meta
def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" db = self._get_database() taskmeta_collection = db[self.mongodb_taskmeta_collection] obj = taskmeta_collection.find_one({"_id": task_id}) if not obj: return {"status": states.PENDING, "result": None} meta = { "task_id": obj["_id"], "status": obj["status"], "result": pickle.loads(str(obj["result"])), "date_done": obj["date_done"], "traceback": pickle.loads(str(obj["traceback"])), } return meta
def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" if task_id in self._cache: return self._cache[task_id] meta = self.get(self.get_cache_key_for_task(task_id)) if not meta: return {"status": "PENDING", "result": None} meta = pickle.loads(str(meta)) if meta.get("status") == "DONE": self._cache[task_id] = meta return meta
def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" if task_id in self._cache: return self._cache[task_id] db = self._get_database() taskmeta_collection = db[self.mongodb_taskmeta_collection] obj = taskmeta_collection.find_one({"_id": task_id}) if not obj: return {"status": "PENDING", "result": None} meta = { "task_id": obj["_id"], "status": obj["status"], "result": pickle.loads(str(obj["result"])), "date_done": obj["date_done"], "traceback": pickle.loads(str(obj["traceback"])), } if meta["status"] == "DONE": self._cache[task_id] = meta return meta
class TestPickle(unittest.TestCase): def test_pickle_regular_exception(self): exc = None try: raise RegularException("RegularException raised") except RegularException, exc: pass pickled = pickle.dumps({"exception": exc}) unpickled = pickle.loads(pickled) exception = unpickled.get("exception") self.assertTrue(exception) self.assertIsInstance(exception, RegularException) self.assertTupleEqual(exception.args, ("RegularException raised", ))
def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" meta = self.get(self.get_key_for_task(task_id)) if not meta: return {"status": states.PENDING, "result": None} return pickle.loads(str(meta))
exc = None try: raise RegularException("RegularException raised") except RegularException, exc: pass pickled = pickle.dumps({"exception": exc}) unpickled = pickle.loads(pickled) exception = unpickled.get("exception") self.assertTrue(exception) self.assertIsInstance(exception, RegularException) self.assertTupleEqual(exception.args, ("RegularException raised", )) def test_pickle_arg_override_exception(self): exc = None try: raise ArgOverrideException("ArgOverrideException raised", status_code=100) except ArgOverrideException, exc: pass pickled = pickle.dumps({"exception": exc}) unpickled = pickle.loads(pickled) exception = unpickled.get("exception") self.assertTrue(exception) self.assertIsInstance(exception, ArgOverrideException) self.assertTupleEqual(exception.args, ("ArgOverrideException raised", 100)) self.assertEqual(exception.status_code, 100)
def _get_task_meta_for(self, task_id): """Get task metadata for a task by id.""" meta = self.get(self.get_key_for_task(task_id)) if not meta: return {"status": states.PENDING, "result": None} return pickle.loads(str(meta))
def run(self, serfun, args, timeout=None, **kwargs): return _dmap(pickle.loads(serfun), args, timeout=timeout)
def run(self, serfunc, args, **kwargs): """The method run by ``celeryd``.""" timeout = kwargs.get("timeout") return TaskSet.map(pickle.loads(serfunc), args, timeout=timeout)
def _restore_taskset(self, taskset_id): """Get task metadata for a task by id.""" meta = self.get(self.get_key_for_taskset(taskset_id)) if meta: meta = pickle.loads(str(meta)) return meta
e = None try: raise RegularException("RegularException raised") except RegularException, e: pass pickled = pickle.dumps({"exception": e}) unpickled = pickle.loads(pickled) exception = unpickled.get("exception") self.assertTrue(exception) self.assertTrue(isinstance(exception, RegularException)) self.assertEquals(exception.args, ("RegularException raised", )) def test_pickle_arg_override_exception(self): e = None try: raise ArgOverrideException("ArgOverrideException raised", status_code=100) except ArgOverrideException, e: pass pickled = pickle.dumps({"exception": e}) unpickled = pickle.loads(pickled) exception = unpickled.get("exception") self.assertTrue(exception) self.assertTrue(isinstance(exception, ArgOverrideException)) self.assertEquals(exception.args, ("ArgOverrideException raised", 100)) self.assertEquals(exception.status_code, 100)
def _restore_taskset(self, taskset_id): """Get task metadata for a task by id.""" meta = self.get(self.get_key_for_taskset(taskset_id)) if meta: meta = pickle.loads(str(meta)) return meta
def run(self, serfun, args, timeout=None, **kwargs): return _dmap(pickle.loads(serfun), args, timeout=timeout)