def test_reduce(self):
a1 = self.app.AsyncResult('uuid')
restored = pickle.loads(pickle.dumps(a1))
assert restored.id == 'uuid'
a2 = self.app.AsyncResult('uuid')
assert pickle.loads(pickle.dumps(a2)).id == 'uuid'
def test_reduce(self):
a1 = self.app.AsyncResult('uuid')
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.id, 'uuid')
a2 = self.app.AsyncResult('uuid')
self.assertEqual(pickle.loads(pickle.dumps(a2)).id, 'uuid')
def test_reduce(self):
a1 = AsyncResult('uuid', task_name=mytask.name)
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.id, 'uuid')
self.assertEqual(restored.task_name, mytask.name)
a2 = AsyncResult('uuid')
self.assertEqual(pickle.loads(pickle.dumps(a2)).id, 'uuid')
def test_reduce(self):
a1 = self.app.AsyncResult("uuid", task_name=self.mytask.name)
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.id, "uuid")
self.assertEqual(restored.task_name, self.mytask.name)
a2 = self.app.AsyncResult("uuid")
self.assertEqual(pickle.loads(pickle.dumps(a2)).id, "uuid")
def test_reduce(self):
a1 = AsyncResult("uuid", task_name="celery.ping")
restored = pickle.loads(pickle.dumps(a1))
self.assertEqual(restored.task_id, "uuid")
self.assertEqual(restored.task_name, "celery.ping")
a2 = AsyncResult("uuid")
self.assertEqual(pickle.loads(pickle.dumps(a2)).task_id, "uuid")
def _do_store():
cf = self._get_column_family()
date_done = datetime.utcnow()
meta = {"status": status,
"result": pickle.dumps(result),
"date_done": date_done.strftime('%Y-%m-%dT%H:%M:%SZ'),
"traceback": pickle.dumps(traceback)}
cf.insert(task_id, meta,
ttl=timedelta_seconds(self.result_expires))
def _store_result(self, task_id, result, status, traceback=None):
"""Store return value and status of an executed task."""
cf = self._get_column_family()
date_done = datetime.utcnow()
lifetime = self.result_expires.days * 86400 + self.result_expires.seconds
meta = {"status": status,
"result": pickle.dumps(result),
"date_done": date_done.strftime('%Y-%m-%dT%H:%M:%SZ'),
"traceback": pickle.dumps(traceback)}
cf.insert(task_id, meta, ttl=lifetime)
def _store_result(self, task_id, result, status, traceback=None):
"""Store return value and status of an executed task."""
cf = self._get_column_family()
date_done = datetime.utcnow()
index_key = 'celery.results.index!%02x' % (
random.randrange(self._index_shards))
index_column_name = '%8x!%s' % (time.mktime(date_done.timetuple()),
task_id)
meta = {"status": status,
"result": pickle.dumps(result),
"date_done": date_done.strftime('%Y-%m-%dT%H:%M:%SZ'),
"traceback": pickle.dumps(traceback)}
cf.insert(task_id, meta)
cf.insert(index_key, {index_column_name: status})
def _store_result(self, task_id, result, status, traceback=None):
"""Store return value and status of an executed task."""
from pymongo.binary import Binary
meta = {"_id": task_id,
"status": status,
"result": Binary(pickle.dumps(result)),
"date_done": datetime.now(),
"traceback": Binary(pickle.dumps(traceback))}
db = self._get_database()
taskmeta_collection = db[self.mongodb_taskmeta_collection]
taskmeta_collection.save(meta, safe=True)
return result
def create_message(channel, **data):
data.setdefault('id', uuid())
channel.no_ack_consumers = set()
return Message(channel, body=pickle.dumps(dict(**data)),
content_type='application/x-python-serialize',
content_encoding='binary',
delivery_info={'consumer_tag': 'mock'})
def test_pickle_app(self):
changes = dict(THE_FOO_BAR="bars", THE_MII_MAR="jars")
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
self.assertLess(len(saved), 2048)
restored = pickle.loads(saved)
self.assertDictContainsSubset(changes, restored.conf)
def create_message(channel, **data):
data.setdefault("id", uuid())
channel.no_ack_consumers = set()
return Message(channel, body=pickle.dumps(dict(**data)),
content_type="application/x-python-serialize",
content_encoding="binary",
delivery_info={"consumer_tag": "mock"})
def create_message(backend, **data):
data.setdefault("id", gen_unique_id())
return Message(
backend,
body=pickle.dumps(dict(**data)),
content_type="application/x-python-serialize",
content_encoding="binary",
)
def test_pickle_app(self):
changes = dict(THE_FOO_BAR="bars", THE_MII_MAR="jars")
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
assert len(saved) < 2048
restored = pickle.loads(saved)
for key, value in items(changes):
assert restored.conf[key] == value
def create_message(channel, **data):
data.setdefault('id', uuid())
m = Message(body=pickle.dumps(dict(**data)),
channel=channel,
content_type='application/x-python-serialize',
content_encoding='binary',
delivery_info={'consumer_tag': 'mock'})
m.accept = ['application/x-python-serialize']
return m
def test_pickle_app(self):
changes = dict(THE_FOO_BAR='bars',
THE_MII_MAR='jars')
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
assert len(saved) < 2048
restored = pickle.loads(saved)
for key, value in items(changes):
assert restored.conf[key] == value
def test_pickle_app(self):
changes = {'THE_FOO_BAR': 'bars',
'THE_MII_MAR': 'jars'}
self.app.conf.update(changes)
saved = pickle.dumps(self.app)
assert len(saved) < 2048
restored = pickle.loads(saved)
for key, value in changes.items():
assert restored.conf[key] == value
def create_message(channel, **data):
data.setdefault('id', uuid())
m = Message(body=pickle.dumps(dict(**data)),
channel=channel,
content_type='application/x-python-serialize',
content_encoding='binary',
delivery_info={'consumer_tag': 'mock'})
m.accept = ['application/x-python-serialize']
return m
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException('RegularException raised')
except RegularException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ('RegularException raised', ))
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException('RegularException raised')
except RegularException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
assert exception
assert isinstance(exception, RegularException)
assert exception.args == ('RegularException raised',)
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException('RegularException raised')
except RegularException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ('RegularException raised', ))
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException('RegularException raised')
except RegularException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
assert exception
assert isinstance(exception, RegularException)
assert exception.args == ('RegularException raised',)
class TestPickle(unittest.TestCase):
def test_pickle_regular_exception(self):
exc = None
try:
raise RegularException("RegularException raised")
except RegularException, exc_:
exc = exc_
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ("RegularException raised", ))
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException('ArgOverrideException raised',
status_code=100)
except ArgOverrideException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
self.assertTrue(exception)
self.assertIsInstance(exception, ArgOverrideException)
self.assertTupleEqual(exception.args,
('ArgOverrideException raised', 100))
self.assertEqual(exception.status_code, 100)
def test_pending_configuration__setattr(self):
with self.Celery(broker='foo://bar') as app:
app.conf.task_default_delivery_mode = 44
app.conf.worker_agent = 'foo:Bar'
assert not app.configured
assert app.conf.worker_agent == 'foo:Bar'
assert app.conf.broker_url == 'foo://bar'
assert app._preconf['worker_agent'] == 'foo:Bar'
assert app.configured
reapp = pickle.loads(pickle.dumps(app))
assert reapp._preconf['worker_agent'] == 'foo:Bar'
assert not reapp.configured
assert reapp.conf.worker_agent == 'foo:Bar'
assert reapp.configured
assert reapp.conf.broker_url == 'foo://bar'
assert reapp._preconf['worker_agent'] == 'foo:Bar'
def test_pending_configuration__setattr(self):
with self.Celery(broker='foo://bar') as app:
app.conf.task_default_delivery_mode = 44
app.conf.worker_agent = 'foo:Bar'
self.assertFalse(app.configured)
self.assertEqual(app.conf.worker_agent, 'foo:Bar')
self.assertEqual(app.conf.broker_url, 'foo://bar')
self.assertEqual(app._preconf['worker_agent'], 'foo:Bar')
self.assertTrue(app.configured)
reapp = pickle.loads(pickle.dumps(app))
self.assertEqual(reapp._preconf['worker_agent'], 'foo:Bar')
self.assertFalse(reapp.configured)
self.assertEqual(reapp.conf.worker_agent, 'foo:Bar')
self.assertTrue(reapp.configured)
self.assertEqual(reapp.conf.broker_url, 'foo://bar')
self.assertEqual(reapp._preconf['worker_agent'], 'foo:Bar')
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException(
'ArgOverrideException raised', status_code=100,
)
except ArgOverrideException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
assert exception
assert isinstance(exception, ArgOverrideException)
assert exception.args == ('ArgOverrideException raised', 100)
assert exception.status_code == 100
def test_pending_configuration__setattr(self):
with self.Celery(broker='foo://bar') as app:
app.conf.task_default_delivery_mode = 44
app.conf.worker_agent = 'foo:Bar'
self.assertFalse(app.configured)
self.assertEqual(app.conf.worker_agent, 'foo:Bar')
self.assertEqual(app.conf.broker_url, 'foo://bar')
self.assertEqual(app._preconf['worker_agent'], 'foo:Bar')
self.assertTrue(app.configured)
reapp = pickle.loads(pickle.dumps(app))
self.assertEqual(reapp._preconf['worker_agent'], 'foo:Bar')
self.assertFalse(reapp.configured)
self.assertEqual(reapp.conf.worker_agent, 'foo:Bar')
self.assertTrue(reapp.configured)
self.assertEqual(reapp.conf.broker_url, 'foo://bar')
self.assertEqual(reapp._preconf['worker_agent'], 'foo:Bar')
def test_pending_configuration__setattr(self):
with self.Celery(broker='foo://bar') as app:
app.conf.task_default_delivery_mode = 44
app.conf.worker_agent = 'foo:Bar'
assert not app.configured
assert app.conf.worker_agent == 'foo:Bar'
assert app.conf.broker_url == 'foo://bar'
assert app._preconf['worker_agent'] == 'foo:Bar'
assert app.configured
reapp = pickle.loads(pickle.dumps(app))
assert reapp._preconf['worker_agent'] == 'foo:Bar'
assert not reapp.configured
assert reapp.conf.worker_agent == 'foo:Bar'
assert reapp.configured
assert reapp.conf.broker_url == 'foo://bar'
assert reapp._preconf['worker_agent'] == 'foo:Bar'
def test_init(self):
from celery import chord
from celery.utils.serialization import pickle
@self.app.task(shared=False)
def addX(x, y):
return x + y
@self.app.task(shared=False)
def sumX(n):
return sum(n)
x = chord(addX.s(i, i) for i in range(10))
# kwargs used to nest and recurse in serialization/deserialization
# (#6810)
assert x.kwargs['kwargs'] == {}
assert pickle.loads(pickle.dumps(x)).kwargs == x.kwargs
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException(
'ArgOverrideException raised', status_code=100,
)
except ArgOverrideException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
assert exception
assert isinstance(exception, ArgOverrideException)
assert exception.args == ('ArgOverrideException raised', 100)
assert exception.status_code == 100
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException(
'ArgOverrideException raised', status_code=100,
)
except ArgOverrideException as exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
self.assertTrue(exception)
self.assertIsInstance(exception, ArgOverrideException)
self.assertTupleEqual(exception.args, (
'ArgOverrideException raised', 100))
self.assertEqual(exception.status_code, 100)
def test_is_pickleable(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
self.assertEqual(pickle.loads(pickle.dumps(ts)), ts)
ts2 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
self.assertEqual(pickle.loads(pickle.dumps(ts2)), ts2)
def test_is_pickleable(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
assert pickle.loads(pickle.dumps(ts)) == ts
ts2 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
assert pickle.loads(pickle.dumps(ts2)) == ts2
def test_pickleable(self):
assert pickle.loads(pickle.dumps(self.ts))
def test_reduce(self):
self.assertTrue(pickle.loads(pickle.dumps(self.ts)))
exc = None
try:
raise RegularException("RegularException raised")
except RegularException, exc_:
exc = exc_
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ("RegularException raised", ))
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException("ArgOverrideException raised",
status_code=100)
except ArgOverrideException, exc_:
exc = exc_
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, ArgOverrideException)
self.assertTupleEqual(exception.args, (
"ArgOverrideException raised", 100))
self.assertEqual(exception.status_code, 100)
def test_pickleable(self):
assert pickle.loads(pickle.dumps(self.ts))
def _store_result(self, task_id, result, status, traceback=None):
meta = {"status": status, "result": result, "traceback": traceback}
self.set(self.get_key_for_task(task_id), pickle.dumps(meta))
return result
def _save_taskset(self, taskset_id, result):
self.set(self.get_key_for_taskset(taskset_id),
pickle.dumps({"result": result}))
return result
def test_is_pickleable(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
assert pickle.loads(pickle.dumps(ts)) == ts
ts2 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
assert pickle.loads(pickle.dumps(ts2)) == ts2
exc = None
try:
raise RegularException("RegularException raised")
except RegularException, exc_:
exc = exc_
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ("RegularException raised", ))
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException("ArgOverrideException raised",
status_code=100)
except ArgOverrideException, exc_:
exc = exc_
pickled = pickle.dumps({"exception": exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get("exception")
self.assertTrue(exception)
self.assertIsInstance(exception, ArgOverrideException)
self.assertTupleEqual(exception.args,
("ArgOverrideException raised", 100))
self.assertEqual(exception.status_code, 100)
def test_pickleable(self):
self.assertTrue(pickle.loads(pickle.dumps(self.ts)))
def create_message(backend, **data):
data.setdefault("id", gen_unique_id())
return Message(backend,
body=pickle.dumps(dict(**data)),
content_type="application/x-python-serialize",
content_encoding="binary")
exc = None
try:
raise RegularException('RegularException raised')
except RegularException, exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
self.assertTrue(exception)
self.assertIsInstance(exception, RegularException)
self.assertTupleEqual(exception.args, ('RegularException raised', ))
def test_pickle_arg_override_exception(self):
exc = None
try:
raise ArgOverrideException('ArgOverrideException raised',
status_code=100)
except ArgOverrideException, exc_:
exc = exc_
pickled = pickle.dumps({'exception': exc})
unpickled = pickle.loads(pickled)
exception = unpickled.get('exception')
self.assertTrue(exception)
self.assertIsInstance(exception, ArgOverrideException)
self.assertTupleEqual(exception.args,
('ArgOverrideException raised', 100))
self.assertEqual(exception.status_code, 100)
def test_pickleable(self):
self.assertTrue(pickle.loads(pickle.dumps(self.ts)))
def test_is_pickleable(self):
ts = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
self.assertEqual(pickle.loads(pickle.dumps(ts)), ts)
ts2 = self.app.GroupResult(uuid(), [self.app.AsyncResult(uuid())])
self.assertEqual(pickle.loads(pickle.dumps(ts2)), ts2)
def _save_taskset(self, taskset_id, result):
meta = {"result": result}
self.set(self.get_key_for_taskset(taskset_id), pickle.dumps(meta))
return result
def encode(value, compress_object=False, pickle_protocol=DEFAULT_PROTOCOL):
return b64encode(maybe_compress(
pickle.dumps(value, pickle_protocol), compress_object),
)
def _store_result(self, task_id, result, status, traceback=None):
meta = {"status": status, "result": result, "traceback": traceback}
self.set(self.get_key_for_task(task_id), pickle.dumps(meta))
return result
def test_reduce(self):
self.assertTrue(pickle.loads(pickle.dumps(self.ts)))
def encode(value, compress_object=False, pickle_protocol=DEFAULT_PROTOCOL):
return b64encode(maybe_compress(
pickle.dumps(value, pickle_protocol), compress_object),
)
