def test_backend__pickle_serialization__dict_result__protocol_1(self):
self.app.conf.result_serializer = 'pickle'
self.app.conf.accept_content = {'pickle', 'json'}
self.b = DatabaseBackend(app=self.app)
tid2 = uuid()
request = self._create_request(
task_id=tid2,
name='my_task',
args=['a', 1, SomeClass(67)],
kwargs={
'c': 6,
'd': 'e',
'f': SomeClass(89)
},
task_protocol=1,
)
result = {'foo': 'baz', 'bar': SomeClass(12345)}
self.b.mark_as_done(tid2, result, request=request)
mindb = self.b.get_task_meta(tid2)
# check task meta
assert mindb.get('result').get('foo') == 'baz'
assert mindb.get('result').get('bar').data == 12345
assert mindb.get('task_name') == 'my_task'
assert mindb.get('task_args')[0] == 'a'
assert mindb.get('task_args')[1] == 1
assert mindb.get('task_args')[2].data == 67
assert mindb.get('task_kwargs')['c'] == 6
assert mindb.get('task_kwargs')['d'] == 'e'
assert mindb.get('task_kwargs')['f'].data == 89
# check task_result object
tr = TaskResult.objects.get(task_id=tid2)
task_args = pickle.loads(b64decode(tr.task_args))
assert task_args[0] == 'a'
assert task_args[1] == 1
assert task_args[2].data == 67
task_kwargs = pickle.loads(b64decode(tr.task_kwargs))
assert task_kwargs['c'] == 6
assert task_kwargs['d'] == 'e'
assert task_kwargs['f'].data == 89
tid3 = uuid()
try:
raise KeyError('foo')
except KeyError as exception:
self.b.mark_as_failure(tid3, exception)
assert self.b.get_status(tid3) == states.FAILURE
assert isinstance(self.b.get_result(tid3), KeyError)
def _unpack(self, payload, sep=str_to_bytes('\x00\x01')):
raw_payload = b64decode(ensure_bytes(payload))
first_sep = raw_payload.find(sep)
signer = raw_payload[:first_sep]
signer_cert = self._cert_store[signer]
# shift 3 bits right to get signature length
# 2048bit rsa key has a signature length of 256
# 4096bit rsa key has a signature length of 512
sig_len = signer_cert.get_pubkey().key_size >> 3
sep_len = len(sep)
signature_start_position = first_sep + sep_len
signature_end_position = signature_start_position + sig_len
signature = raw_payload[
signature_start_position:signature_end_position
]
v = raw_payload[signature_end_position + sep_len:].split(sep)
return {
'signer': signer,
'signature': signature,
'content_type': bytes_to_str(v[0]),
'content_encoding': bytes_to_str(v[1]),
'body': bytes_to_str(v[2]),
}
def _unpack(self, payload, sep=str_to_bytes('\x00\x01')):
raw_payload = b64decode(ensure_bytes(payload))
first_sep = raw_payload.find(sep)
signer = raw_payload[:first_sep]
signer_cert = self._cert_store[signer]
# shift 3 bits right to get signature length
# 2048bit rsa key has a signature length of 256
# 4096bit rsa key has a signature length of 512
sig_len = signer_cert.get_pubkey().key_size >> 3
sep_len = len(sep)
signature_start_position = first_sep + sep_len
signature_end_position = signature_start_position + sig_len
signature = raw_payload[
signature_start_position:signature_end_position]
v = raw_payload[signature_end_position + sep_len:].split(sep)
return {
'signer': signer,
'signature': signature,
'content_type': bytes_to_str(v[0]),
'content_encoding': bytes_to_str(v[1]),
'body': v[2],
}
def test_backend__pickle_serialization__bytes_result__protocol_1(self):
self.app.conf.result_serializer = 'pickle'
self.app.conf.accept_content = {'pickle', 'json'}
self.b = DatabaseBackend(app=self.app)
tid2 = uuid()
request = self._create_request(
task_id=tid2,
name='my_task',
args=['a', 1, SomeClass(67)],
kwargs={
'c': 6,
'd': 'e',
'f': SomeClass(89)
},
task_protocol=1,
)
result = b'foo'
self.b.mark_as_done(tid2, result, request=request)
mindb = self.b.get_task_meta(tid2)
# check task meta
assert mindb.get('result') == b'foo'
assert mindb.get('task_name') == 'my_task'
assert mindb.get('task_args')[0] == 'a'
assert mindb.get('task_args')[1] == 1
assert mindb.get('task_args')[2].data == 67
assert mindb.get('task_kwargs')['c'] == 6
assert mindb.get('task_kwargs')['d'] == 'e'
assert mindb.get('task_kwargs')['f'].data == 89
# check task_result object
tr = TaskResult.objects.get(task_id=tid2)
task_args = pickle.loads(b64decode(tr.task_args))
assert task_args[0] == 'a'
assert task_args[1] == 1
assert task_args[2].data == 67
task_kwargs = pickle.loads(b64decode(tr.task_kwargs))
assert task_kwargs['c'] == 6
assert task_kwargs['d'] == 'e'
assert task_kwargs['f'].data == 89
def test_backend__pickle_serialization__bytes_result(self):
self.app.conf.result_serializer = 'pickle'
self.app.conf.accept_content = {'pickle', 'json'}
self.b = DatabaseBackend(app=self.app)
tid2 = uuid()
request = self._create_request(
task_id=tid2,
name='my_task',
args=['a', 1, SomeClass(67)],
kwargs={
'c': 6,
'd': 'e',
'f': SomeClass(89)
},
)
result = b'foo'
self.b.mark_as_done(tid2, result, request=request)
mindb = self.b.get_task_meta(tid2)
# check task meta
assert mindb.get('result') == b'foo'
assert mindb.get('task_name') == 'my_task'
assert len(mindb.get('worker')) > 1
assert bool(
re.match(r"\['a', 1, <.*SomeClass object at .*>\]",
mindb.get('task_args')))
assert bool(
re.match(r"{'c': 6, 'd': 'e', 'f': <.*SomeClass object at .*>}",
mindb.get('task_kwargs')))
# check task_result objects
tr = TaskResult.objects.get(task_id=tid2)
task_args = pickle.loads(b64decode(tr.task_args))
task_kwargs = pickle.loads(b64decode(tr.task_kwargs))
assert task_args == mindb.get('task_args')
assert task_kwargs == mindb.get('task_kwargs')
# check async_result
ar = AsyncResult(tid2)
assert ar.args == mindb.get('task_args')
assert ar.kwargs == mindb.get('task_kwargs')
def test_secrets__pickle_serialization(self):
self.app.conf.result_serializer = 'pickle'
self.app.conf.accept_content = {'pickle', 'json'}
self.b = DatabaseBackend(app=self.app)
tid = uuid()
request = self._create_request(
task_id=tid,
name='my_task',
args=['a', 1, 'password'],
kwargs={
'c': 3,
'd': 'e',
'password': '******'
},
argsrepr='argsrepr',
kwargsrepr='kwargsrepr',
)
result = {'foo': 'baz'}
self.b.mark_as_done(tid, result, request=request)
mindb = self.b.get_task_meta(tid)
# check task meta
assert mindb.get('result') == {'foo': 'baz'}
assert mindb.get('task_args') == 'argsrepr'
assert mindb.get('task_kwargs') == 'kwargsrepr'
assert len(mindb.get('worker')) > 1
# check task_result object
tr = TaskResult.objects.get(task_id=tid)
task_args = pickle.loads(b64decode(tr.task_args))
task_kwargs = pickle.loads(b64decode(tr.task_kwargs))
assert task_args == 'argsrepr'
assert task_kwargs == 'kwargsrepr'
# check async_result
ar = AsyncResult(tid)
assert ar.args == mindb.get('task_args')
assert ar.kwargs == mindb.get('task_kwargs')
def _unpack(self, payload, sep=str_to_bytes("\x00\x01")):
raw_payload = b64decode(ensure_bytes(payload))
first_sep = raw_payload.find(sep)
signer = raw_payload[:first_sep]
signer_cert = self._cert_store[signer]
sig_len = signer_cert._cert.get_pubkey().bits() >> 3
signature = raw_payload[first_sep + len(sep) : first_sep + len(sep) + sig_len]
end_of_sig = first_sep + len(sep) + sig_len + len(sep)
v = raw_payload[end_of_sig:].split(sep)
return {
"signer": signer,
"signature": signature,
"content_type": bytes_to_str(v[0]),
"content_encoding": bytes_to_str(v[1]),
"body": bytes_to_str(v[2]),
}
def _unpack(self, payload, sep=str_to_bytes('\x00\x01')):
raw_payload = b64decode(ensure_bytes(payload))
first_sep = raw_payload.find(sep)
signer = raw_payload[:first_sep]
signer_cert = self._cert_store[signer]
sig_len = signer_cert._cert.get_pubkey().bits() >> 3
signature = raw_payload[first_sep + len(sep):first_sep + len(sep) +
sig_len]
end_of_sig = first_sep + len(sep) + sig_len + len(sep)
v = raw_payload[end_of_sig:].split(sep)
return {
'signer': signer,
'signature': signature,
'content_type': bytes_to_str(v[0]),
'content_encoding': bytes_to_str(v[1]),
'body': bytes_to_str(v[2]),
}
def decode_content(self, obj, content):
if content:
if obj.content_encoding == 'binary':
content = b64decode(content)
return self.decode(content)
def test_backend__pickle_serialization__dict_result(self):
self.app.conf.result_serializer = 'pickle'
self.app.conf.accept_content = {'pickle', 'json'}
self.b = DatabaseBackend(app=self.app)
tid2 = uuid()
request = self._create_request(
task_id=tid2,
name='my_task',
args=['a', 1, SomeClass(67)],
kwargs={
'c': 6,
'd': 'e',
'f': SomeClass(89)
},
)
result = {'foo': 'baz', 'bar': SomeClass(12345)}
self.b.mark_as_done(tid2, result, request=request)
mindb = self.b.get_task_meta(tid2)
# check task meta
assert mindb.get('result').get('foo') == 'baz'
assert mindb.get('result').get('bar').data == 12345
assert len(mindb.get('worker')) > 1
assert mindb.get('task_name') == 'my_task'
assert bool(
re.match(r"\['a', 1, <.*SomeClass object at .*>\]",
mindb.get('task_args')))
assert bool(
re.match(r"{'c': 6, 'd': 'e', 'f': <.*SomeClass object at .*>}",
mindb.get('task_kwargs')))
# check task_result object
tr = TaskResult.objects.get(task_id=tid2)
task_args = pickle.loads(b64decode(tr.task_args))
task_kwargs = pickle.loads(b64decode(tr.task_kwargs))
assert task_args == mindb.get('task_args')
assert task_kwargs == mindb.get('task_kwargs')
# check async_result
ar = AsyncResult(tid2)
assert ar.args == mindb.get('task_args')
assert ar.kwargs == mindb.get('task_kwargs')
# check backward compatibility
task_kwargs2 = str(request.kwargs)
task_args2 = str(request.args)
assert tr.task_args != task_args2
assert tr.task_kwargs != task_kwargs2
tr.task_args = task_args2
tr.task_kwargs = task_kwargs2
tr.save()
mindb = self.b.get_task_meta(tid2)
assert bool(
re.match(r"\['a', 1, <.*SomeClass object at .*>\]",
mindb.get('task_args')))
assert bool(
re.match(r"{'c': 6, 'd': 'e', 'f': <.*SomeClass object at .*>}",
mindb.get('task_kwargs')))
ar = AsyncResult(tid2)
assert ar.args == mindb.get('task_args')
assert ar.kwargs == mindb.get('task_kwargs')
tid3 = uuid()
try:
raise KeyError('foo')
except KeyError as exception:
self.b.mark_as_failure(tid3, exception)
assert self.b.get_status(tid3) == states.FAILURE
assert isinstance(self.b.get_result(tid3), KeyError)
def decode_content(self, obj, content):
if content:
if obj.content_encoding == 'binary':
content = b64decode(content)
return self.decode(content)
