def deserialize_object(buffers, g=None):
"""reconstruct an object serialized by serialize_object from data buffers.
Parameters
----------
buffers : list of buffers/bytes
g : globals to be used when uncanning
Returns
-------
(newobj, bufs) : unpacked object, and the list of remaining unused buffers.
"""
bufs = list(buffers)
pobj = bufs.pop(0)
canned = pickle.loads(pobj)
if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:
for c in canned:
_restore_buffers(c, bufs)
newobj = uncan_sequence(canned, g)
elif istype(canned, dict) and len(canned) < MAX_ITEMS:
newobj = {}
for k in sorted(canned):
c = canned[k]
_restore_buffers(c, bufs)
newobj[k] = uncan(c, g)
else:
_restore_buffers(canned, bufs)
newobj = uncan(canned, g)
return newobj, bufs
def unpack_apply_message(bufs, g=None, copy=True):
"""unpack f,args,kwargs from buffers packed by pack_apply_message()
Returns: original f,args,kwargs"""
bufs = list(bufs) # allow us to pop
assert len(bufs) >= 2, "not enough buffers!"
pf = bufs.pop(0)
f = uncan(pickle.loads(pf), g)
pinfo = bufs.pop(0)
info = pickle.loads(pinfo)
arg_bufs, kwarg_bufs = bufs[: info["narg_bufs"]], bufs[info["narg_bufs"] :]
args = []
for _ in range(info["nargs"]):
arg, arg_bufs = deserialize_object(arg_bufs, g)
args.append(arg)
args = tuple(args)
assert not arg_bufs, "Shouldn't be any arg bufs left over"
kwargs = {}
for key in info["kw_keys"]:
kwarg, kwarg_bufs = deserialize_object(kwarg_bufs, g)
kwargs[key] = kwarg
assert not kwarg_bufs, "Shouldn't be any kwarg bufs left over"
return f, args, kwargs
def test_can_partial():
def foo(x, y, z):
return x * y * z
partial_foo = partial(foo, 2, y=5)
canned = can(partial_foo)
assert isinstance(canned, canning.CannedPartial)
dumped = pickle.dumps(canned)
loaded = pickle.loads(dumped)
pfoo2 = uncan(loaded)
assert pfoo2(z=3) == partial_foo(z=3)
def test_can_partial():
def foo(x, y, z):
return x * y * z
partial_foo = partial(foo, 2, y=5)
canned = can(partial_foo)
assert isinstance(canned, canning.CannedPartial)
dumped = pickle.dumps(canned)
loaded = pickle.loads(dumped)
pfoo2 = uncan(loaded)
assert pfoo2(z=3) == partial_foo(z=3)
def test_can_partial_buffers():
def foo(arg1, arg2, kwarg1, kwarg2):
return '%s%s%s%s' % (arg1, arg2[:32], b2a_hex(kwarg1.tobytes()[:32]), kwarg2)
buf1 = os.urandom(1024 * 1024)
buf2 = memoryview(os.urandom(1024 * 1024))
partial_foo = partial(foo, 5, buf1, kwarg1=buf2, kwarg2=10)
canned = can(partial_foo)
assert len(canned.buffers) == 2
assert isinstance(canned, canning.CannedPartial)
buffers, canned.buffers = canned.buffers, []
dumped = pickle.dumps(canned)
loaded = pickle.loads(dumped)
loaded.buffers = buffers
pfoo2 = uncan(loaded)
assert pfoo2() == partial_foo()
def test_can_partial_buffers():
def foo(arg1, arg2, kwarg1, kwarg2):
return f'{arg1}{arg2[:32]}{b2a_hex(kwarg1.tobytes()[:32])}{kwarg2}'
buf1 = os.urandom(1024 * 1024)
buf2 = memoryview(os.urandom(1024 * 1024))
partial_foo = partial(foo, 5, buf1, kwarg1=buf2, kwarg2=10)
canned = can(partial_foo)
assert len(canned.buffers) == 2
assert isinstance(canned, canning.CannedPartial)
buffers, canned.buffers = canned.buffers, []
dumped = pickle.dumps(canned)
loaded = pickle.loads(dumped)
loaded.buffers = buffers
pfoo2 = uncan(loaded)
assert pfoo2() == partial_foo()
def test_uncan_bytes_buffer():
data = b'data'
canned = can(data)
canned.buffers = [memoryview(buf) for buf in canned.buffers]
out = uncan(canned)
assert out == data
def loads(obj):
return uncan(pickle.loads(obj))
def test_uncan_bytes_buffer():
data = b'data'
canned = can(data)
canned.buffers = [memoryview(buf) for buf in canned.buffers]
out = uncan(canned)
assert out == data
def loads(obj):
return uncan(pickle.loads(obj))
def test_uncan_bytes_buffer():
data = b'data'
canned = can(data)
canned.buffers = [memoryview(buf) for buf in canned.buffers]
out = uncan(canned)
nt.assert_equal(out, data)
def test_uncan_bytes_buffer():
data = b'data'
canned = can(data)
canned.buffers = [memoryview(buf) for buf in canned.buffers]
out = uncan(canned)
nt.assert_equal(out, data)
