def deserialize_object(buffers, g=None): """reconstruct an object serialized by serialize_object from data buffers. Parameters ---------- buffers : list of buffers/bytes g : globals to be used when uncanning Returns ------- (newobj, bufs) : unpacked object, and the list of remaining unused buffers. """ bufs = list(buffers) pobj = bufs.pop(0) canned = pickle.loads(pobj) if istype(canned, sequence_types) and len(canned) < MAX_ITEMS: for c in canned: _restore_buffers(c, bufs) newobj = uncan_sequence(canned, g) elif istype(canned, dict) and len(canned) < MAX_ITEMS: newobj = {} for k in sorted(canned): c = canned[k] _restore_buffers(c, bufs) newobj[k] = uncan(c, g) else: _restore_buffers(canned, bufs) newobj = uncan(canned, g) return newobj, bufs
def unpack_apply_message(bufs, g=None, copy=True): """unpack f,args,kwargs from buffers packed by pack_apply_message() Returns: original f,args,kwargs""" bufs = list(bufs) # allow us to pop assert len(bufs) >= 2, "not enough buffers!" pf = bufs.pop(0) f = uncan(pickle.loads(pf), g) pinfo = bufs.pop(0) info = pickle.loads(pinfo) arg_bufs, kwarg_bufs = bufs[: info["narg_bufs"]], bufs[info["narg_bufs"] :] args = [] for _ in range(info["nargs"]): arg, arg_bufs = deserialize_object(arg_bufs, g) args.append(arg) args = tuple(args) assert not arg_bufs, "Shouldn't be any arg bufs left over" kwargs = {} for key in info["kw_keys"]: kwarg, kwarg_bufs = deserialize_object(kwarg_bufs, g) kwargs[key] = kwarg assert not kwarg_bufs, "Shouldn't be any kwarg bufs left over" return f, args, kwargs
def test_can_partial(): def foo(x, y, z): return x * y * z partial_foo = partial(foo, 2, y=5) canned = can(partial_foo) assert isinstance(canned, canning.CannedPartial) dumped = pickle.dumps(canned) loaded = pickle.loads(dumped) pfoo2 = uncan(loaded) assert pfoo2(z=3) == partial_foo(z=3)
def test_can_partial_buffers(): def foo(arg1, arg2, kwarg1, kwarg2): return '%s%s%s%s' % (arg1, arg2[:32], b2a_hex(kwarg1.tobytes()[:32]), kwarg2) buf1 = os.urandom(1024 * 1024) buf2 = memoryview(os.urandom(1024 * 1024)) partial_foo = partial(foo, 5, buf1, kwarg1=buf2, kwarg2=10) canned = can(partial_foo) assert len(canned.buffers) == 2 assert isinstance(canned, canning.CannedPartial) buffers, canned.buffers = canned.buffers, [] dumped = pickle.dumps(canned) loaded = pickle.loads(dumped) loaded.buffers = buffers pfoo2 = uncan(loaded) assert pfoo2() == partial_foo()
def test_can_partial_buffers(): def foo(arg1, arg2, kwarg1, kwarg2): return f'{arg1}{arg2[:32]}{b2a_hex(kwarg1.tobytes()[:32])}{kwarg2}' buf1 = os.urandom(1024 * 1024) buf2 = memoryview(os.urandom(1024 * 1024)) partial_foo = partial(foo, 5, buf1, kwarg1=buf2, kwarg2=10) canned = can(partial_foo) assert len(canned.buffers) == 2 assert isinstance(canned, canning.CannedPartial) buffers, canned.buffers = canned.buffers, [] dumped = pickle.dumps(canned) loaded = pickle.loads(dumped) loaded.buffers = buffers pfoo2 = uncan(loaded) assert pfoo2() == partial_foo()
def test_uncan_bytes_buffer(): data = b'data' canned = can(data) canned.buffers = [memoryview(buf) for buf in canned.buffers] out = uncan(canned) assert out == data
def loads(obj): return uncan(pickle.loads(obj))
def test_uncan_bytes_buffer(): data = b'data' canned = can(data) canned.buffers = [memoryview(buf) for buf in canned.buffers] out = uncan(canned) nt.assert_equal(out, data)