def unpack_apply_message(bufs, g=None, copy=True):
"""Unpack f,args,kwargs from buffers packed by pack_apply_message().
Returns: original f,args,kwargs
"""
bufs = list(bufs) # allow us to pop
assert len(bufs) >= 2, "not enough buffers!"
pf = buffer_to_bytes_py2(bufs.pop(0))
f = uncan(pickle.loads(pf), g)
pinfo = buffer_to_bytes_py2(bufs.pop(0))
info = pickle.loads(pinfo)
arg_bufs, kwarg_bufs = bufs[:info['narg_bufs']], bufs[info['narg_bufs']:]
args = []
for i in range(info['nargs']):
arg, arg_bufs = deserialize_object(arg_bufs, g)
args.append(arg)
args = tuple(args)
assert not arg_bufs, "Shouldn't be any arg bufs left over"
kwargs = {}
for key in info['kw_keys']:
kwarg, kwarg_bufs = deserialize_object(kwarg_bufs, g)
kwargs[key] = kwarg
assert not kwarg_bufs, "Shouldn't be any kwarg bufs left over"
return f, args, kwargs
def unpack_apply_message(bufs, g=None, copy=True):
"""unpack f,args,kwargs from buffers packed by pack_apply_message()
Returns: original f,args,kwargs"""
bufs = list(bufs) # allow us to pop
assert len(bufs) >= 2, "not enough buffers!"
pf = buffer_to_bytes_py2(bufs.pop(0))
f = uncan(pickle.loads(pf), g)
pinfo = buffer_to_bytes_py2(bufs.pop(0))
info = pickle.loads(pinfo)
arg_bufs, kwarg_bufs = bufs[:info['narg_bufs']], bufs[info['narg_bufs']:]
args = []
for i in range(info['nargs']):
arg, arg_bufs = deserialize_object(arg_bufs, g)
args.append(arg)
args = tuple(args)
assert not arg_bufs, "Shouldn't be any arg bufs left over"
kwargs = {}
for key in info['kw_keys']:
kwarg, kwarg_bufs = deserialize_object(kwarg_bufs, g)
kwargs[key] = kwarg
assert not kwarg_bufs, "Shouldn't be any kwarg bufs left over"
return f,args,kwargs
def deserialize_object(buffers, g=None):
"""reconstruct an object serialized by serialize_object from data buffers.
Parameters
----------
bufs : list of buffers/bytes
g : globals to be used when uncanning
Returns
-------
(newobj, bufs) : unpacked object, and the list of remaining unused buffers.
"""
bufs = list(buffers)
pobj = buffer_to_bytes_py2(bufs.pop(0))
canned = pickle.loads(pobj)
if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:
for c in canned:
_restore_buffers(c, bufs)
newobj = uncan_sequence(canned, g)
elif istype(canned, dict) and len(canned) < MAX_ITEMS:
newobj = {}
for k in sorted(canned):
c = canned[k]
_restore_buffers(c, bufs)
newobj[k] = uncan(c, g)
else:
_restore_buffers(canned, bufs)
newobj = uncan(canned, g)
return newobj, bufs
def deserialize_object(buffers, g=None):
"""reconstruct an object serialized by serialize_object from data buffers.
Parameters
----------
bufs : list of buffers/bytes
g : globals to be used when uncanning
Returns
-------
(newobj, bufs) : unpacked object, and the list of remaining unused buffers.
"""
bufs = list(buffers)
pobj = buffer_to_bytes_py2(bufs.pop(0))
canned = pickle.loads(pobj)
if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:
for c in canned:
_restore_buffers(c, bufs)
newobj = uncan_sequence(canned, g)
elif istype(canned, dict) and len(canned) < MAX_ITEMS:
newobj = {}
for k in sorted(canned):
c = canned[k]
_restore_buffers(c, bufs)
newobj[k] = uncan(c, g)
else:
_restore_buffers(canned, bufs)
newobj = uncan(canned, g)
return newobj, bufs
def get_object(self, g=None):
from numpy import frombuffer
data = self.buffers[0]
if self.pickled:
# we just pickled it
return pickle.loads(buffer_to_bytes_py2(data))
else:
if not py3compat.PY3 and isinstance(data, memoryview):
# frombuffer doesn't accept memoryviews on Python 2,
# so cast to old-style buffer
data = buffer(data.tobytes())
return frombuffer(data, dtype=self.dtype).reshape(self.shape)
def get_object(self, g=None):
from numpy import frombuffer
data = self.buffers[0]
if self.pickled:
# we just pickled it
return pickle.loads(buffer_to_bytes_py2(data))
else:
if not py3compat.PY3 and isinstance(data, memoryview):
# frombuffer doesn't accept memoryviews on Python 2,
# so cast to old-style buffer
data = buffer(data.tobytes())
return frombuffer(data, dtype=self.dtype).reshape(self.shape)