def deserialize_object(buffers, g=None): """reconstruct an object serialized by serialize_object from data buffers. Parameters ---------- bufs : list of buffers/bytes g : globals to be used when uncanning Returns ------- (newobj, bufs) : unpacked object, and the list of remaining unused buffers. """ bufs = list(buffers) pobj = buffer_to_bytes_py2(bufs.pop(0)) canned = pickle.loads(pobj) if istype(canned, sequence_types) and len(canned) < MAX_ITEMS: for c in canned: _restore_buffers(c, bufs) newobj = uncan_sequence(canned, g) elif istype(canned, dict) and len(canned) < MAX_ITEMS: newobj = {} for k in sorted(canned): c = canned[k] _restore_buffers(c, bufs) newobj[k] = uncan(c, g) else: _restore_buffers(canned, bufs) newobj = uncan(canned, g) return newobj, bufs
def test_uncan_function_globals(self): """test that uncanning a module function restores it into its module""" from re import search cf = can(search) csearch = uncan(cf) self.assertEqual(csearch.__module__, search.__module__) self.assertNotEqual(csearch('asd', 'asdf'), None) csearch = uncan(cf, dict(a=5)) self.assertEqual(csearch.__module__, search.__module__) self.assertNotEqual(csearch('asd', 'asdf'), None)
def test_uncan_function_globals(self): """test that uncanning a module function restores it into its module""" from re import search cf = can(search) csearch = uncan(cf) self.assertEqual(csearch.__module__, search.__module__) self.assertNotEqual(csearch("asd", "asdf"), None) csearch = uncan(cf, dict(a=5)) self.assertEqual(csearch.__module__, search.__module__) self.assertNotEqual(csearch("asd", "asdf"), None)
def unpack_apply_message(bufs, g=None, copy=True): """unpack f,args,kwargs from buffers packed by pack_apply_message() Returns: original f,args,kwargs""" bufs = list(bufs) # allow us to pop assert len(bufs) >= 3, "not enough buffers!" if not copy: for i in range(3): bufs[i] = bufs[i].bytes cf = pickle.loads(bufs.pop(0)) sargs = list(pickle.loads(bufs.pop(0))) skwargs = dict(pickle.loads(bufs.pop(0))) # print sargs, skwargs f = uncan(cf, g) for sa in sargs: if sa.data is None: m = bufs.pop(0) if sa.getTypeDescriptor() in ('buffer', 'ndarray'): # always use a buffer, until memoryviews get sorted out sa.data = buffer(m) # disable memoryview support # if copy: # sa.data = buffer(m) # else: # sa.data = m.buffer else: if copy: sa.data = m else: sa.data = m.bytes args = uncanSequence(map(unserialize, sargs), g) kwargs = {} for k in sorted(skwargs.iterkeys()): sa = skwargs[k] if sa.data is None: m = bufs.pop(0) if sa.getTypeDescriptor() in ('buffer', 'ndarray'): # always use a buffer, until memoryviews get sorted out sa.data = buffer(m) # disable memoryview support # if copy: # sa.data = buffer(m) # else: # sa.data = m.buffer else: if copy: sa.data = m else: sa.data = m.bytes kwargs[k] = uncan(unserialize(sa), g) return f,args,kwargs
def unpack_apply_message(bufs, g=None, copy=True): """unpack f,args,kwargs from buffers packed by pack_apply_message() Returns: original f,args,kwargs""" bufs = list(bufs) # allow us to pop assert len(bufs) >= 3, "not enough buffers!" if not copy: for i in range(3): bufs[i] = bufs[i].bytes cf = pickle.loads(bufs.pop(0)) sargs = list(pickle.loads(bufs.pop(0))) skwargs = dict(pickle.loads(bufs.pop(0))) # print sargs, skwargs f = uncan(cf, g) for sa in sargs: if sa.data is None: m = bufs.pop(0) if sa.getTypeDescriptor() in ('buffer', 'ndarray'): # always use a buffer, until memoryviews get sorted out sa.data = buffer(m) # disable memoryview support # if copy: # sa.data = buffer(m) # else: # sa.data = m.buffer else: if copy: sa.data = m else: sa.data = m.bytes args = uncanSequence(map(unserialize, sargs), g) kwargs = {} for k in sorted(skwargs.iterkeys()): sa = skwargs[k] if sa.data is None: m = bufs.pop(0) if sa.getTypeDescriptor() in ('buffer', 'ndarray'): # always use a buffer, until memoryviews get sorted out sa.data = buffer(m) # disable memoryview support # if copy: # sa.data = buffer(m) # else: # sa.data = m.buffer else: if copy: sa.data = m else: sa.data = m.bytes kwargs[k] = uncan(unserialize(sa), g) return f, args, kwargs
def unpack_apply_message(bufs, g=None, copy=True): """unpack f,args,kwargs from buffers packed by pack_apply_message() Returns: original f,args,kwargs""" bufs = list(bufs) # allow us to pop assert len(bufs) >= 2, "not enough buffers!" pf = buffer_to_bytes_py2(bufs.pop(0)) f = uncan(pickle.loads(pf), g) pinfo = buffer_to_bytes_py2(bufs.pop(0)) info = pickle.loads(pinfo) arg_bufs, kwarg_bufs = bufs[:info['narg_bufs']], bufs[info['narg_bufs']:] args = [] for i in range(info['nargs']): arg, arg_bufs = deserialize_object(arg_bufs, g) args.append(arg) args = tuple(args) assert not arg_bufs, "Shouldn't be any arg bufs left over" kwargs = {} for key in info['kw_keys']: kwarg, kwarg_bufs = deserialize_object(kwarg_bufs, g) kwargs[key] = kwarg assert not kwarg_bufs, "Shouldn't be any kwarg bufs left over" return f, args, kwargs
def unpack_apply_message(bufs, g=None, copy=True): """unpack f,args,kwargs from buffers packed by pack_apply_message() Returns: original f,args,kwargs""" bufs = list(bufs) # allow us to pop assert len(bufs) >= 2, "not enough buffers!" if not copy: for i in range(2): bufs[i] = bufs[i].bytes f = uncan(pickle.loads(bufs.pop(0)), g) info = pickle.loads(bufs.pop(0)) arg_bufs, kwarg_bufs = bufs[:info['narg_bufs']], bufs[info['narg_bufs']:] args = [] for i in range(info['nargs']): arg, arg_bufs = deserialize_object(arg_bufs, g) args.append(arg) args = tuple(args) assert not arg_bufs, "Shouldn't be any arg bufs left over" kwargs = {} for key in info['kw_keys']: kwarg, kwarg_bufs = deserialize_object(kwarg_bufs, g) kwargs[key] = kwarg assert not kwarg_bufs, "Shouldn't be any kwarg bufs left over" return f,args,kwargs
def unserialize_object(bufs): """reconstruct an object serialized by serialize_object from data buffers.""" bufs = list(bufs) sobj = pickle.loads(bufs.pop(0)) if isinstance(sobj, (list, tuple)): for s in sobj: if s.data is None: s.data = bufs.pop(0) return uncanSequence(map(unserialize, sobj)), bufs elif isinstance(sobj, dict): newobj = {} for k in sorted(sobj.iterkeys()): s = sobj[k] if s.data is None: s.data = bufs.pop(0) newobj[k] = uncan(unserialize(s)) return newobj, bufs else: if sobj.data is None: sobj.data = bufs.pop(0) return uncan(unserialize(sobj)), bufs
def _require(*modules, **mapping): """Helper for @require decorator.""" from IPython.parallel.error import UnmetDependency from IPython.utils.pickleutil import uncan user_ns = globals() for name in modules: try: exec 'import %s' % name in user_ns except ImportError: raise UnmetDependency(name) for name, cobj in mapping.items(): user_ns[name] = uncan(cobj, user_ns) return True
def unserialize_object(buffers, g=None): """reconstruct an object serialized by serialize_object from data buffers. Parameters ---------- bufs : list of buffers/bytes g : globals to be used when uncanning Returns ------- (newobj, bufs) : unpacked object, and the list of remaining unused buffers. """ bufs = list(buffers) pobj = bufs.pop(0) if not isinstance(pobj, bytes): # a zmq message pobj = bytes(pobj) canned = pickle.loads(pobj) if isinstance(canned, (list, tuple)) and len(canned) < MAX_ITEMS: for c in canned: _restore_buffers(c, bufs) newobj = uncan_sequence(canned, g) elif isinstance(canned, dict) and len(canned) < MAX_ITEMS: newobj = {} for k in sorted(canned.iterkeys()): c = canned[k] _restore_buffers(c, bufs) newobj[k] = uncan(c, g) else: _restore_buffers(canned, bufs) newobj = uncan(canned, g) return newobj, bufs
def unpack_apply_message(bufs, g=None, copy=True): """unpack f,args,kwargs from buffers packed by pack_apply_message() Returns: original f,args,kwargs""" bufs = list(bufs) # allow us to pop assert len(bufs) >= 3, "not enough buffers!" if not copy: for i in range(3): bufs[i] = bufs[i].bytes f = uncan(pickle.loads(bufs.pop(0)), g) # sargs = bufs.pop(0) # pop kwargs out, so first n-elements are args, serialized skwargs = bufs.pop(1) args, bufs = unserialize_object(bufs, g) # put skwargs back in as the first element bufs.insert(0, skwargs) kwargs, bufs = unserialize_object(bufs, g) assert not bufs, "Shouldn't be any data left over" return f,args,kwargs
def loads(obj): return uncan(pickle.loads(obj))
def cancan(self, f): """decorator to pass through canning into self.user_ns""" return uncan(can(f), self.user_ns)
def run_roundtrip(self, obj): o = uncan(can(obj)) assert o == obj, "failed assertion: %r == %r" % (o, obj)
def run_roundtrip(self, obj): o = uncan(can(obj)) assert o == obj, "failed assertion: %r == %r"%(o,obj)