def save_classobj(pickler, obj): if obj.__module__ == '__main__': pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__, obj.__dict__), obj=obj) else: Pickler.save_global(pickler, obj)
def save_type(pickler, obj): if obj in _typemap: log.info("T1: %s" % obj) pickler.save_reduce(_load_type, (_typemap[obj], ), obj=obj) elif obj.__module__ == '__main__': if type(obj) == type: # we are pickling the interpreter if is_dill(pickler) and pickler._session: log.info("T2: %s" % obj) _dict = _dict_from_dictproxy(obj.__dict__) else: # otherwise punt to StockPickler log.info("T5: %s" % obj) StockPickler.save_global(pickler, obj) return else: log.info("T3: %s" % obj) _dict = obj.__dict__ #print _dict #print "%s\n%s" % (type(obj), obj.__name__) #print "%s\n%s" % (obj.__bases__, obj.__dict__) pickler.save_reduce(_create_type, (type(obj), obj.__name__, obj.__bases__, _dict), obj=obj) else: log.info("T4: %s" % obj) #print obj.__dict__ #print "%s\n%s" % (type(obj), obj.__name__) #print "%s\n%s" % (obj.__bases__, obj.__dict__) StockPickler.save_global(pickler, obj) return
def save_function(pickler, obj): if not _locate_function(obj): pickler.save_reduce(FunctionType, (obj.func_code, obj.func_globals, obj.func_name, obj.func_defaults, obj.func_closure), obj=obj) else: Pickler.save_global(pickler, obj)
def save_type(pickler, obj): if obj in _typemap: pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj) elif obj.__module__ == '__main__': if type(obj) == type: # we are pickling the interpreter if is_dill(pickler) and pickler._session: _dict = _dict_from_dictproxy(obj.__dict__) else: # otherwise punt to StockPickler StockPickler.save_global(pickler, obj) return else: _dict = obj.__dict__ #print _dict #print "%s\n%s" % (type(obj), obj.__name__) #print "%s\n%s" % (obj.__bases__, obj.__dict__) pickler.save_reduce(_create_type, (type(obj), obj.__name__, obj.__bases__, _dict), obj=obj) else: #print obj.__dict__ #print "%s\n%s" % (type(obj), obj.__name__) #print "%s\n%s" % (obj.__bases__, obj.__dict__) StockPickler.save_global(pickler, obj) return
def save_builtin_method(pickler, obj): if obj.__self__ is not None: log.info("B1: %s" % obj) pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) else: log.info("B2: %s" % obj) StockPickler.save_global(pickler, obj) return
def save_builtin_method(pickler, obj): if obj.__self__ is not None: if _DEBUG: print "B1: %s" % obj pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) else: if _DEBUG: print "B2: %s" % obj StockPickler.save_global(pickler, obj) return
def save_builtin_method(pickler, obj): if obj.__self__ is not None: pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) else: StockPickler.save_global(pickler, obj) return
def save_function(pickler, obj): if not _locate_function(obj): pickler.save_reduce(FunctionType, (obj.func_code, obj.func_globals, obj.func_name, obj.func_defaults, obj.func_closure), obj=obj) else: Pickler.save_global(pickler, obj)
def save_builtin_method(pickler, obj): if obj.__self__ is not None: log.info("B1: %s" % obj) pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) else: log.info("B2: %s" % obj) StockPickler.save_global(pickler, obj) return
def save_type(pickler, obj): if obj in _typemap: pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj) elif obj.__module__ == '__main__': pickler.save_reduce(_create_type, (type(obj), obj.__name__, obj.__bases__, obj.__dict__), obj=obj) else: Pickler.save_global(pickler, obj)
def save_classobj(pickler, obj): if obj.__module__ == '__main__': pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__, obj.__dict__), obj=obj) #XXX: or obj.__dict__.copy()), obj=obj) ? else: StockPickler.save_global(pickler, obj) return
def save_type(pickler, obj): if obj in _typemap: pickler.save_reduce(_load_type, (_typemap[obj], ), obj=obj) elif obj.__module__ == '__main__': pickler.save_reduce( _create_type, (type(obj), obj.__name__, obj.__bases__, obj.__dict__), obj=obj) else: Pickler.save_global(pickler, obj)
def save_function(pickler, obj): if not _locate_function(obj): #, pickler._session): log.info("F1: %s" % obj) pickler.save_reduce(FunctionType, (obj.func_code, obj.func_globals, obj.func_name, obj.func_defaults, obj.func_closure), obj=obj) else: log.info("F2: %s" % obj) StockPickler.save_global(pickler, obj) return
def save_classobj(pickler, obj): if obj.__module__ == '__main__': log.info("C1: %s" % obj) pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__, obj.__dict__), obj=obj) #XXX: or obj.__dict__.copy()), obj=obj) ? else: log.info("C2: %s" % obj) StockPickler.save_global(pickler, obj) return
def save_function(pickler, obj): if not _locate_function(obj): #, pickler._session): log.info("F1: %s" % obj) pickler.save_reduce(FunctionType, (obj.func_code, obj.func_globals, obj.func_name, obj.func_defaults, obj.func_closure), obj=obj) else: log.info("F2: %s" % obj) StockPickler.save_global(pickler, obj) return
def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj is type(None): return self.save_reduce(type, (None, ), obj=obj) elif obj is type(Ellipsis): return self.save_reduce(type, (Ellipsis, ), obj=obj) elif obj is type(NotImplemented): return self.save_reduce(type, (NotImplemented, ), obj=obj) if obj.__module__ == "__main__": return self.save_dynamic_class(obj) try: return Pickler.save_global(self, obj, name=name) except Exception: if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): return self.save_dynamic_class(obj) raise
def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): self.save_dynamic_class(obj) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj is type(None): return self.save_reduce(type, (None,), obj=obj) elif obj is type(Ellipsis): return self.save_reduce(type, (Ellipsis,), obj=obj) elif obj is type(NotImplemented): return self.save_reduce(type, (NotImplemented,), obj=obj) if obj.__module__ == "__main__": return self.save_dynamic_class(obj) try: return Pickler.save_global(self, obj, name=name) except Exception: if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce( _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): return self.save_dynamic_class(obj) raise
def save_global(self, obj, name=None, pack=struct.pack): if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ self.save(_load_class) self.save_reduce(typ, (obj.__name__, obj.__bases__, {"__doc__": obj.__doc__}), obj=obj) d.pop('__doc__', None) # handle property and staticmethod dd = {} for k, v in d.items(): if isinstance(v, property): k = ('property', k) v = (v.fget, v.fset, v.fdel, v.__doc__) elif isinstance(v, staticmethod) and hasattr(v, '__func__'): k = ('staticmethod', k) v = v.__func__ elif isinstance(v, classmethod) and hasattr(v, '__func__'): k = ('classmethod', k) v = v.__func__ dd[k] = v self.save(dd) self.write(pickle.TUPLE2) self.write(pickle.REDUCE) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ if _is_global(obj, name=name): return Pickler.save_global(self, obj, name=name) elif PYPY and isinstance(obj.__code__, builtin_code_type): return self.save_pypy_builtin_func(obj) else: return self.save_function_tuple(obj)
def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj is type(None): return self.save_reduce(type, (None,), obj=obj) elif obj is type(Ellipsis): return self.save_reduce(type, (Ellipsis,), obj=obj) elif obj is type(NotImplemented): return self.save_reduce(type, (NotImplemented,), obj=obj) elif obj in _BUILTIN_TYPE_NAMES: return self.save_reduce( _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) elif name is not None: Pickler.save_global(self, obj, name=name) elif not _is_global(obj, name=name): self.save_dynamic_class(obj) else: Pickler.save_global(self, obj, name=name)
def save_builtin_function_or_method(self, obj): is_bound = getattr(obj, '__self__', None) is not None if is_bound: # obj is a bound builtin method. rv = (getattr, (obj.__self__, obj.__name__)) return self.save_reduce(obj=obj, *rv) is_unbound = hasattr(obj, '__objclass__') if is_unbound: # obj is an unbound builtin method (accessed from its class) rv = (getattr, (obj.__objclass__, obj.__name__)) return self.save_reduce(obj=obj, *rv) # Otherwise, obj is not a method, but a function. Fallback to # default pickling by attribute. return Pickler.save_global(self, obj)
def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = '__main__' if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, _class_type)): self.save_dynamic_class(obj) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack): if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack): if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = '__main__' if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): self.save_dynamic_class(obj) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_numpy_ufunc(pickler, obj): log.info("Nu: %s" % obj) StockPickler.save_global(pickler, obj) return
def save_builtin_method(pickler, obj): if obj.__self__ is not None: pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) else: Pickler.save_global(pickler, obj)
def save_numpy_ufunc(pickler, obj): StockPickler.save_global(pickler, obj) return
def save_global(self, obj, name=None, pack=struct.pack): if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ self.save(_load_class) self.save_reduce(typ, (obj.__name__, obj.__bases__, { "__doc__": obj.__doc__ }), obj=obj) d.pop('__doc__', None) # handle property and staticmethod dd = {} for k, v in d.items(): if isinstance(v, property): k = ('property', k) v = (v.fget, v.fset, v.fdel, v.__doc__) elif isinstance(v, staticmethod) and hasattr(v, '__func__'): k = ('staticmethod', k) v = v.__func__ elif isinstance(v, classmethod) and hasattr(v, '__func__'): k = ('classmethod', k) v = v.__func__ dd[k] = v self.save(dd) self.write(pickle.TUPLE2) self.write(pickle.REDUCE) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack): # pylint: disable=too-many-branches if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = '__main__' if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ # workaround for namedtuple (hijacked by PySpark) if getattr(obj, '_is_namedtuple_', False): self.save_reduce(_load_namedtuple, (obj.__name__, obj._fields)) return self.save(_load_class) self.save_reduce(typ, (obj.__name__, obj.__bases__, { "__doc__": obj.__doc__ }), obj=obj) d.pop('__doc__', None) # handle property and staticmethod dd = {} for k, v in d.items(): if isinstance(v, property): k = ('property', k) v = (v.fget, v.fset, v.fdel, v.__doc__) elif isinstance(v, staticmethod) and hasattr(v, '__func__'): k = ('staticmethod', k) v = v.__func__ elif isinstance(v, classmethod) and hasattr(v, '__func__'): k = ('classmethod', k) v = v.__func__ dd[k] = v self.save(dd) self.write(pickle.TUPLE2) self.write(pickle.REDUCE) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack): # pylint: disable=too-many-branches if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = '__main__' if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ # workaround for namedtuple (hijacked by PySpark) if getattr(obj, '_is_namedtuple_', False): self.save_reduce(_load_namedtuple, (obj.__name__, obj._fields)) return self.save(_load_class) self.save_reduce(typ, (obj.__name__, obj.__bases__, {"__doc__": obj.__doc__}), obj=obj) d.pop('__doc__', None) # handle property and staticmethod dd = {} for k, v in d.items(): if isinstance(v, property): k = ('property', k) v = (v.fget, v.fset, v.fdel, v.__doc__) elif isinstance(v, staticmethod) and hasattr(v, '__func__'): k = ('staticmethod', k) v = v.__func__ elif isinstance(v, classmethod) and hasattr(v, '__func__'): k = ('classmethod', k) v = v.__func__ dd[k] = v self.save(dd) self.write(pickle.TUPLE2) self.write(pickle.REDUCE) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_classobj(pickler, obj): if obj.__module__ == '__main__': pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__, obj.__dict__), obj=obj) else: Pickler.save_global(pickler, obj)