def save_classobj(pickler, obj):
if obj.__module__ == '__main__':
pickler.save_reduce(ClassType,
(obj.__name__, obj.__bases__, obj.__dict__),
obj=obj)
else:
Pickler.save_global(pickler, obj)
def save_type(pickler, obj):
if obj in _typemap:
log.info("T1: %s" % obj)
pickler.save_reduce(_load_type, (_typemap[obj], ), obj=obj)
elif obj.__module__ == '__main__':
if type(obj) == type:
# we are pickling the interpreter
if is_dill(pickler) and pickler._session:
log.info("T2: %s" % obj)
_dict = _dict_from_dictproxy(obj.__dict__)
else: # otherwise punt to StockPickler
log.info("T5: %s" % obj)
StockPickler.save_global(pickler, obj)
return
else:
log.info("T3: %s" % obj)
_dict = obj.__dict__
#print _dict
#print "%s\n%s" % (type(obj), obj.__name__)
#print "%s\n%s" % (obj.__bases__, obj.__dict__)
pickler.save_reduce(_create_type,
(type(obj), obj.__name__, obj.__bases__, _dict),
obj=obj)
else:
log.info("T4: %s" % obj)
#print obj.__dict__
#print "%s\n%s" % (type(obj), obj.__name__)
#print "%s\n%s" % (obj.__bases__, obj.__dict__)
StockPickler.save_global(pickler, obj)
return
def save_function(pickler, obj):
if not _locate_function(obj):
pickler.save_reduce(FunctionType, (obj.func_code, obj.func_globals,
obj.func_name, obj.func_defaults,
obj.func_closure), obj=obj)
else:
Pickler.save_global(pickler, obj)
def save_type(pickler, obj):
if obj in _typemap:
pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj)
elif obj.__module__ == '__main__':
if type(obj) == type:
# we are pickling the interpreter
if is_dill(pickler) and pickler._session:
_dict = _dict_from_dictproxy(obj.__dict__)
else: # otherwise punt to StockPickler
StockPickler.save_global(pickler, obj)
return
else:
_dict = obj.__dict__
#print _dict
#print "%s\n%s" % (type(obj), obj.__name__)
#print "%s\n%s" % (obj.__bases__, obj.__dict__)
pickler.save_reduce(_create_type, (type(obj), obj.__name__,
obj.__bases__, _dict), obj=obj)
else:
#print obj.__dict__
#print "%s\n%s" % (type(obj), obj.__name__)
#print "%s\n%s" % (obj.__bases__, obj.__dict__)
StockPickler.save_global(pickler, obj)
return
def save_builtin_method(pickler, obj):
if obj.__self__ is not None:
log.info("B1: %s" % obj)
pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
else:
log.info("B2: %s" % obj)
StockPickler.save_global(pickler, obj)
return
def save_builtin_method(pickler, obj):
if obj.__self__ is not None:
if _DEBUG: print "B1: %s" % obj
pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
else:
if _DEBUG: print "B2: %s" % obj
StockPickler.save_global(pickler, obj)
return
def save_builtin_method(pickler, obj):
if obj.__self__ is not None:
pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
else:
StockPickler.save_global(pickler, obj)
return
def save_function(pickler, obj):
if not _locate_function(obj):
pickler.save_reduce(FunctionType,
(obj.func_code, obj.func_globals, obj.func_name,
obj.func_defaults, obj.func_closure),
obj=obj)
else:
Pickler.save_global(pickler, obj)
def save_builtin_method(pickler, obj):
if obj.__self__ is not None:
log.info("B1: %s" % obj)
pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
else:
log.info("B2: %s" % obj)
StockPickler.save_global(pickler, obj)
return
def save_type(pickler, obj):
if obj in _typemap:
pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj)
elif obj.__module__ == '__main__':
pickler.save_reduce(_create_type, (type(obj), obj.__name__,
obj.__bases__, obj.__dict__),
obj=obj)
else:
Pickler.save_global(pickler, obj)
def save_classobj(pickler, obj):
if obj.__module__ == '__main__':
pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__,
obj.__dict__), obj=obj)
#XXX: or obj.__dict__.copy()), obj=obj) ?
else:
StockPickler.save_global(pickler, obj)
return
def save_type(pickler, obj):
if obj in _typemap:
pickler.save_reduce(_load_type, (_typemap[obj], ), obj=obj)
elif obj.__module__ == '__main__':
pickler.save_reduce(
_create_type,
(type(obj), obj.__name__, obj.__bases__, obj.__dict__),
obj=obj)
else:
Pickler.save_global(pickler, obj)
def save_function(pickler, obj):
if not _locate_function(obj): #, pickler._session):
log.info("F1: %s" % obj)
pickler.save_reduce(FunctionType, (obj.func_code, obj.func_globals,
obj.func_name, obj.func_defaults,
obj.func_closure), obj=obj)
else:
log.info("F2: %s" % obj)
StockPickler.save_global(pickler, obj)
return
def save_classobj(pickler, obj):
if obj.__module__ == '__main__':
log.info("C1: %s" % obj)
pickler.save_reduce(ClassType,
(obj.__name__, obj.__bases__, obj.__dict__),
obj=obj)
#XXX: or obj.__dict__.copy()), obj=obj) ?
else:
log.info("C2: %s" % obj)
StockPickler.save_global(pickler, obj)
return
def save_function(pickler, obj):
if not _locate_function(obj): #, pickler._session):
log.info("F1: %s" % obj)
pickler.save_reduce(FunctionType,
(obj.func_code, obj.func_globals, obj.func_name,
obj.func_defaults, obj.func_closure),
obj=obj)
else:
log.info("F2: %s" % obj)
StockPickler.save_global(pickler, obj)
return
def save_global(self, obj, name=None, pack=struct.pack):
"""
Save a "global".
The name of this method is somewhat misleading: all types get
dispatched here.
"""
if obj is type(None):
return self.save_reduce(type, (None, ), obj=obj)
elif obj is type(Ellipsis):
return self.save_reduce(type, (Ellipsis, ), obj=obj)
elif obj is type(NotImplemented):
return self.save_reduce(type, (NotImplemented, ), obj=obj)
if obj.__module__ == "__main__":
return self.save_dynamic_class(obj)
try:
return Pickler.save_global(self, obj, name=name)
except Exception:
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type,
(_BUILTIN_TYPE_NAMES[obj], ),
obj=obj)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
return self.save_dynamic_class(obj)
raise
def save_global(self, obj, name=None, pack=struct.pack):
"""
Save a "global".
The name of this method is somewhat misleading: all types get
dispatched here.
"""
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
self.save_dynamic_class(obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack):
"""
Save a "global".
The name of this method is somewhat misleading: all types get
dispatched here.
"""
if obj is type(None):
return self.save_reduce(type, (None,), obj=obj)
elif obj is type(Ellipsis):
return self.save_reduce(type, (Ellipsis,), obj=obj)
elif obj is type(NotImplemented):
return self.save_reduce(type, (NotImplemented,), obj=obj)
if obj.__module__ == "__main__":
return self.save_dynamic_class(obj)
try:
return Pickler.save_global(self, obj, name=name)
except Exception:
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(
_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
return self.save_dynamic_class(obj)
raise
def save_global(self, obj, name=None, pack=struct.pack):
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
self.save(_load_class)
self.save_reduce(typ, (obj.__name__, obj.__bases__, {"__doc__": obj.__doc__}), obj=obj)
d.pop('__doc__', None)
# handle property and staticmethod
dd = {}
for k, v in d.items():
if isinstance(v, property):
k = ('property', k)
v = (v.fget, v.fset, v.fdel, v.__doc__)
elif isinstance(v, staticmethod) and hasattr(v, '__func__'):
k = ('staticmethod', k)
v = v.__func__
elif isinstance(v, classmethod) and hasattr(v, '__func__'):
k = ('classmethod', k)
v = v.__func__
dd[k] = v
self.save(dd)
self.write(pickle.TUPLE2)
self.write(pickle.REDUCE)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_function(self, obj, name=None):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
if _is_global(obj, name=name):
return Pickler.save_global(self, obj, name=name)
elif PYPY and isinstance(obj.__code__, builtin_code_type):
return self.save_pypy_builtin_func(obj)
else:
return self.save_function_tuple(obj)
def save_global(self, obj, name=None, pack=struct.pack):
"""
Save a "global".
The name of this method is somewhat misleading: all types get
dispatched here.
"""
if obj is type(None):
return self.save_reduce(type, (None,), obj=obj)
elif obj is type(Ellipsis):
return self.save_reduce(type, (Ellipsis,), obj=obj)
elif obj is type(NotImplemented):
return self.save_reduce(type, (NotImplemented,), obj=obj)
elif obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(
_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
elif name is not None:
Pickler.save_global(self, obj, name=name)
elif not _is_global(obj, name=name):
self.save_dynamic_class(obj)
else:
Pickler.save_global(self, obj, name=name)
def save_builtin_function_or_method(self, obj):
is_bound = getattr(obj, '__self__', None) is not None
if is_bound:
# obj is a bound builtin method.
rv = (getattr, (obj.__self__, obj.__name__))
return self.save_reduce(obj=obj, *rv)
is_unbound = hasattr(obj, '__objclass__')
if is_unbound:
# obj is an unbound builtin method (accessed from its class)
rv = (getattr, (obj.__objclass__, obj.__name__))
return self.save_reduce(obj=obj, *rv)
# Otherwise, obj is not a method, but a function. Fallback to
# default pickling by attribute.
return Pickler.save_global(self, obj)
def save_global(self, obj, name=None, pack=struct.pack):
"""
Save a "global".
The name of this method is somewhat misleading: all types get
dispatched here.
"""
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type,
(_BUILTIN_TYPE_NAMES[obj], ),
obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = '__main__'
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, _class_type)):
self.save_dynamic_class(obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack):
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type,
(_BUILTIN_TYPE_NAMES[obj], ),
obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack):
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack):
"""
Save a "global".
The name of this method is somewhat misleading: all types get
dispatched here.
"""
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = '__main__'
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
self.save_dynamic_class(obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_numpy_ufunc(pickler, obj):
log.info("Nu: %s" % obj)
StockPickler.save_global(pickler, obj)
return
def save_builtin_method(pickler, obj):
if obj.__self__ is not None:
pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
else:
Pickler.save_global(pickler, obj)
def save_numpy_ufunc(pickler, obj):
StockPickler.save_global(pickler, obj)
return
def save_global(self, obj, name=None, pack=struct.pack):
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type,
(_BUILTIN_TYPE_NAMES[obj], ),
obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
self.save(_load_class)
self.save_reduce(typ, (obj.__name__, obj.__bases__, {
"__doc__": obj.__doc__
}),
obj=obj)
d.pop('__doc__', None)
# handle property and staticmethod
dd = {}
for k, v in d.items():
if isinstance(v, property):
k = ('property', k)
v = (v.fget, v.fset, v.fdel, v.__doc__)
elif isinstance(v, staticmethod) and hasattr(v, '__func__'):
k = ('staticmethod', k)
v = v.__func__
elif isinstance(v, classmethod) and hasattr(v, '__func__'):
k = ('classmethod', k)
v = v.__func__
dd[k] = v
self.save(dd)
self.write(pickle.TUPLE2)
self.write(pickle.REDUCE)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack): # pylint: disable=too-many-branches
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type,
(_BUILTIN_TYPE_NAMES[obj], ),
obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = '__main__'
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
# workaround for namedtuple (hijacked by PySpark)
if getattr(obj, '_is_namedtuple_', False):
self.save_reduce(_load_namedtuple, (obj.__name__, obj._fields))
return
self.save(_load_class)
self.save_reduce(typ, (obj.__name__, obj.__bases__, {
"__doc__": obj.__doc__
}),
obj=obj)
d.pop('__doc__', None)
# handle property and staticmethod
dd = {}
for k, v in d.items():
if isinstance(v, property):
k = ('property', k)
v = (v.fget, v.fset, v.fdel, v.__doc__)
elif isinstance(v, staticmethod) and hasattr(v, '__func__'):
k = ('staticmethod', k)
v = v.__func__
elif isinstance(v, classmethod) and hasattr(v, '__func__'):
k = ('classmethod', k)
v = v.__func__
dd[k] = v
self.save(dd)
self.write(pickle.TUPLE2)
self.write(pickle.REDUCE)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack): # pylint: disable=too-many-branches
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = '__main__'
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
# workaround for namedtuple (hijacked by PySpark)
if getattr(obj, '_is_namedtuple_', False):
self.save_reduce(_load_namedtuple, (obj.__name__, obj._fields))
return
self.save(_load_class)
self.save_reduce(typ, (obj.__name__, obj.__bases__, {"__doc__": obj.__doc__}), obj=obj)
d.pop('__doc__', None)
# handle property and staticmethod
dd = {}
for k, v in d.items():
if isinstance(v, property):
k = ('property', k)
v = (v.fget, v.fset, v.fdel, v.__doc__)
elif isinstance(v, staticmethod) and hasattr(v, '__func__'):
k = ('staticmethod', k)
v = v.__func__
elif isinstance(v, classmethod) and hasattr(v, '__func__'):
k = ('classmethod', k)
v = v.__func__
dd[k] = v
self.save(dd)
self.write(pickle.TUPLE2)
self.write(pickle.REDUCE)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_classobj(pickler, obj):
if obj.__module__ == '__main__':
pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__,
obj.__dict__), obj=obj)
else:
Pickler.save_global(pickler, obj)
