def _whichmodule(obj, name):
"""Find the module an object belongs to.
This function differs from ``pickle.whichmodule`` in two ways:
- it does not mangle the cases where obj's module is __main__ and obj was
not found in any module.
- Errors arising during module introspection are ignored, as those errors
are considered unwanted side effects.
"""
module_name = _get_module_attr(obj)
if module_name is not None:
return module_name
# Protect the iteration by using a copy of sys.modules against dynamic
# modules that trigger imports of other modules upon calls to getattr or
# other threads importing at the same time.
for module_name, module in sys.modules.copy().items():
# Some modules such as coverage can inject non-module objects inside
# sys.modules
if (module_name == '__main__' or module is None
or not isinstance(module, types.ModuleType)):
continue
try:
if _getattribute(module, name)[0] is obj:
return module_name
except Exception:
pass
return None
def _lookup_module_and_qualname(obj, name=None): # если и ф-и имя и модуль
if name is None:
name = getattr(obj, "__qualname__", None)
if name is None:
name = getattr(obj, "__name__", None)
module_name = _whichmodule(obj, name)
if module_name is None:
return None
if module_name == "__main__":
return None
module = sys.modules.get(module_name, None)
if module is None:
return None
try:
obj2, parent = pickle._getattribute(module, name)
except AttributeError:
return None
if obj2 is not obj:
return None
return module, name
def whichmodule(self, obj: Any, name: str) -> str:
"""Find the module name an object belongs to.
This should be considered internal for end-users, but developers of
an importer can override it to customize the behavior.
Taken from pickle.py, but modified to exclude the search into sys.modules
"""
module_name = getattr(obj, "__module__", None)
if module_name is not None:
return module_name
# Protect the iteration by using a list copy of self.modules against dynamic
# modules that trigger imports of other modules upon calls to getattr.
for module_name, module in self.modules.copy().items():
if (module_name == "__main__"
or module_name == "__mp_main__" # bpo-42406
or module is None):
continue
try:
if _getattribute(module, name)[0] is obj:
return module_name
except AttributeError:
pass
return "__main__"
def save_global(self, obj, name=None):
# unfortunately the pickler code is factored in a way that
# forces us to copy/paste this function. The only change is marked
# CHANGED below.
write = self.write
memo = self.memo
# CHANGED: import module from module environment instead of __import__
try:
module_name, name = self.importer.get_name(obj, name)
except (ObjNotFoundError, ObjMismatchError) as err:
raise PicklingError(f"Can't pickle {obj}: {str(err)}") from None
module = self.importer.import_module(module_name)
_, parent = _getattribute(module, name)
# END CHANGED
if self.proto >= 2:
code = _extension_registry.get((module_name, name))
if code:
assert code > 0
if code <= 0xFF:
write(EXT1 + pack("<B", code))
elif code <= 0xFFFF:
write(EXT2 + pack("<H", code))
else:
write(EXT4 + pack("<i", code))
return
lastname = name.rpartition(".")[2]
if parent is module:
name = lastname
# Non-ASCII identifiers are supported only with protocols >= 3.
if self.proto >= 4:
self.save(module_name)
self.save(name)
write(STACK_GLOBAL)
elif parent is not module:
self.save_reduce(getattr, (parent, lastname))
elif self.proto >= 3:
write(GLOBAL + bytes(module_name, "utf-8") + b"\n" +
bytes(name, "utf-8") + b"\n")
else:
if self.fix_imports:
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
if (module_name, name) in r_name_mapping:
module_name, name = r_name_mapping[(module_name, name)]
elif module_name in r_import_mapping:
module_name = r_import_mapping[module_name]
try:
write(GLOBAL + bytes(module_name, "ascii") + b"\n" +
bytes(name, "ascii") + b"\n")
except UnicodeEncodeError:
raise PicklingError(
"can't pickle global identifier '%s.%s' using "
"pickle protocol %i" %
(module, name, self.proto)) from None
self.memoize(obj)
def find_class_py3(module, name, proto=2, fix_imports=True):
if proto < 3 and fix_imports:
if (module, name) in _compat_pickle.NAME_MAPPING:
module, name = _compat_pickle.NAME_MAPPING[(module, name)]
if module in _compat_pickle.IMPORT_MAPPING:
module = _compat_pickle.IMPORT_MAPPING[module]
__import__(module, level=0)
return _getattribute(sys.modules[module], name,
allow_qualname=proto >= 4)
def find_class(self, module, name):
# Subclasses may override this.
if self.proto < 3 and self.fix_imports:
if (module, name) in _compat_pickle.NAME_MAPPING:
module, name = _compat_pickle.NAME_MAPPING[(module, name)]
elif module in _compat_pickle.IMPORT_MAPPING:
module = _compat_pickle.IMPORT_MAPPING[module]
__import__(module, level=0)
if self.proto >= 4:
return _getattribute(sys.modules[module], name)[0]
else:
return getattr(sys.modules[module], name)
def _whichmodule(obj, name): # какому модулю пренадлежит
if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar):
module_name = None
else:
module_name = getattr(obj, "__module__", None)
if module_name is not None:
return module_name
for module_name, module in sys.modules.copy().items():
if (module_name == "__main__" or module is None
or not isinstance(module, types.ModuleType)):
continue
try:
if pickle._getattribute(module, name)[0] is obj:
return module_name
except Exception:
pass
return None
def _whichmodule(obj, name):
"""Find the module an object belongs to.
This function differs from ``pickle.whichmodule`` in two ways:
- it does not mangle the cases where obj's module is __main__ and obj was
not found in any module.
- Errors arising during module introspection are ignored, as those errors
are considered unwanted side effects.
"""
if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa
# Workaround bug in old Python versions: prior to Python 3.7,
# T.__module__ would always be set to "typing" even when the TypeVar T
# would be defined in a different module.
if name is not None and getattr(typing, name, None) is obj:
# Built-in TypeVar defined in typing such as AnyStr
return 'typing'
else:
# User defined or third-party TypeVar: __module__ attribute is
# irrelevant, thus trigger a exhaustive search for obj in all
# modules.
module_name = None
else:
module_name = getattr(obj, '__module__', None)
if module_name is not None:
return module_name
# Protect the iteration by using a copy of sys.modules against dynamic
# modules that trigger imports of other modules upon calls to getattr or
# other threads importing at the same time.
for module_name, module in sys.modules.copy().items():
# Some modules such as coverage can inject non-module objects inside
# sys.modules
if (
module_name == '__main__' or
module is None or
not isinstance(module, types.ModuleType)
):
continue
try:
if _getattribute(module, name)[0] is obj:
return module_name
except Exception:
pass
return None
def _lookup_module_and_qualname(obj, name=None):
if name is None:
name = getattr(obj, '__qualname__', None)
if name is None: # pragma: no cover
# This used to be needed for Python 2.7 support but is probably not
# needed anymore. However we keep the __name__ introspection in case
# users of cloudpickle rely on this old behavior for unknown reasons.
name = getattr(obj, '__name__', None)
module_name = _whichmodule(obj, name)
if module_name is None:
# In this case, obj.__module__ is None AND obj was not found in any
# imported module. obj is thus treated as dynamic.
return None
if module_name == "__main__":
return None
if _is_dynamic_module(module_name):
return None
# Note: if module_name is in sys.modules, the corresponding module is
# assumed importable at unpickling time. See #357
module = sys.modules.get(module_name, None)
if module is None:
# The main reason why obj's module would not be imported is that this
# module has been dynamically created, using for example
# types.ModuleType. The other possibility is that module was removed
# from sys.modules after obj was created/imported. But this case is not
# supported, as the standard pickle does not support it either.
return None
try:
obj2, parent = _getattribute(module, name)
except AttributeError:
# obj was not found inside the module it points to
return None
if obj2 is not obj:
return None
return module, name
def _whichmodule(obj, name):
"""Find the module an object belongs to.
This function differs from ``pickle.whichmodule`` in two ways:
- it does not mangle the cases where obj's module is __main__ and obj was
not found in any module.
- Errors arising during module introspection are ignored, as those errors
are considered unwanted side effects.
"""
module_name = getattr(obj, '__module__', None)
if module_name is not None:
return module_name
# Protect the iteration by using a list copy of sys.modules against dynamic
# modules that trigger imports of other modules upon calls to getattr.
for module_name, module in list(sys.modules.items()):
if module_name == '__main__' or module is None:
continue
try:
if _getattribute(module, name)[0] is obj:
return module_name
except Exception:
pass
return None
def _is_global(obj, name=None):
"""Determine if obj can be pickled as attribute of a file-backed module"""
if name is None:
name = getattr(obj, '__qualname__', None)
if name is None:
name = getattr(obj, '__name__', None)
module_name = _whichmodule(obj, name)
if module_name is None:
# In this case, obj.__module__ is None AND obj was not found in any
# imported module. obj is thus treated as dynamic.
return False
if module_name == "__main__":
return False
module = sys.modules.get(module_name, None)
if module is None:
# The main reason why obj's module would not be imported is that this
# module has been dynamically created, using for example
# types.ModuleType. The other possibility is that module was removed
# from sys.modules after obj was created/imported. But this case is not
# supported, as the standard pickle does not support it either.
return False
# module has been added to sys.modules, but it can still be dynamic.
if _is_dynamic(module):
return False
try:
obj2, parent = _getattribute(module, name)
except AttributeError:
# obj was not found inside the module it points to
return False
return obj2 is obj
def save_global(self, obj, name=None):
# unfortunately the pickler code is factored in a way that
# forces us to copy/paste this function. The only change is marked
# CHANGED below.
write = self.write
memo = self.memo
if name is None:
name = getattr(obj, '__qualname__', None)
if name is None:
name = obj.__name__
orig_module_name = whichmodule(obj, name)
# CHANGED: demangle the module name before importing. If this obj came
# out of a PackageImporter, `__module__` will be mangled. See
# mangling.md for details.
module_name = demangle(orig_module_name)
try:
# CHANGED: self.import_module rather than
# __import__
module = self.import_module(module_name)
obj2, parent = _getattribute(module, name)
except (ImportError, KeyError, AttributeError):
raise PicklingError("Can't pickle %r: it's not found as %s.%s" %
(obj, module_name, name)) from None
else:
if obj2 is not obj:
# CHANGED: More specific error message in the case of mangling.
obj_module_name = getattr(obj, "__module__", orig_module_name)
obj2_module_name = getattr(obj2, "__module__",
orig_module_name)
msg = f"Can't pickle {obj}: it's not the same object as {obj2_module_name}.{name}."
is_obj_mangled = is_mangled(obj_module_name)
is_obj2_mangled = is_mangled(obj2_module_name)
if is_obj_mangled or is_obj2_mangled:
obj_location = (get_mangle_prefix(obj_module_name)
if is_obj_mangled else
"the current Python environment")
obj2_location = (get_mangle_prefix(obj2_module_name)
if is_obj2_mangled else
"the current Python environment")
obj_importer_name = (
f"the importer for {get_mangle_prefix(obj_module_name)}"
if is_obj_mangled else "'importlib.import_module'")
obj2_importer_name = (
f"the importer for {get_mangle_prefix(obj2_module_name)}"
if is_obj2_mangled else "'importlib.import_module'")
msg += (
f"\n\nThe object being pickled is from '{orig_module_name}', "
f"which is coming from {obj_location}."
f"\nHowever, when we import '{orig_module_name}', it's coming from {obj2_location}."
"\nTo fix this, make sure 'PackageExporter.importers' lists "
f"{obj_importer_name} before {obj2_importer_name}")
raise PicklingError(msg)
if self.proto >= 2:
code = _extension_registry.get((module_name, name))
if code:
assert code > 0
if code <= 0xff:
write(EXT1 + pack("<B", code))
elif code <= 0xffff:
write(EXT2 + pack("<H", code))
else:
write(EXT4 + pack("<i", code))
return
lastname = name.rpartition('.')[2]
if parent is module:
name = lastname
# Non-ASCII identifiers are supported only with protocols >= 3.
if self.proto >= 4:
self.save(module_name)
self.save(name)
write(STACK_GLOBAL)
elif parent is not module:
self.save_reduce(getattr, (parent, lastname))
elif self.proto >= 3:
write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
bytes(name, "utf-8") + b'\n')
else:
if self.fix_imports:
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
if (module_name, name) in r_name_mapping:
module_name, name = r_name_mapping[(module_name, name)]
elif module_name in r_import_mapping:
module_name = r_import_mapping[module_name]
try:
write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
bytes(name, "ascii") + b'\n')
except UnicodeEncodeError:
raise PicklingError(
"can't pickle global identifier '%s.%s' using "
"pickle protocol %i" %
(module, name, self.proto)) from None
self.memoize(obj)
def update_event(self, inp=-1):
self.set_output_val(0,
pickle._getattribute(self.input(0), self.input(1)))
def save_global_py3(obj, name=None, proto=2, fix_imports=True):
if name is None and proto >= 4:
name = getattr(obj, '__qualname__', None)
if name is None:
name = obj.__name__
module_name = whichmodule(obj, name, allow_qualname=proto >= 4)
try:
__import__(module_name, level=0)
module = sys.modules[module_name]
obj2 = _getattribute(module, name, allow_qualname=proto >= 4)
except (ImportError, KeyError, AttributeError):
raise PicklingError(
"Can't pickle %r: it's not found as %s.%s" %
(obj, module_name, name))
else:
if obj2 is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module_name, name))
if proto >= 2:
code = _extension_registry.get((module_name, name))
if code:
# assert code > 0
# if code <= 0xff:
# write(EXT1 + pack("<B", code))
# elif code <= 0xffff:
# write(EXT2 + pack("<H", code))
# else:
# write(EXT4 + pack("<i", code))
# return
raise H5itPicklingError("h5it Can't pickle %r: extension codes are not"
" supported yet." % obj)
# Non-ASCII identifiers are supported only with protocols >= 3.
if proto >= 4:
# self.save(module_name)
# self.save(name)
# write(STACK_GLOBAL)
raise H5itPicklingError("h5it Can't pickle %r: protocol %i is not "
"supported yet." % (obj, proto))
elif proto >= 3:
# write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
# bytes(name, "utf-8") + b'\n')
raise H5itPicklingError("h5it Can't pickle %r: protocol %i is not "
"supported yet." % (obj, proto))
else:
if fix_imports:
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
if (module_name, name) in r_name_mapping:
module_name, name = r_name_mapping[(module_name, name)]
if module_name in r_import_mapping:
module_name = r_import_mapping[module_name]
try:
return GlobalTuple(bytes(module_name, "ascii"),
bytes(name, "ascii"))
except UnicodeEncodeError:
raise PicklingError(
"can't pickle global identifier '%s.%s' using "
"pickle protocol %i" % (module, name, proto))
def save_function(self, obj, name=None):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
write = self.write
if name is None:
name = getattr(obj, '__qualname__', None)
if name is None:
name = getattr(obj, '__name__', None)
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = None
# print('which gives %s %s %s' % (modname, obj, name))
try:
themodule = sys.modules[modname]
except KeyError:
# eval'd items such as namedtuple give invalid items for their function __module__
modname = '__main__'
if modname == '__main__':
themodule = None
try:
lookedup_by_name, _ = _getattribute(themodule, name)
except Exception:
lookedup_by_name = None
if themodule:
if lookedup_by_name is obj:
return self.save_global(obj, name)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if (islambda(obj)
or getattr(obj.__code__, 'co_filename', None) == '<stdin>'
or themodule is None):
self.save_function_tuple(obj)
return
else:
# func is nested
if lookedup_by_name is None or lookedup_by_name is not obj:
self.save_function_tuple(obj)
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def save_global(self, obj, name=None):
# unfortunately the pickler code is factored in a way that
# forces us to copy/paste this function. The only change is marked
# CHANGED below.
write = self.write
memo = self.memo
if name is None:
name = getattr(obj, '__qualname__', None)
if name is None:
name = obj.__name__
module_name = whichmodule(obj, name)
try:
# CHANGED: self.import_module rather than
# __import__
module = self.import_module(module_name)
obj2, parent = _getattribute(module, name)
except (ImportError, KeyError, AttributeError):
raise PicklingError("Can't pickle %r: it's not found as %s.%s" %
(obj, module_name, name)) from None
else:
if obj2 is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module_name, name))
if self.proto >= 2:
code = _extension_registry.get((module_name, name))
if code:
assert code > 0
if code <= 0xff:
write(EXT1 + pack("<B", code))
elif code <= 0xffff:
write(EXT2 + pack("<H", code))
else:
write(EXT4 + pack("<i", code))
return
lastname = name.rpartition('.')[2]
if parent is module:
name = lastname
# Non-ASCII identifiers are supported only with protocols >= 3.
if self.proto >= 4:
self.save(module_name)
self.save(name)
write(STACK_GLOBAL)
elif parent is not module:
self.save_reduce(getattr, (parent, lastname))
elif self.proto >= 3:
write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
bytes(name, "utf-8") + b'\n')
else:
if self.fix_imports:
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
if (module_name, name) in r_name_mapping:
module_name, name = r_name_mapping[(module_name, name)]
elif module_name in r_import_mapping:
module_name = r_import_mapping[module_name]
try:
write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
bytes(name, "ascii") + b'\n')
except UnicodeEncodeError:
raise PicklingError(
"can't pickle global identifier '%s.%s' using "
"pickle protocol %i" %
(module, name, self.proto)) from None
self.memoize(obj)
async def save_global(self, obj, name=None):
write = self.write
memo = self.memo
if name is None:
name = getattr(obj, '__qualname__', None)
if name is None:
name = obj.__name__
module_name = whichmodule(obj, name)
try:
__import__(module_name, level=0)
module = sys.modules[module_name]
obj2, parent = _getattribute(module, name)
except (ImportError, KeyError, AttributeError):
raise PicklingError(
"Can't pickle %r: it's not found as %s.%s" %
(obj, module_name, name)) from None
else:
if obj2 is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module_name, name))
if self.proto >= 2:
code = _extension_registry.get((module_name, name))
if code:
assert code > 0
if code <= 0xff:
await write(EXT1 + pack("<B", code))
elif code <= 0xffff:
await write(EXT2 + pack("<H", code))
else:
await write(EXT4 + pack("<i", code))
return
lastname = name.rpartition('.')[2]
if parent is module:
name = lastname
# Non-ASCII identifiers are supported only with protocols >= 3.
if self.proto >= 4:
await self.save(module_name)
await self.save(name)
await write(STACK_GLOBAL)
elif parent is not module:
await self.save_reduce(getattr, (parent, lastname))
elif self.proto >= 3:
await write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
bytes(name, "utf-8") + b'\n')
else:
if self.fix_imports:
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
if (module_name, name) in r_name_mapping:
module_name, name = r_name_mapping[(module_name, name)]
elif module_name in r_import_mapping:
module_name = r_import_mapping[module_name]
try:
await write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
bytes(name, "ascii") + b'\n')
except UnicodeEncodeError:
raise PicklingError(
"can't pickle global identifier '%s.%s' using "
"pickle protocol %i" % (module, name, self.proto)) from None
await self.memoize(obj)
def get_name(self,
obj: Any,
name: Optional[str] = None) -> Tuple[str, str]:
"""Given an object, return a name that can be used to retrieve the
object from this environment.
Args:
obj: An object to get the the module-environment-relative name for.
name: If set, use this name instead of looking up __name__ or __qualname__ on `obj`.
This is only here to match how Pickler handles __reduce__ functions that return a string,
don't use otherwise.
Returns:
A tuple (parent_module_name, attr_name) that can be used to retrieve `obj` from this environment.
Use it like:
mod = importer.import_module(parent_module_name)
obj = getattr(mod, attr_name)
Raises:
ObjNotFoundError: we couldn't retrieve `obj by name.
ObjMisMatchError: we found a different object with the same name as `obj`.
"""
if name is None and obj and _Pickler.dispatch.get(type(obj)) is None:
# Honor the string return variant of __reduce__, which will give us
# a global name to search for in this environment.
# TODO: I guess we should do copyreg too?
reduce = getattr(obj, "__reduce__", None)
if reduce is not None:
try:
rv = reduce()
if isinstance(rv, str):
name = rv
except Exception:
pass
if name is None:
name = getattr(obj, "__qualname__", None)
if name is None:
name = obj.__name__
orig_module_name = self.whichmodule(obj, name)
# Demangle the module name before importing. If this obj came out of a
# PackageImporter, `__module__` will be mangled. See mangling.md for
# details.
module_name = demangle(orig_module_name)
# Check that this name will indeed return the correct object
try:
module = self.import_module(module_name)
obj2, _ = _getattribute(module, name)
except (ImportError, KeyError, AttributeError):
raise ObjNotFoundError(
f"{obj} was not found as {module_name}.{name}") from None
if obj is obj2:
return module_name, name
def get_obj_info(obj):
assert name is not None
module_name = self.whichmodule(obj, name)
is_mangled_ = is_mangled(module_name)
location = (get_mangle_prefix(module_name)
if is_mangled_ else "the current Python environment")
importer_name = (
f"the importer for {get_mangle_prefix(module_name)}"
if is_mangled_ else "'sys_importer'")
return module_name, location, importer_name
obj_module_name, obj_location, obj_importer_name = get_obj_info(obj)
obj2_module_name, obj2_location, obj2_importer_name = get_obj_info(
obj2)
msg = (
f"\n\nThe object provided is from '{obj_module_name}', "
f"which is coming from {obj_location}."
f"\nHowever, when we import '{obj2_module_name}', it's coming from {obj2_location}."
"\nTo fix this, make sure this 'PackageExporter's importer lists "
f"{obj_importer_name} before {obj2_importer_name}.")
raise ObjMismatchError(msg)
