def _whichmodule(obj, name): """Find the module an object belongs to. This function differs from ``pickle.whichmodule`` in two ways: - it does not mangle the cases where obj's module is __main__ and obj was not found in any module. - Errors arising during module introspection are ignored, as those errors are considered unwanted side effects. """ module_name = _get_module_attr(obj) if module_name is not None: return module_name # Protect the iteration by using a copy of sys.modules against dynamic # modules that trigger imports of other modules upon calls to getattr or # other threads importing at the same time. for module_name, module in sys.modules.copy().items(): # Some modules such as coverage can inject non-module objects inside # sys.modules if (module_name == '__main__' or module is None or not isinstance(module, types.ModuleType)): continue try: if _getattribute(module, name)[0] is obj: return module_name except Exception: pass return None
def _lookup_module_and_qualname(obj, name=None): # если и ф-и имя и модуль if name is None: name = getattr(obj, "__qualname__", None) if name is None: name = getattr(obj, "__name__", None) module_name = _whichmodule(obj, name) if module_name is None: return None if module_name == "__main__": return None module = sys.modules.get(module_name, None) if module is None: return None try: obj2, parent = pickle._getattribute(module, name) except AttributeError: return None if obj2 is not obj: return None return module, name
def whichmodule(self, obj: Any, name: str) -> str: """Find the module name an object belongs to. This should be considered internal for end-users, but developers of an importer can override it to customize the behavior. Taken from pickle.py, but modified to exclude the search into sys.modules """ module_name = getattr(obj, "__module__", None) if module_name is not None: return module_name # Protect the iteration by using a list copy of self.modules against dynamic # modules that trigger imports of other modules upon calls to getattr. for module_name, module in self.modules.copy().items(): if (module_name == "__main__" or module_name == "__mp_main__" # bpo-42406 or module is None): continue try: if _getattribute(module, name)[0] is obj: return module_name except AttributeError: pass return "__main__"
def save_global(self, obj, name=None): # unfortunately the pickler code is factored in a way that # forces us to copy/paste this function. The only change is marked # CHANGED below. write = self.write memo = self.memo # CHANGED: import module from module environment instead of __import__ try: module_name, name = self.importer.get_name(obj, name) except (ObjNotFoundError, ObjMismatchError) as err: raise PicklingError(f"Can't pickle {obj}: {str(err)}") from None module = self.importer.import_module(module_name) _, parent = _getattribute(module, name) # END CHANGED if self.proto >= 2: code = _extension_registry.get((module_name, name)) if code: assert code > 0 if code <= 0xFF: write(EXT1 + pack("<B", code)) elif code <= 0xFFFF: write(EXT2 + pack("<H", code)) else: write(EXT4 + pack("<i", code)) return lastname = name.rpartition(".")[2] if parent is module: name = lastname # Non-ASCII identifiers are supported only with protocols >= 3. if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) elif parent is not module: self.save_reduce(getattr, (parent, lastname)) elif self.proto >= 3: write(GLOBAL + bytes(module_name, "utf-8") + b"\n" + bytes(name, "utf-8") + b"\n") else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: write(GLOBAL + bytes(module_name, "ascii") + b"\n" + bytes(name, "ascii") + b"\n") except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, self.proto)) from None self.memoize(obj)
def find_class_py3(module, name, proto=2, fix_imports=True): if proto < 3 and fix_imports: if (module, name) in _compat_pickle.NAME_MAPPING: module, name = _compat_pickle.NAME_MAPPING[(module, name)] if module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) return _getattribute(sys.modules[module], name, allow_qualname=proto >= 4)
def find_class(self, module, name): # Subclasses may override this. if self.proto < 3 and self.fix_imports: if (module, name) in _compat_pickle.NAME_MAPPING: module, name = _compat_pickle.NAME_MAPPING[(module, name)] elif module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) if self.proto >= 4: return _getattribute(sys.modules[module], name)[0] else: return getattr(sys.modules[module], name)
def _whichmodule(obj, name): # какому модулю пренадлежит if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): module_name = None else: module_name = getattr(obj, "__module__", None) if module_name is not None: return module_name for module_name, module in sys.modules.copy().items(): if (module_name == "__main__" or module is None or not isinstance(module, types.ModuleType)): continue try: if pickle._getattribute(module, name)[0] is obj: return module_name except Exception: pass return None
def _whichmodule(obj, name): """Find the module an object belongs to. This function differs from ``pickle.whichmodule`` in two ways: - it does not mangle the cases where obj's module is __main__ and obj was not found in any module. - Errors arising during module introspection are ignored, as those errors are considered unwanted side effects. """ if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa # Workaround bug in old Python versions: prior to Python 3.7, # T.__module__ would always be set to "typing" even when the TypeVar T # would be defined in a different module. if name is not None and getattr(typing, name, None) is obj: # Built-in TypeVar defined in typing such as AnyStr return 'typing' else: # User defined or third-party TypeVar: __module__ attribute is # irrelevant, thus trigger a exhaustive search for obj in all # modules. module_name = None else: module_name = getattr(obj, '__module__', None) if module_name is not None: return module_name # Protect the iteration by using a copy of sys.modules against dynamic # modules that trigger imports of other modules upon calls to getattr or # other threads importing at the same time. for module_name, module in sys.modules.copy().items(): # Some modules such as coverage can inject non-module objects inside # sys.modules if ( module_name == '__main__' or module is None or not isinstance(module, types.ModuleType) ): continue try: if _getattribute(module, name)[0] is obj: return module_name except Exception: pass return None
def _lookup_module_and_qualname(obj, name=None): if name is None: name = getattr(obj, '__qualname__', None) if name is None: # pragma: no cover # This used to be needed for Python 2.7 support but is probably not # needed anymore. However we keep the __name__ introspection in case # users of cloudpickle rely on this old behavior for unknown reasons. name = getattr(obj, '__name__', None) module_name = _whichmodule(obj, name) if module_name is None: # In this case, obj.__module__ is None AND obj was not found in any # imported module. obj is thus treated as dynamic. return None if module_name == "__main__": return None if _is_dynamic_module(module_name): return None # Note: if module_name is in sys.modules, the corresponding module is # assumed importable at unpickling time. See #357 module = sys.modules.get(module_name, None) if module is None: # The main reason why obj's module would not be imported is that this # module has been dynamically created, using for example # types.ModuleType. The other possibility is that module was removed # from sys.modules after obj was created/imported. But this case is not # supported, as the standard pickle does not support it either. return None try: obj2, parent = _getattribute(module, name) except AttributeError: # obj was not found inside the module it points to return None if obj2 is not obj: return None return module, name
def _whichmodule(obj, name): """Find the module an object belongs to. This function differs from ``pickle.whichmodule`` in two ways: - it does not mangle the cases where obj's module is __main__ and obj was not found in any module. - Errors arising during module introspection are ignored, as those errors are considered unwanted side effects. """ module_name = getattr(obj, '__module__', None) if module_name is not None: return module_name # Protect the iteration by using a list copy of sys.modules against dynamic # modules that trigger imports of other modules upon calls to getattr. for module_name, module in list(sys.modules.items()): if module_name == '__main__' or module is None: continue try: if _getattribute(module, name)[0] is obj: return module_name except Exception: pass return None
def _is_global(obj, name=None): """Determine if obj can be pickled as attribute of a file-backed module""" if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = getattr(obj, '__name__', None) module_name = _whichmodule(obj, name) if module_name is None: # In this case, obj.__module__ is None AND obj was not found in any # imported module. obj is thus treated as dynamic. return False if module_name == "__main__": return False module = sys.modules.get(module_name, None) if module is None: # The main reason why obj's module would not be imported is that this # module has been dynamically created, using for example # types.ModuleType. The other possibility is that module was removed # from sys.modules after obj was created/imported. But this case is not # supported, as the standard pickle does not support it either. return False # module has been added to sys.modules, but it can still be dynamic. if _is_dynamic(module): return False try: obj2, parent = _getattribute(module, name) except AttributeError: # obj was not found inside the module it points to return False return obj2 is obj
def save_global(self, obj, name=None): # unfortunately the pickler code is factored in a way that # forces us to copy/paste this function. The only change is marked # CHANGED below. write = self.write memo = self.memo if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ orig_module_name = whichmodule(obj, name) # CHANGED: demangle the module name before importing. If this obj came # out of a PackageImporter, `__module__` will be mangled. See # mangling.md for details. module_name = demangle(orig_module_name) try: # CHANGED: self.import_module rather than # __import__ module = self.import_module(module_name) obj2, parent = _getattribute(module, name) except (ImportError, KeyError, AttributeError): raise PicklingError("Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) from None else: if obj2 is not obj: # CHANGED: More specific error message in the case of mangling. obj_module_name = getattr(obj, "__module__", orig_module_name) obj2_module_name = getattr(obj2, "__module__", orig_module_name) msg = f"Can't pickle {obj}: it's not the same object as {obj2_module_name}.{name}." is_obj_mangled = is_mangled(obj_module_name) is_obj2_mangled = is_mangled(obj2_module_name) if is_obj_mangled or is_obj2_mangled: obj_location = (get_mangle_prefix(obj_module_name) if is_obj_mangled else "the current Python environment") obj2_location = (get_mangle_prefix(obj2_module_name) if is_obj2_mangled else "the current Python environment") obj_importer_name = ( f"the importer for {get_mangle_prefix(obj_module_name)}" if is_obj_mangled else "'importlib.import_module'") obj2_importer_name = ( f"the importer for {get_mangle_prefix(obj2_module_name)}" if is_obj2_mangled else "'importlib.import_module'") msg += ( f"\n\nThe object being pickled is from '{orig_module_name}', " f"which is coming from {obj_location}." f"\nHowever, when we import '{orig_module_name}', it's coming from {obj2_location}." "\nTo fix this, make sure 'PackageExporter.importers' lists " f"{obj_importer_name} before {obj2_importer_name}") raise PicklingError(msg) if self.proto >= 2: code = _extension_registry.get((module_name, name)) if code: assert code > 0 if code <= 0xff: write(EXT1 + pack("<B", code)) elif code <= 0xffff: write(EXT2 + pack("<H", code)) else: write(EXT4 + pack("<i", code)) return lastname = name.rpartition('.')[2] if parent is module: name = lastname # Non-ASCII identifiers are supported only with protocols >= 3. if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) elif parent is not module: self.save_reduce(getattr, (parent, lastname)) elif self.proto >= 3: write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + bytes(name, "utf-8") + b'\n') else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: write(GLOBAL + bytes(module_name, "ascii") + b'\n' + bytes(name, "ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, self.proto)) from None self.memoize(obj)
def update_event(self, inp=-1): self.set_output_val(0, pickle._getattribute(self.input(0), self.input(1)))
def save_global_py3(obj, name=None, proto=2, fix_imports=True): if name is None and proto >= 4: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ module_name = whichmodule(obj, name, allow_qualname=proto >= 4) try: __import__(module_name, level=0) module = sys.modules[module_name] obj2 = _getattribute(module, name, allow_qualname=proto >= 4) except (ImportError, KeyError, AttributeError): raise PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) else: if obj2 is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module_name, name)) if proto >= 2: code = _extension_registry.get((module_name, name)) if code: # assert code > 0 # if code <= 0xff: # write(EXT1 + pack("<B", code)) # elif code <= 0xffff: # write(EXT2 + pack("<H", code)) # else: # write(EXT4 + pack("<i", code)) # return raise H5itPicklingError("h5it Can't pickle %r: extension codes are not" " supported yet." % obj) # Non-ASCII identifiers are supported only with protocols >= 3. if proto >= 4: # self.save(module_name) # self.save(name) # write(STACK_GLOBAL) raise H5itPicklingError("h5it Can't pickle %r: protocol %i is not " "supported yet." % (obj, proto)) elif proto >= 3: # write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + # bytes(name, "utf-8") + b'\n') raise H5itPicklingError("h5it Can't pickle %r: protocol %i is not " "supported yet." % (obj, proto)) else: if fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] if module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: return GlobalTuple(bytes(module_name, "ascii"), bytes(name, "ascii")) except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, proto))
def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ write = self.write if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = getattr(obj, '__name__', None) try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = None # print('which gives %s %s %s' % (modname, obj, name)) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None try: lookedup_by_name, _ = _getattribute(themodule, name) except Exception: lookedup_by_name = None if themodule: if lookedup_by_name is obj: return self.save_global(obj, name) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if (islambda(obj) or getattr(obj.__code__, 'co_filename', None) == '<stdin>' or themodule is None): self.save_function_tuple(obj) return else: # func is nested if lookedup_by_name is None or lookedup_by_name is not obj: self.save_function_tuple(obj) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def save_global(self, obj, name=None): # unfortunately the pickler code is factored in a way that # forces us to copy/paste this function. The only change is marked # CHANGED below. write = self.write memo = self.memo if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ module_name = whichmodule(obj, name) try: # CHANGED: self.import_module rather than # __import__ module = self.import_module(module_name) obj2, parent = _getattribute(module, name) except (ImportError, KeyError, AttributeError): raise PicklingError("Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) from None else: if obj2 is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module_name, name)) if self.proto >= 2: code = _extension_registry.get((module_name, name)) if code: assert code > 0 if code <= 0xff: write(EXT1 + pack("<B", code)) elif code <= 0xffff: write(EXT2 + pack("<H", code)) else: write(EXT4 + pack("<i", code)) return lastname = name.rpartition('.')[2] if parent is module: name = lastname # Non-ASCII identifiers are supported only with protocols >= 3. if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) elif parent is not module: self.save_reduce(getattr, (parent, lastname)) elif self.proto >= 3: write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + bytes(name, "utf-8") + b'\n') else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: write(GLOBAL + bytes(module_name, "ascii") + b'\n' + bytes(name, "ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, self.proto)) from None self.memoize(obj)
async def save_global(self, obj, name=None): write = self.write memo = self.memo if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ module_name = whichmodule(obj, name) try: __import__(module_name, level=0) module = sys.modules[module_name] obj2, parent = _getattribute(module, name) except (ImportError, KeyError, AttributeError): raise PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) from None else: if obj2 is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module_name, name)) if self.proto >= 2: code = _extension_registry.get((module_name, name)) if code: assert code > 0 if code <= 0xff: await write(EXT1 + pack("<B", code)) elif code <= 0xffff: await write(EXT2 + pack("<H", code)) else: await write(EXT4 + pack("<i", code)) return lastname = name.rpartition('.')[2] if parent is module: name = lastname # Non-ASCII identifiers are supported only with protocols >= 3. if self.proto >= 4: await self.save(module_name) await self.save(name) await write(STACK_GLOBAL) elif parent is not module: await self.save_reduce(getattr, (parent, lastname)) elif self.proto >= 3: await write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + bytes(name, "utf-8") + b'\n') else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: await write(GLOBAL + bytes(module_name, "ascii") + b'\n' + bytes(name, "ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, self.proto)) from None await self.memoize(obj)
def get_name(self, obj: Any, name: Optional[str] = None) -> Tuple[str, str]: """Given an object, return a name that can be used to retrieve the object from this environment. Args: obj: An object to get the the module-environment-relative name for. name: If set, use this name instead of looking up __name__ or __qualname__ on `obj`. This is only here to match how Pickler handles __reduce__ functions that return a string, don't use otherwise. Returns: A tuple (parent_module_name, attr_name) that can be used to retrieve `obj` from this environment. Use it like: mod = importer.import_module(parent_module_name) obj = getattr(mod, attr_name) Raises: ObjNotFoundError: we couldn't retrieve `obj by name. ObjMisMatchError: we found a different object with the same name as `obj`. """ if name is None and obj and _Pickler.dispatch.get(type(obj)) is None: # Honor the string return variant of __reduce__, which will give us # a global name to search for in this environment. # TODO: I guess we should do copyreg too? reduce = getattr(obj, "__reduce__", None) if reduce is not None: try: rv = reduce() if isinstance(rv, str): name = rv except Exception: pass if name is None: name = getattr(obj, "__qualname__", None) if name is None: name = obj.__name__ orig_module_name = self.whichmodule(obj, name) # Demangle the module name before importing. If this obj came out of a # PackageImporter, `__module__` will be mangled. See mangling.md for # details. module_name = demangle(orig_module_name) # Check that this name will indeed return the correct object try: module = self.import_module(module_name) obj2, _ = _getattribute(module, name) except (ImportError, KeyError, AttributeError): raise ObjNotFoundError( f"{obj} was not found as {module_name}.{name}") from None if obj is obj2: return module_name, name def get_obj_info(obj): assert name is not None module_name = self.whichmodule(obj, name) is_mangled_ = is_mangled(module_name) location = (get_mangle_prefix(module_name) if is_mangled_ else "the current Python environment") importer_name = ( f"the importer for {get_mangle_prefix(module_name)}" if is_mangled_ else "'sys_importer'") return module_name, location, importer_name obj_module_name, obj_location, obj_importer_name = get_obj_info(obj) obj2_module_name, obj2_location, obj2_importer_name = get_obj_info( obj2) msg = ( f"\n\nThe object provided is from '{obj_module_name}', " f"which is coming from {obj_location}." f"\nHowever, when we import '{obj2_module_name}', it's coming from {obj2_location}." "\nTo fix this, make sure this 'PackageExporter's importer lists " f"{obj_importer_name} before {obj2_importer_name}.") raise ObjMismatchError(msg)