def save_global(self, obj, name=None): assert obj assert id(obj) not in self.memo if name is None: name = obj.__name__ module = getattr(obj, "__module__", None) if module is None or module == "__main__": module = pickle.whichmodule(obj, name) if module is None or module == "__main__": raise pickle.PicklingError( "Can't pickle %r: module not found: %s" % (obj, module)) try: __import__(module) mod = sys.modules[module] klass = getattr(mod, name) except (ImportError, KeyError, AttributeError): raise pickle.PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, module, name)) else: if klass is not obj: raise pickle.PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module, name)) assert "\n" not in module assert "\n" not in name self.write(pickle.GLOBAL + bytes(module + '\n' + name + '\n', "utf8")) self.memoize(obj)
def reduce_function(obj): name = obj.__name__ if not name or name == "<lambda>": return load_closure, (dump_closure(obj),) module = getattr(obj, "__module__", None) if module is None: module = whichmodule(obj, name) if module == "__main__" and name not in ( "load_closure", "load_module", "load_method", "load_local_class", ): # fix for test return load_closure, (dump_closure(obj),) try: f = get_global_function(module, name) except (ImportError, KeyError, AttributeError): return load_closure, (dump_closure(obj),) else: if f is not obj: return load_closure, (dump_closure(obj),) return name
def save_global(coder, obj, name=None): if name is None: name = obj.__name__ module = getattr(obj, "__module__", None) if module is None: module = whichmodule(obj, name) try: __import__ (module) mod = sys.modules[module] klass= getattr(mod, name) except (ImportError, KeyError, AttributeError): raise PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, module, name)) else: if klass is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module, name)) code = copy_reg._extension_registry.get((module, name)) if code: coder.encodeInt_forKey_(kOP_GLOBAL_EXT, kKIND) coder.encodeInt_forKey_(code, kCODE) else: coder.encodeInt_forKey_(kOP_GLOBAL, kKIND) coder.encodeObject_forKey_(unicode(module), kMODULE) coder.encodeObject_forKey_(unicode(name), kNAME)
def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ write = self.write if name is None: name = obj.__name__ try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = None # print('which gives %s %s %s' % (modname, obj, name)) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their # function __module__ modname = '__main__' if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) if getattr(themodule, name, None) is obj: return self.save_global(obj, name) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if islambda(obj) or obj.__code__.co_filename == '<stdin>' or \ themodule is None: # print("save global", islambda(obj), obj.__code__.co_filename, # modname, themodule) self.save_function_tuple(obj) return else: # func is nested klass = getattr(themodule, name, None) if klass is None or klass is not obj: self.save_function_tuple(obj) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def save_global(self, obj, name=None, pack=struct.pack): if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ self.save(_load_class) self.save_reduce(typ, (obj.__name__, obj.__bases__, {"__doc__": obj.__doc__}), obj=obj) d.pop('__doc__', None) # handle property and staticmethod dd = {} for k, v in d.items(): if isinstance(v, property): k = ('property', k) v = (v.fget, v.fset, v.fdel, v.__doc__) elif isinstance(v, staticmethod) and hasattr(v, '__func__'): k = ('staticmethod', k) v = v.__func__ elif isinstance(v, classmethod) and hasattr(v, '__func__'): k = ('classmethod', k) v = v.__func__ dd[k] = v self.save(dd) self.write(pickle.TUPLE2) self.write(pickle.REDUCE) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_function(self, obj, name=None, pack=struct.pack): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ write = self.write name = obj.__name__ modname = pickle.whichmodule(obj, name) #print 'which gives %s %s %s' % (modname, obj, name) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) if getattr(themodule, name, None) is obj: return self.save_global(obj, name) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if islambda(obj) or obj.func_code.co_filename == '<stdin>' or themodule is None: #Force server to import modules that have been imported in main modList = None if themodule is None and not self.savedForceImports: mainmod = sys.modules['__main__'] if useForcedImports and hasattr(mainmod,'___pyc_forcedImports__'): modList = list(mainmod.___pyc_forcedImports__) self.savedForceImports = True self.save_function_tuple(obj, modList) return else: # func is nested klass = getattr(themodule, name, None) if klass is None or klass is not obj: self.save_function_tuple(obj, [themodule]) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def _jelly_function(self, func): ''' (internal) Jelly a function. Returns a list of the form (function "name" (module "name")) ''' name = func.__name__ module = sys.modules[pickle.whichmodule(func, name)] if self.taster.isModuleAllowed(module.__name__): jfunc = self._prepare(func) jfunc.append(function_atom) jfunc.append(name) jfunc.append(self.jelly(module)) return self._preserve(func, jfunc) else: return self.unpersistable("module %s deemed insecure" % str(module.__name__))
def jelly(self, obj): if isinstance(obj, Jellyable): preRef = self._checkMutable(obj) if preRef: return preRef return obj.jellyFor(self) objType = type(obj) if self.taster.isTypeAllowed(qual(objType)): # "Immutable" Types if ((objType is StringType) or (objType is IntType) or (objType is LongType) or (objType is FloatType)): return obj elif objType is MethodType: return ["method", obj.im_func.__name__, self.jelly(obj.im_self), self.jelly(obj.im_class)] elif UnicodeType and objType is UnicodeType: return ['unicode', obj.encode('UTF-8')] elif objType is NoneType: return ['None'] elif objType is FunctionType: name = obj.__name__ return ['function', str(pickle.whichmodule(obj, obj.__name__)) + '.' + name] elif objType is ModuleType: return ['module', obj.__name__] elif objType is BooleanType: return ['boolean', obj and 'true' or 'false'] elif objType is datetime.datetime: if obj.tzinfo: raise NotImplementedError( "Currently can't jelly datetime objects with tzinfo") return ['datetime', '%s %s %s %s %s %s %s' % ( obj.year, obj.month, obj.day, obj.hour, obj.minute, obj.second, obj.microsecond)] elif objType is datetime.time: if obj.tzinfo: raise NotImplementedError( "Currently can't jelly datetime objects with tzinfo") return ['time', '%s %s %s %s' % (obj.hour, obj.minute, obj.second, obj.microsecond)] elif objType is datetime.date: return ['date', '%s %s %s' % (obj.year, obj.month, obj.day)] elif objType is datetime.timedelta: r
def GetServiceClassString(cls, argv = None): if argv is None: argv = sys.argv import pickle modName = pickle.whichmodule(cls, cls.__name__) if modName == '__main__': try: fname = win32api.GetFullPathName(argv[0]) path = os.path.split(fname)[0] # Eaaaahhhh - sometimes this will be a short filename, which causes # problems with 1.5.1 and the silly filename case rule. # Get the long name fname = os.path.join(path, win32api.FindFiles(fname)[0][8]) except win32api.error: raise error("Could not resolve the path name '%s' to a full path" % (argv[0])) modName = os.path.splitext(fname)[0] return modName + "." + cls.__name__
def GetServiceClassString(cls, argv=None): if argv is None: argv = sys.argv import pickle modName = pickle.whichmodule(cls, cls.__name__) if modName == '__main__': try: fname = win32api.GetFullPathName(argv[0]) path = os.path.split(fname)[0] # Eaaaahhhh - sometimes this will be a short filename, which causes # problems with 1.5.1 and the silly filename case rule. # Get the long name fname = os.path.join(path, win32api.FindFiles(fname)[0][8]) except win32api.error: raise error("Could not resolve the path name '%s' to a full path" % (argv[0])) modName = os.path.splitext(fname)[0] return modName + "." + cls.__name__
def save_global(coder, obj, name=None): if name is None: name = obj.__name__ module = getattr(obj, "__module__", None) if module is None: module = whichmodule(obj, name) try: __import__(module) mod = sys.modules[module] klass = getattr(mod, name) except (ImportError, KeyError, AttributeError): raise PicklingError("Can't pickle %r: it's not found as %s.%s" % (obj, module, name)) else: if klass is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module, name)) code = copy_reg._extension_registry.get((module, name)) if coder.allowsKeyedCoding(): if code: coder.encodeInt_forKey_(kOP_GLOBAL_EXT, kKIND) coder.encodeInt_forKey_(code, kCODE) else: coder.encodeInt_forKey_(kOP_GLOBAL, kKIND) coder.encodeObject_forKey_(unicode(module), kMODULE) coder.encodeObject_forKey_(unicode(name), kNAME) else: if code: coder.__pyobjc__encodeInt_(kOP_GLOBAL_EXT) coder.__pyobjc__encodeInt_(code) else: coder.__pyobjc__encodeInt_(kOP_GLOBAL) coder.encodeObject_(unicode(module)) coder.encodeObject_(unicode(name))
def globalname( self, client): """Try to find the name for the client""" name = getattr( client, '__name__', None ) if not name: raise ValueError( """Attempt to put %r in the root cellar, has no __name__ attribute, so can't create a global reference to it"""%( client,)) module = getattr( client, '__module__', None ) if not module: module = whichmodule( client, name ) # sanity check... try: __import__(module) mod = sys.modules[module] klass = getattr(mod, name) except (ImportError, KeyError, AttributeError), error: raise RootCellarError( """Can't store %r: can't import %s.%s: %r"""%( client, module, name, error ) )
def _class_string(cls): """Return a string we can use later to find the base class of cls. Raise TypeError if we can't compute a useful string. """ log.debug("_class_string: %r", cls) try: return _class_string_cache[cls] except KeyError: pass # Most of this is borrowed from pickle.py name = getattr(cls, '__qualname__', None) if name is None: name = cls.__name__ module_name = whichmodule(cls, name) try: __import__(module_name, level=0) module = sys.modules[module_name] obj2 = module for subpath in name.split('.'): if subpath == '<locals>': raise AttributeError( "<locals> in class name {}".format(name)) try: parent = obj2 obj2 = getattr(obj2, subpath) except AttributeError: raise AttributeError( "Can't get attribute {!r} in {!r}".format(name, obj2)) except (ImportError, KeyError, AttributeError): raise TypeError("Can't persist {!r} instance, class not found" " as {}.{}".format(cls, module_name, name)) else: if obj2 is not cls: raise TypeError("Can't persist {!r} instance, class is" " not the same object as {}.{}".format( cls, module_name, name)) if module_name == '__builtin__': module_name = 'builtins' res = _class_string_cache[cls] = "{}:{}".format(module_name, name) log.debug("new _class_string: %r", res) return res
def reduce_function(obj): name = obj.__name__ if not name or name == '<lambda>': return load_closure, (dump_closure(obj),) module = getattr(obj, "__module__", None) if module is None: module = whichmodule(obj, name) if module == '__main__' and name != 'load_closure': # fix for test return load_closure, (dump_closure(obj),) try: f = get_global_function(module, name) except (ImportError, KeyError, AttributeError): return load_closure, (dump_closure(obj),) else: if f is not obj: return load_closure, (dump_closure(obj),) return name
def reduce_function(obj): name = obj.__name__ if not name or name == '<lambda>': return load_closure, (dump_closure(obj), ) module = getattr(obj, "__module__", None) if module is None: module = whichmodule(obj, name) if module == '__main__' and name != 'load_closure': # fix for test return load_closure, (dump_closure(obj), ) try: f = get_global_function(module, name) except (ImportError, KeyError, AttributeError): return load_closure, (dump_closure(obj), ) else: if f is not obj: return load_closure, (dump_closure(obj), ) return name
def _class_string(cls): """Return a string we can use later to find the base class of cls. Raise TypeError if we can't compute a useful string. """ log.debug("_class_string: %r", cls) try: return _class_string_cache[cls] except KeyError: pass # Most of this is borrowed from pickle.py name = getattr(cls, '__qualname__', None) if name is None: name = cls.__name__ module_name = whichmodule(cls, name) try: __import__(module_name, level=0) module = sys.modules[module_name] obj2 = module for subpath in name.split('.'): if subpath == '<locals>': raise AttributeError("<locals> in class name {}".format(name)) try: parent = obj2 obj2 = getattr(obj2, subpath) except AttributeError: raise AttributeError("Can't get attribute {!r} in {!r}".format( name, obj2)) except (ImportError, KeyError, AttributeError): raise TypeError("Can't persist {!r} instance, class not found" " as {}.{}".format(cls, module_name, name)) else: if obj2 is not cls: raise TypeError("Can't persist {!r} instance, class is" " not the same object as {}.{}".format( cls, module_name, name)) if module_name == '__builtin__': module_name = 'builtins' res = _class_string_cache[cls] = "{}:{}".format(module_name, name) log.debug("new _class_string: %r", res) return res
def GetServiceClassString(cls, argv = None): if argv is None: argv = sys.argv import pickle modName = pickle.whichmodule(cls, cls.__name__) if modName == '__main__': try: fname = win32api.GetFullPathName(argv[0]) path = os.path.split(fname)[0] # Eaaaahhhh - sometimes this will be a short filename, which causes # problems with 1.5.1 and the silly filename case rule. filelist = win32api.FindFiles(fname) # win32api.FindFiles will not detect files in a zip or exe. If list is empty, # skip the test and hope the file really exists. if len(filelist) != 0: # Get the long name fname = os.path.join(path, filelist[0][8]) except win32api.error: raise error("Could not resolve the path name '%s' to a full path" % (argv[0])) modName = os.path.splitext(fname)[0] return modName + "." + cls.__name__
def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = '__main__' if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, _class_type)): self.save_dynamic_class(obj) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack): if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def _pickle_global(obj): name = obj.__name__ module = getattr(obj, "__module__", None) if module is None: module = pickle.whichmodule(obj, name) try: __import__(module) mod = sys.modules[module] klass = getattr(mod, name) except (ImportError, KeyError, AttributeError): raise pickle.PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, module, name)) else: if klass is not obj: raise pickle.PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module, name)) return (_unpickle_global, (module, name))
def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = '__main__' if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): self.save_dynamic_class(obj) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack): if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global_py2(obj, name=None, proto=2): if name is None: name = obj.__name__ module = getattr(obj, "__module__", None) if module is None: module = whichmodule(obj, name) try: __import__(module) mod = sys.modules[module] klass = getattr(mod, name) except (ImportError, KeyError, AttributeError): raise PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, module, name)) else: if klass is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module, name)) if proto >= 2: code = _extension_registry.get((module, name)) if code: # assert code > 0 # if code <= 0xff: # write(EXT1 + chr(code)) # elif code <= 0xffff: # write("%c%c%c" % (EXT2, code&0xff, code>>8)) # else: # write(EXT4 + pack("<i", code)) # return raise H5itPicklingError( "h5it Can't pickle %r: extension codes " "are not supported yet." % obj) return GlobalTuple(module, name)
def _ModuleElementType(self, obj, obj_type, obj_id, name=None): self._seen.add(obj_id) if name is None: name = obj.__name__ modulename = getattr(obj, "__module__", None) if modulename is None: modulename = pickle.whichmodule(obj, name) try: __import__(modulename) module = sys.modules[modulename] same_as_obj = getattr(module, name) except (ImportError, KeyError, AttributeError): raise pickle.PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, modulename, name)) else: if same_as_obj is not obj: raise pickle.PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, modulename, name)) code = copy_reg._extension_registry.get((modulename, name)) if code: assert code > 0 if code <= 0xFF: size = 2 elif code <= 0xFFFF: size = 3 else: size = 5 else: size = 3 + len(modulename) + len(name) return size
def _save_global(self, path, obj, name=None, pack=struct.pack): if name is None: name = obj.__name__ module = getattr(obj, "__module__", None) if module is None: module = whichmodule(obj, name) try: __import__(module) mod = sys.modules[module] klass = getattr(mod, name) except (ImportError, KeyError, AttributeError): raise PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, module, name)) else: if klass is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module, name)) pickletype = None code = _extension_registry.get((module, name)) if code: assert code > 0 pickletype = EXT4 stuff = pack("<i", code) if not pickletype: stuff = module + '\n' + name pickletype = GLOBAL array = self.file.save_array(path, str(stuff)) self.file.set_attr(array, 'pickletype', pickletype)
def _ufunc_reduce(func): from pickle import whichmodule name = func.__name__ return _ufunc_reconstruct, (whichmodule(func, name), name)
def save_function(self, obj, name=None, pack=struct.pack): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ write = self.write name = obj.__name__ modname = pickle.whichmodule(obj, name) #print 'which gives %s %s %s' % (modname, obj, name) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) if self.os_env_vars: # recursion hackery os_env_vars = self.os_env_vars self.os_env_vars = [] old_saved_django = self.savedDjangoEnv self.savedDjangoEnv = True write(pickle.MARK) if isinstance(os_env_vars, dict): os_mapping = os_env_vars else: os_mapping = {} for env_var in os_env_vars: if env_var in os.environ: os_mapping[env_var] = os.environ[env_var] self.save_reduce(env_vars_load, (os_mapping,)) # note: nothing sensible to memoize write(pickle.POP_MARK) self.savedDjangoEnv = old_saved_django if modname == 'cloud.shell': # don't save django environment if we are using shell.exec self.savedDjangoEnv = True elif not self.savedDjangoEnv: # hack for django - if we detect the settings module, we transport it # unfortunately this module is dynamically, not statically, resolved, so # dependency analysis never detects it django_settings = os.environ.get('DJANGO_SETTINGS_MODULE', '') if django_settings: django_mod = sys.modules.get(django_settings) if django_mod: cloudLog.debug('Transporting django settings %s during save of %s', django_mod, name) self.savedDjangoEnv = True self.modules.add(django_mod) write(pickle.MARK) self.save_reduce(django_settings_load, (django_mod.__name__,), obj=django_mod) write(pickle.POP_MARK) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if islambda(obj) or obj.__code__.co_filename == '<stdin>' or themodule == None: #Force server to import modules that have been imported in main modList = None if themodule == None and not self.savedForceImports: mainmod = sys.modules['__main__'] if useForcedImports and hasattr(mainmod,'___pyc_forcedImports__'): modList = list(mainmod.___pyc_forcedImports__) self.savedForceImports = True self.save_function_tuple(obj, modList) return else: # func is nested klass = getattr(themodule, name, None) if klass is None or klass is not obj: self.save_function_tuple(obj, [themodule]) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) if six.PY3: write(pickle.MARK + pickle.GLOBAL + (modname + '\n' + name + '\n').encode('utf-8')) else: write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: if six.PY3: write(pickle.GLOBAL + (modname + '\n' + name + '\n').encode('utf-8')) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def save_global(self, obj, name=None, pack=struct.pack): if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ self.save(_load_class) self.save_reduce(typ, (obj.__name__, obj.__bases__, { "__doc__": obj.__doc__ }), obj=obj) d.pop('__doc__', None) # handle property and staticmethod dd = {} for k, v in d.items(): if isinstance(v, property): k = ('property', k) v = (v.fget, v.fset, v.fdel, v.__doc__) elif isinstance(v, staticmethod) and hasattr(v, '__func__'): k = ('staticmethod', k) v = v.__func__ elif isinstance(v, classmethod) and hasattr(v, '__func__'): k = ('classmethod', k) v = v.__func__ dd[k] = v self.save(dd) self.write(pickle.TUPLE2) self.write(pickle.REDUCE) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def get_memmapping_reducers( pool_id, forward_reducers=None, backward_reducers=None, temp_folder=None, max_nbytes=1e6, mmap_mode='r', verbose=0, prewarm=False, **kwargs): """Construct a pair of memmapping reducer linked to a tmpdir. This function manage the creation and the clean up of the temporary folders underlying the memory maps and should be use to get the reducers necessary to construct joblib pool or executor. """ if forward_reducers is None: forward_reducers = dict() if backward_reducers is None: backward_reducers = dict() # Prepare a sub-folder name for the serialization of this particular # pool instance (do not create in advance to spare FS write access if # no array is to be dumped): pool_folder_name = "joblib_memmapping_folder_{}_{}".format( os.getpid(), pool_id) pool_folder, use_shared_mem = _get_temp_dir(pool_folder_name, temp_folder) # Register the garbage collector at program exit in case caller forgets # to call terminate explicitly: note we do not pass any reference to # self to ensure that this callback won't prevent garbage collection of # the pool instance and related file handler resources such as POSIX # semaphores and pipes pool_module_name = whichmodule(delete_folder, 'delete_folder') def _cleanup(): # In some cases the Python runtime seems to set delete_folder to # None just before exiting when accessing the delete_folder # function from the closure namespace. So instead we reimport # the delete_folder function explicitly. # https://github.com/joblib/joblib/issues/328 # We cannot just use from 'joblib.pool import delete_folder' # because joblib should only use relative imports to allow # easy vendoring. delete_folder = __import__( pool_module_name, fromlist=['delete_folder']).delete_folder try: delete_folder(pool_folder) except WindowsError: warnings.warn("Failed to clean temporary folder: {}" .format(pool_folder)) atexit.register(_cleanup) if np is not None: # Register smart numpy.ndarray reducers that detects memmap backed # arrays and that is also able to dump to memmap large in-memory # arrays over the max_nbytes threshold if prewarm == "auto": prewarm = not use_shared_mem forward_reduce_ndarray = ArrayMemmapReducer( max_nbytes, pool_folder, mmap_mode, verbose, prewarm=prewarm) forward_reducers[np.ndarray] = forward_reduce_ndarray forward_reducers[np.memmap] = reduce_memmap # Communication from child process to the parent process always # pickles in-memory numpy.ndarray without dumping them as memmap # to avoid confusing the caller and make it tricky to collect the # temporary folder backward_reduce_ndarray = ArrayMemmapReducer( None, pool_folder, mmap_mode, verbose) backward_reducers[np.ndarray] = backward_reduce_ndarray backward_reducers[np.memmap] = reduce_memmap return forward_reducers, backward_reducers, pool_folder
def save_global(self, obj, name=None): # unfortunately the pickler code is factored in a way that # forces us to copy/paste this function. The only change is marked # CHANGED below. write = self.write memo = self.memo if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ orig_module_name = whichmodule(obj, name) # CHANGED: demangle the module name before importing. If this obj came # out of a PackageImporter, `__module__` will be mangled. See # mangling.md for details. module_name = demangle(orig_module_name) try: # CHANGED: self.import_module rather than # __import__ module = self.import_module(module_name) obj2, parent = _getattribute(module, name) except (ImportError, KeyError, AttributeError): raise PicklingError("Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) from None else: if obj2 is not obj: # CHANGED: More specific error message in the case of mangling. obj_module_name = getattr(obj, "__module__", orig_module_name) obj2_module_name = getattr(obj2, "__module__", orig_module_name) msg = f"Can't pickle {obj}: it's not the same object as {obj2_module_name}.{name}." is_obj_mangled = is_mangled(obj_module_name) is_obj2_mangled = is_mangled(obj2_module_name) if is_obj_mangled or is_obj2_mangled: obj_location = (get_mangle_prefix(obj_module_name) if is_obj_mangled else "the current Python environment") obj2_location = (get_mangle_prefix(obj2_module_name) if is_obj2_mangled else "the current Python environment") obj_importer_name = ( f"the importer for {get_mangle_prefix(obj_module_name)}" if is_obj_mangled else "'importlib.import_module'") obj2_importer_name = ( f"the importer for {get_mangle_prefix(obj2_module_name)}" if is_obj2_mangled else "'importlib.import_module'") msg += ( f"\n\nThe object being pickled is from '{orig_module_name}', " f"which is coming from {obj_location}." f"\nHowever, when we import '{orig_module_name}', it's coming from {obj2_location}." "\nTo fix this, make sure 'PackageExporter.importers' lists " f"{obj_importer_name} before {obj2_importer_name}") raise PicklingError(msg) if self.proto >= 2: code = _extension_registry.get((module_name, name)) if code: assert code > 0 if code <= 0xff: write(EXT1 + pack("<B", code)) elif code <= 0xffff: write(EXT2 + pack("<H", code)) else: write(EXT4 + pack("<i", code)) return lastname = name.rpartition('.')[2] if parent is module: name = lastname # Non-ASCII identifiers are supported only with protocols >= 3. if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) elif parent is not module: self.save_reduce(getattr, (parent, lastname)) elif self.proto >= 3: write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + bytes(name, "utf-8") + b'\n') else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: write(GLOBAL + bytes(module_name, "ascii") + b'\n' + bytes(name, "ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, self.proto)) from None self.memoize(obj)
def __init__(self, processes=None, temp_folder=None, max_nbytes=1e6, mmap_mode='r', forward_reducers=None, backward_reducers=None, verbose=0, context_id=None, prewarm=False, **kwargs): if forward_reducers is None: forward_reducers = dict() if backward_reducers is None: backward_reducers = dict() if context_id is not None: warnings.warn('context_id is deprecated and ignored in joblib' ' 0.9.4 and will be removed in 0.11', DeprecationWarning) # Prepare a sub-folder name for the serialization of this particular # pool instance (do not create in advance to spare FS write access if # no array is to be dumped): use_shared_mem = False pool_folder_name = "joblib_memmaping_pool_%d_%d" % ( os.getpid(), id(self)) if temp_folder is None: temp_folder = os.environ.get('JOBLIB_TEMP_FOLDER', None) if temp_folder is None: if os.path.exists(SYSTEM_SHARED_MEM_FS): try: temp_folder = SYSTEM_SHARED_MEM_FS pool_folder = os.path.join(temp_folder, pool_folder_name) if not os.path.exists(pool_folder): os.makedirs(pool_folder) use_shared_mem = True except IOError: # Missing rights in the the /dev/shm partition, # fallback to regular temp folder. temp_folder = None if temp_folder is None: # Fallback to the default tmp folder, typically /tmp temp_folder = tempfile.gettempdir() temp_folder = os.path.abspath(os.path.expanduser(temp_folder)) pool_folder = os.path.join(temp_folder, pool_folder_name) self._temp_folder = pool_folder # Register the garbage collector at program exit in case caller forgets # to call terminate explicitly: note we do not pass any reference to # self to ensure that this callback won't prevent garbage collection of # the pool instance and related file handler resources such as POSIX # semaphores and pipes pool_module_name = whichmodule(delete_folder, 'delete_folder') def _cleanup(): # In some cases the Python runtime seems to set delete_folder to # None just before exiting when accessing the delete_folder # function from the closure namespace. So instead we reimport # the delete_folder function explicitly. # https://github.com/joblib/joblib/issues/328 # We cannot just use from 'joblib.pool import delete_folder' # because joblib should only use relative imports to allow # easy vendoring. delete_folder = __import__( pool_module_name, fromlist=['delete_folder']).delete_folder delete_folder(pool_folder) atexit.register(_cleanup) if np is not None: # Register smart numpy.ndarray reducers that detects memmap backed # arrays and that is alse able to dump to memmap large in-memory # arrays over the max_nbytes threshold if prewarm == "auto": prewarm = not use_shared_mem forward_reduce_ndarray = ArrayMemmapReducer( max_nbytes, pool_folder, mmap_mode, verbose, prewarm=prewarm) forward_reducers[np.ndarray] = forward_reduce_ndarray forward_reducers[np.memmap] = reduce_memmap # Communication from child process to the parent process always # pickles in-memory numpy.ndarray without dumping them as memmap # to avoid confusing the caller and make it tricky to collect the # temporary folder backward_reduce_ndarray = ArrayMemmapReducer( None, pool_folder, mmap_mode, verbose) backward_reducers[np.ndarray] = backward_reduce_ndarray backward_reducers[np.memmap] = reduce_memmap poolargs = dict( processes=processes, forward_reducers=forward_reducers, backward_reducers=backward_reducers) poolargs.update(kwargs) super(MemmapingPool, self).__init__(**poolargs)
def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ write = self.write if name is None: name = obj.__name__ try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = None # print('which gives %s %s %s' % (modname, obj, name)) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) if getattr(themodule, name, None) is obj: return self.save_global(obj, name) # a builtin_function_or_method which comes in as an attribute of some # object (e.g., object.__new__, itertools.chain.from_iterable) will end # up with modname "__main__" and so end up here. But these functions # have no __code__ attribute in CPython, so the handling for # user-defined functions below will fail. # So we pickle them here using save_reduce; have to do it differently # for different python versions. if not hasattr(obj, '__code__'): if PY3: if sys.version_info < (3, 4): raise pickle.PicklingError("Can't pickle %r" % obj) else: rv = obj.__reduce_ex__(self.proto) else: if hasattr(obj, '__self__'): rv = (getattr, (obj.__self__, name)) else: raise pickle.PicklingError("Can't pickle %r" % obj) return Pickler.save_reduce(self, obj=obj, *rv) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if (islambda(obj) or getattr(obj.__code__, 'co_filename', None) == '<stdin>' or themodule is None): self.save_function_tuple(obj) return else: # func is nested klass = getattr(themodule, name, None) if klass is None or klass is not obj: self.save_function_tuple(obj) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def jelly(self, obj): if isinstance(obj, Jellyable): preRef = self._checkMutable(obj) if preRef: return preRef return obj.jellyFor(self) objType = type(obj) if self.taster.isTypeAllowed(qual(objType)): # "Immutable" Types if ((objType is StringType) or (objType is IntType) or (objType is LongType) or (objType is FloatType)): return obj elif objType is MethodType: return ["method", obj.im_func.__name__, self.jelly(obj.im_self), self.jelly(obj.im_class)] elif UnicodeType and objType is UnicodeType: return ['unicode', obj.encode('UTF-8')] elif objType is NoneType: return ['None'] elif objType is FunctionType: name = obj.__name__ return ['function', str(pickle.whichmodule(obj, obj.__name__)) + '.' + name] elif objType is ModuleType: return ['module', obj.__name__] elif objType is BooleanType: return ['boolean', obj and 'true' or 'false'] elif objType is datetime.datetime: if obj.tzinfo: raise NotImplementedError( "Currently can't jelly datetime objects with tzinfo") return ['datetime', '%s %s %s %s %s %s %s' % ( obj.year, obj.month, obj.day, obj.hour, obj.minute, obj.second, obj.microsecond)] elif objType is datetime.time: if obj.tzinfo: raise NotImplementedError( "Currently can't jelly datetime objects with tzinfo") return ['time', '%s %s %s %s' % (obj.hour, obj.minute, obj.second, obj.microsecond)] elif objType is datetime.date: return ['date', '%s %s %s' % (obj.year, obj.month, obj.day)] elif objType is datetime.timedelta: return ['timedelta', '%s %s %s' % (obj.days, obj.seconds, obj.microseconds)] elif objType is ClassType or issubclass(objType, type): return ['class', qual(obj)] elif decimal is not None and objType is decimal.Decimal: return self.jelly_decimal(obj) else: preRef = self._checkMutable(obj) if preRef: return preRef # "Mutable" Types sxp = self.prepare(obj) if objType is ListType: sxp.extend(self._jellyIterable(list_atom, obj)) elif objType is TupleType: sxp.extend(self._jellyIterable(tuple_atom, obj)) elif objType in DictTypes: sxp.append(dictionary_atom) for key, val in obj.items(): sxp.append([self.jelly(key), self.jelly(val)]) elif (_set is not None and objType is set or objType is _sets.Set): sxp.extend(self._jellyIterable(set_atom, obj)) elif (_set is not None and objType is frozenset or objType is _sets.ImmutableSet): sxp.extend(self._jellyIterable(frozenset_atom, obj)) else: className = qual(obj.__class__) persistent = None if self.persistentStore: persistent = self.persistentStore(obj, self) if persistent is not None: sxp.append(persistent_atom) sxp.append(persistent) elif self.taster.isClassAllowed(obj.__class__): sxp.append(className) if hasattr(obj, "__getstate__"): state = obj.__getstate__() else: state = obj.__dict__ sxp.append(self.jelly(state)) else: self.unpersistable( "instance of class %s deemed insecure" % qual(obj.__class__), sxp) return self.preserve(obj, sxp) else: if objType is InstanceType: raise InsecureJelly("Class not allowed for instance: %s %s" % (obj.__class__, obj)) raise InsecureJelly("Type not allowed for object: %s %s" % (objType, obj))
def sliceBody(self, streamable, banana): name = self.obj.__name__ fullname = str(whichmodule(self.obj, self.obj.__name__)) + '.' + name yield fullname
def save_global(self, obj, name=None, pack=struct.pack): # pylint: disable=too-many-branches if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = '__main__' if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ # workaround for namedtuple (hijacked by PySpark) if getattr(obj, '_is_namedtuple_', False): self.save_reduce(_load_namedtuple, (obj.__name__, obj._fields)) return self.save(_load_class) self.save_reduce(typ, (obj.__name__, obj.__bases__, { "__doc__": obj.__doc__ }), obj=obj) d.pop('__doc__', None) # handle property and staticmethod dd = {} for k, v in d.items(): if isinstance(v, property): k = ('property', k) v = (v.fget, v.fset, v.fdel, v.__doc__) elif isinstance(v, staticmethod) and hasattr(v, '__func__'): k = ('staticmethod', k) v = v.__func__ elif isinstance(v, classmethod) and hasattr(v, '__func__'): k = ('classmethod', k) v = v.__func__ dd[k] = v self.save(dd) self.write(pickle.TUPLE2) self.write(pickle.REDUCE) else: raise pickle.PicklingError("Can't pickle %r" % obj)
def get_class_path(cls): module_name = pickle.whichmodule(cls, cls.__name__) return module_name + "." + cls.__name__
def save_function(self, obj, name=None, pack=struct.pack): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ write = self.write name = obj.__name__ modname = pickle.whichmodule(obj, name) #print 'which gives %s %s %s' % (modname, obj, name) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) if getattr(themodule, name, None) is obj: return self.save_global(obj, name) if not self.savedDjangoEnv: #hack for django - if we detect the settings module, we transport it django_settings = os.environ.get('DJANGO_SETTINGS_MODULE', '') if django_settings: django_mod = sys.modules.get(django_settings) if django_mod: cloudLog.debug( 'Transporting django settings %s during save of %s', django_mod, name) self.savedDjangoEnv = True self.modules.add(django_mod) write(pickle.MARK) self.save_reduce(django_settings_load, (django_mod.__name__, ), obj=django_mod) write(pickle.POP_MARK) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if islambda( obj ) or obj.func_code.co_filename == '<stdin>' or themodule is None: #Force server to import modules that have been imported in main modList = None if themodule is None and not self.savedForceImports: mainmod = sys.modules['__main__'] if useForcedImports and hasattr(mainmod, '___pyc_forcedImports__'): modList = list(mainmod.___pyc_forcedImports__) self.savedForceImports = True self.save_function_tuple(obj, modList) return else: # func is nested klass = getattr(themodule, name, None) if klass is None or klass is not obj: self.save_function_tuple(obj, [themodule]) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def fullFuncName(func): qualName = str(pickle.whichmodule(func, func.__name__)) + "." + func.__name__ if namedObject(qualName) is not func: raise Exception("Couldn't find {} as {}.".format(func, qualName)) return qualName
def save_global(self, obj, name=None, pack=struct.pack): write = self.write memo = self.memo if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) try: __import__(modname) themodule = sys.modules[modname] except (ImportError, KeyError, AttributeError): #should never occur raise pickle.PicklingError( "Can't pickle %r: Module %s cannot be found" % (obj, modname)) if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) sendRef = True typ = type(obj) #print 'saving', obj, typ try: try: #Deal with case when getattribute fails with exceptions klass = getattr(themodule, name) except (AttributeError): if modname == '__builtin__': #new.* are misrepeported modname = 'new' __import__(modname) themodule = sys.modules[modname] try: klass = getattr(themodule, name) except AttributeError as a: #print themodule, name, obj, type(obj) raise pickle.PicklingError("Can't pickle builtin %s" % obj) else: raise except (ImportError, KeyError, AttributeError): if typ == type: sendRef = False else: #we can't deal with this raise else: if klass is not obj and (typ == type): sendRef = False if not sendRef: self.save_class_obj(obj, name, pack) return if self.proto >= 2: code = _extension_registry.get((modname, name)) if code: assert code > 0 if code <= 0xff: write(pickle.EXT1 + chr(code)) elif code <= 0xffff: write("%c%c%c" % (pickle.EXT2, code&0xff, code>>8)) else: write(pickle.EXT4 + pack("<i", code)) return if six.PY3: write(pickle.GLOBAL + (modname + '\n' + name + '\n').encode('utf-8')) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ try: should_special_case = obj in _BUILTIN_TYPE_CONSTRUCTORS except TypeError: # Methods of builtin types aren't hashable in python 2. should_special_case = False if should_special_case: # We keep a special-cased cache of built-in type constructors at # global scope, because these functions are structured very # differently in different python versions and implementations (for # example, they're instances of types.BuiltinFunctionType in # CPython, but they're ordinary types.FunctionType instances in # PyPy). # # If the function we've received is in that cache, we just # serialize it as a lookup into the cache. return self.save_reduce(_BUILTIN_TYPE_CONSTRUCTORS[obj], (), obj=obj) write = self.write if name is None: name = obj.__name__ try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = None # print('which gives %s %s %s' % (modname, obj, name)) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None try: lookedup_by_name = getattr(themodule, name, None) except Exception: lookedup_by_name = None if themodule: if lookedup_by_name is obj: return self.save_global(obj, name) # a builtin_function_or_method which comes in as an attribute of some # object (e.g., itertools.chain.from_iterable) will end # up with modname "__main__" and so end up here. But these functions # have no __code__ attribute in CPython, so the handling for # user-defined functions below will fail. # So we pickle them here using save_reduce; have to do it differently # for different python versions. if not hasattr(obj, '__code__'): if PY3: # pragma: no branch rv = obj.__reduce_ex__(self.proto) else: if hasattr(obj, '__self__'): rv = (getattr, (obj.__self__, name)) else: raise pickle.PicklingError("Can't pickle %r" % obj) return self.save_reduce(obj=obj, *rv) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if (islambda(obj) or getattr(obj.__code__, 'co_filename', None) == '<stdin>' or themodule is None): self.save_function_tuple(obj) return else: # func is nested if lookedup_by_name is None or lookedup_by_name is not obj: self.save_function_tuple(obj) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def save_global(self, obj, name=None, pack=struct.pack): write = self.write memo = self.memo if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) try: __import__(modname) themodule = sys.modules[modname] except (ImportError, KeyError, AttributeError): #should never occur raise pickle.PicklingError( "Can't pickle %r: Module %s cannot be found" % (obj, modname)) if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) sendRef = True typ = type(obj) #print 'saving', obj, typ try: try: #Deal with case when getattribute fails with exceptions klass = getattr(themodule, name) except (AttributeError): if modname == '__builtin__': #new.* are misrepeported modname = 'new' __import__(modname) themodule = sys.modules[modname] try: klass = getattr(themodule, name) except AttributeError, a: #print themodule, name, obj, type(obj) raise pickle.PicklingError("Can't pickle builtin %s" % obj) else: raise except (ImportError, KeyError, AttributeError): if typ == types.TypeType or typ == types.ClassType: sendRef = False else: #we can't deal with this raise else: if klass is not obj and (typ == types.TypeType or typ == types.ClassType): sendRef = False if not sendRef: #note: Third party types might crash this - add better checks! d = dict(obj.__dict__) #copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__',None) d.pop('__weakref__',None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ self.save_reduce(type(obj),(obj.__name__,obj.__bases__, d),obj=obj) #print 'internal reduce dask %s %s' % (obj, d) return if self.proto >= 2: code = _extension_registry.get((modname, name)) if code: assert code > 0 if code <= 0xff: write(pickle.EXT1 + chr(code)) elif code <= 0xffff: write("%c%c%c" % (pickle.EXT2, code&0xff, code>>8)) else: write(pickle.EXT4 + pack("<i", code)) return write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ write = self.write if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = getattr(obj, '__name__', None) try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = None # print('which gives %s %s %s' % (modname, obj, name)) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None try: lookedup_by_name, _ = _getattribute(themodule, name) except Exception: lookedup_by_name = None if themodule: if lookedup_by_name is obj: return self.save_global(obj, name) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if (islambda(obj) or getattr(obj.__code__, 'co_filename', None) == '<stdin>' or themodule is None): self.save_function_tuple(obj) return else: # func is nested if lookedup_by_name is None or lookedup_by_name is not obj: self.save_function_tuple(obj) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def save_global_py3(obj, name=None, proto=2, fix_imports=True): if name is None and proto >= 4: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ module_name = whichmodule(obj, name, allow_qualname=proto >= 4) try: __import__(module_name, level=0) module = sys.modules[module_name] obj2 = _getattribute(module, name, allow_qualname=proto >= 4) except (ImportError, KeyError, AttributeError): raise PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) else: if obj2 is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module_name, name)) if proto >= 2: code = _extension_registry.get((module_name, name)) if code: # assert code > 0 # if code <= 0xff: # write(EXT1 + pack("<B", code)) # elif code <= 0xffff: # write(EXT2 + pack("<H", code)) # else: # write(EXT4 + pack("<i", code)) # return raise H5itPicklingError("h5it Can't pickle %r: extension codes are not" " supported yet." % obj) # Non-ASCII identifiers are supported only with protocols >= 3. if proto >= 4: # self.save(module_name) # self.save(name) # write(STACK_GLOBAL) raise H5itPicklingError("h5it Can't pickle %r: protocol %i is not " "supported yet." % (obj, proto)) elif proto >= 3: # write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + # bytes(name, "utf-8") + b'\n') raise H5itPicklingError("h5it Can't pickle %r: protocol %i is not " "supported yet." % (obj, proto)) else: if fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] if module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: return GlobalTuple(bytes(module_name, "ascii"), bytes(name, "ascii")) except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, proto))
def jelly(self, obj): if isinstance(obj, Jellyable): preRef = self._checkMutable(obj) if preRef: return preRef return obj.jellyFor(self) objType = type(obj) if self.taster.isTypeAllowed(qual(objType)): # "Immutable" Types if ((objType is StringType) or (objType is IntType) or (objType is LongType) or (objType is FloatType)): return obj elif objType is MethodType: return [ "method", obj.im_func.__name__, self.jelly(obj.im_self), self.jelly(obj.im_class) ] elif UnicodeType and objType is UnicodeType: return ['unicode', obj.encode('UTF-8')] elif objType is NoneType: return ['None'] elif objType is FunctionType: name = obj.__name__ return [ 'function', str(pickle.whichmodule(obj, obj.__name__)) + '.' + name ] elif objType is ModuleType: return ['module', obj.__name__] elif objType is BooleanType: return ['boolean', obj and 'true' or 'false'] elif objType is datetime.datetime: if obj.tzinfo: raise NotImplementedError( "Currently can't jelly datetime objects with tzinfo") return [ 'datetime', '%s %s %s %s %s %s %s' % (obj.year, obj.month, obj.day, obj.hour, obj.minute, obj.second, obj.microsecond) ] elif objType is datetime.time: if obj.tzinfo: raise NotImplementedError( "Currently can't jelly datetime objects with tzinfo") return [ 'time', '%s %s %s %s' % (obj.hour, obj.minute, obj.second, obj.microsecond) ] elif objType is datetime.date: return ['date', '%s %s %s' % (obj.year, obj.month, obj.day)] elif objType is datetime.timedelta: return [ 'timedelta', '%s %s %s' % (obj.days, obj.seconds, obj.microseconds) ] elif objType is ClassType or issubclass(objType, type): return ['class', qual(obj)] elif decimal is not None and objType is decimal.Decimal: return self.jelly_decimal(obj) else: preRef = self._checkMutable(obj) if preRef: return preRef # "Mutable" Types sxp = self.prepare(obj) if objType is ListType: sxp.extend(self._jellyIterable(list_atom, obj)) elif objType is TupleType: sxp.extend(self._jellyIterable(tuple_atom, obj)) elif objType in DictTypes: sxp.append(dictionary_atom) for key, val in obj.items(): sxp.append([self.jelly(key), self.jelly(val)]) elif (_set is not None and objType is set or objType is _sets.Set): sxp.extend(self._jellyIterable(set_atom, obj)) elif (_set is not None and objType is frozenset or objType is _sets.ImmutableSet): sxp.extend(self._jellyIterable(frozenset_atom, obj)) else: className = qual(obj.__class__) persistent = None if self.persistentStore: persistent = self.persistentStore(obj, self) if persistent is not None: sxp.append(persistent_atom) sxp.append(persistent) elif self.taster.isClassAllowed(obj.__class__): sxp.append(className) if hasattr(obj, "__getstate__"): state = obj.__getstate__() else: state = obj.__dict__ sxp.append(self.jelly(state)) else: self.unpersistable( "instance of class %s deemed insecure" % qual(obj.__class__), sxp) return self.preserve(obj, sxp) else: if objType is InstanceType: raise InsecureJelly("Class not allowed for instance: %s %s" % (obj.__class__, obj)) raise InsecureJelly("Type not allowed for object: %s %s" % (objType, obj))
self.__listenTask.servListen() if self.__listenTask.hasStopServs(): time.sleep(4) self.__listenTask.servReStart() time.sleep(4) def SvcStop(self): self.__run = False time.sleep(2) self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) win32event.SetEvent(self.hWaitStop) if __name__ == '__main__': if len(sys.argv) == 1: try: evtsrc_dll = os.path.abspath(servicemanager.__file__) servicemanager.PrepareToHostSingle(AIZYingListenService) servicemanager.Initialize('AIZYingListenService', evtsrc_dll) servicemanager.StartServiceCtrlDispatcher() except win32service.error as details: if details[0] == winerror.ERROR_FAILED_SERVICE_CONTROLLER_CONNECT: win32serviceutil.usage() else: import pickle modName = pickle.whichmodule(AIZYingListenService, AIZYingListenService.__name__) win32serviceutil.HandleCommandLine(AIZYingListenService)
def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ write = self.write if name is None: name = obj.__name__ modname = pickle.whichmodule(obj, name) # print('which gives %s %s %s' % (modname, obj, name)) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) if getattr(themodule, name, None) is obj: return self.save_global(obj, name) # a builtin_function_or_method which comes in as an attribute of some # object (e.g., object.__new__, itertools.chain.from_iterable) will end # up with modname "__main__" and so end up here. But these functions # have no __code__ attribute in CPython, so the handling for # user-defined functions below will fail. # So we pickle them here using save_reduce; have to do it differently # for different python versions. if not hasattr(obj, '__code__'): if sys.version_info >= (3, 4): rv = obj.__reduce_ex__(self.proto) elif hasattr(obj, '__self__'): rv = (getattr, (obj.__self__, name)) else: raise pickle.PicklingError("Can't pickle %r" % obj) return Pickler.save_reduce(self, obj=obj, *rv) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if (islambda(obj) or getattr(obj.__code__, 'co_filename', None) == '<stdin>' or themodule is None): self.save_function_tuple(obj) return else: # func is nested klass = getattr(themodule, name, None) if klass is None or klass is not obj: self.save_function_tuple(obj) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def save_global(self, obj, name=None): # unfortunately the pickler code is factored in a way that # forces us to copy/paste this function. The only change is marked # CHANGED below. write = self.write memo = self.memo if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ module_name = whichmodule(obj, name) try: # CHANGED: self.import_module rather than # __import__ module = self.import_module(module_name) obj2, parent = _getattribute(module, name) except (ImportError, KeyError, AttributeError): raise PicklingError("Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) from None else: if obj2 is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module_name, name)) if self.proto >= 2: code = _extension_registry.get((module_name, name)) if code: assert code > 0 if code <= 0xff: write(EXT1 + pack("<B", code)) elif code <= 0xffff: write(EXT2 + pack("<H", code)) else: write(EXT4 + pack("<i", code)) return lastname = name.rpartition('.')[2] if parent is module: name = lastname # Non-ASCII identifiers are supported only with protocols >= 3. if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) elif parent is not module: self.save_reduce(getattr, (parent, lastname)) elif self.proto >= 3: write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + bytes(name, "utf-8") + b'\n') else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: write(GLOBAL + bytes(module_name, "ascii") + b'\n' + bytes(name, "ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, self.proto)) from None self.memoize(obj)
def fullFuncName(func): qualName = (str(pickle.whichmodule(func, func.__name__)) + '.' + func.__name__) if namedObject(qualName) is not func: raise Exception("Couldn't find %s as %s." % (func, qualName)) return qualName
def jelly(self, obj): if isinstance(obj, Jellyable): refId = self._ref_id self._ref_id += 1 preRef = self._checkMutable(obj, refId) if preRef: return preRef return obj.jellyFor(self) objType = type(obj) if self.taster.isTypeAllowed( string.replace(objType.__name__, ' ', '_')): if ((objType is StringType) or (objType is IntType) or (objType is LongType) or (objType is FloatType)): return obj refId = self._ref_id self._ref_id += 1 if objType is MethodType: return ["method", obj.im_func.__name__, self.jelly(obj.im_self), self.jelly(obj.im_class)] elif UnicodeType and objType is UnicodeType: return ['unicode', obj.encode('UTF-8')] elif objType is NoneType: return ['None'] elif objType is FunctionType: name = obj.__name__ return ['function', str(pickle.whichmodule(obj, obj.__name__)) + '.' + name] elif objType is ModuleType: return ['module', obj.__name__] elif objType is BooleanType: return ['boolean', obj and 'true' or 'false'] elif objType is ClassType or issubclass(objType, type): return ['class', qual(obj)] else: preRef = self._checkMutable(obj, refId) if preRef: return preRef sxp = [] if objType is ListType: sxp.append(list_atom) for item in obj: sxp.append(self.jelly(item)) elif objType is TupleType: sxp.append(tuple_atom) for item in obj: sxp.append(self.jelly(item)) elif objType in DictTypes: sxp.append(dictionary_atom) for key, val in obj.items(): self._ref_id += 1 sxp.append([self.jelly(key), self.jelly(val)]) elif objType is InstanceType: className = qual(obj.__class__) if self.taster.isClassAllowed(obj.__class__): sxp.append(className) if hasattr(obj, "__getstate__"): state = obj.__getstate__() else: state = obj.__dict__ sxp.append(self.jelly(state)) else: self.unpersistable( "instance of class %s deemed insecure" % qual(obj.__class__), sxp) else: raise NotImplementedError("Don't know the type: %s" % objType) return sxp else: if objType is types.InstanceType: raise InsecureJelly("Class not allowed for instance: %s %s" % (obj.__class__, obj)) raise InsecureJelly("Type not allowed for object: %s %s" % (objType, obj))
def __init__(self, processes=None, temp_folder=None, max_nbytes=1e6, mmap_mode='r', forward_reducers=None, backward_reducers=None, verbose=0, context_id=None, prewarm=False, **kwargs): if forward_reducers is None: forward_reducers = dict() if backward_reducers is None: backward_reducers = dict() if context_id is not None: warnings.warn( 'context_id is deprecated and ignored in joblib' ' 0.9.4 and will be removed in 0.11', DeprecationWarning) # Prepare a sub-folder name for the serialization of this particular # pool instance (do not create in advance to spare FS write access if # no array is to be dumped): use_shared_mem = False pool_folder_name = "joblib_memmaping_pool_%d_%d" % (os.getpid(), id(self)) if temp_folder is None: temp_folder = os.environ.get('JOBLIB_TEMP_FOLDER', None) if temp_folder is None: if os.path.exists(SYSTEM_SHARED_MEM_FS): try: temp_folder = SYSTEM_SHARED_MEM_FS pool_folder = os.path.join(temp_folder, pool_folder_name) if not os.path.exists(pool_folder): os.makedirs(pool_folder) use_shared_mem = True except IOError: # Missing rights in the the /dev/shm partition, # fallback to regular temp folder. temp_folder = None if temp_folder is None: # Fallback to the default tmp folder, typically /tmp temp_folder = tempfile.gettempdir() temp_folder = os.path.abspath(os.path.expanduser(temp_folder)) pool_folder = os.path.join(temp_folder, pool_folder_name) self._temp_folder = pool_folder # Register the garbage collector at program exit in case caller forgets # to call terminate explicitly: note we do not pass any reference to # self to ensure that this callback won't prevent garbage collection of # the pool instance and related file handler resources such as POSIX # semaphores and pipes pool_module_name = whichmodule(delete_folder, 'delete_folder') def _cleanup(): # In some cases the Python runtime seems to set delete_folder to # None just before exiting when accessing the delete_folder # function from the closure namespace. So instead we reimport # the delete_folder function explicitly. # https://github.com/joblib/joblib/issues/328 # We cannot just use from 'joblib.pool import delete_folder' # because joblib should only use relative imports to allow # easy vendoring. delete_folder = __import__(pool_module_name, fromlist=['delete_folder' ]).delete_folder delete_folder(pool_folder) atexit.register(_cleanup) if np is not None: # Register smart numpy.ndarray reducers that detects memmap backed # arrays and that is alse able to dump to memmap large in-memory # arrays over the max_nbytes threshold if prewarm == "auto": prewarm = not use_shared_mem forward_reduce_ndarray = ArrayMemmapReducer(max_nbytes, pool_folder, mmap_mode, verbose, prewarm=prewarm) forward_reducers[np.ndarray] = forward_reduce_ndarray forward_reducers[np.memmap] = reduce_memmap # Communication from child process to the parent process always # pickles in-memory numpy.ndarray without dumping them as memmap # to avoid confusing the caller and make it tricky to collect the # temporary folder backward_reduce_ndarray = ArrayMemmapReducer( None, pool_folder, mmap_mode, verbose) backward_reducers[np.ndarray] = backward_reduce_ndarray backward_reducers[np.memmap] = reduce_memmap poolargs = dict(processes=processes, forward_reducers=forward_reducers, backward_reducers=backward_reducers) poolargs.update(kwargs) super(MemmapingPool, self).__init__(**poolargs)
def get_memmapping_reducers(pool_id, forward_reducers=None, backward_reducers=None, temp_folder=None, max_nbytes=1e6, mmap_mode='r', verbose=0, prewarm=False, **kwargs): """Construct a pair of memmapping reducer linked to a tmpdir. This function manage the creation and the clean up of the temporary folders underlying the memory maps and should be use to get the reducers necessary to construct joblib pool or executor. """ if forward_reducers is None: forward_reducers = dict() if backward_reducers is None: backward_reducers = dict() # Prepare a sub-folder name for the serialization of this particular # pool instance (do not create in advance to spare FS write access if # no array is to be dumped): pool_folder_name = "joblib_memmapping_folder_{}_{}".format( os.getpid(), pool_id) pool_folder, use_shared_mem = _get_temp_dir(pool_folder_name, temp_folder) # Register the garbage collector at program exit in case caller forgets # to call terminate explicitly: note we do not pass any reference to # self to ensure that this callback won't prevent garbage collection of # the pool instance and related file handler resources such as POSIX # semaphores and pipes pool_module_name = whichmodule(delete_folder, 'delete_folder') def _cleanup(): # In some cases the Python runtime seems to set delete_folder to # None just before exiting when accessing the delete_folder # function from the closure namespace. So instead we reimport # the delete_folder function explicitly. # https://github.com/joblib/joblib/issues/328 # We cannot just use from 'joblib.pool import delete_folder' # because joblib should only use relative imports to allow # easy vendoring. delete_folder = __import__(pool_module_name, fromlist=['delete_folder']).delete_folder try: delete_folder(pool_folder) except WindowsError: warnings.warn( "Failed to clean temporary folder: {}".format(pool_folder)) atexit.register(_cleanup) if np is not None: # Register smart numpy.ndarray reducers that detects memmap backed # arrays and that is also able to dump to memmap large in-memory # arrays over the max_nbytes threshold if prewarm == "auto": prewarm = not use_shared_mem forward_reduce_ndarray = ArrayMemmapReducer(max_nbytes, pool_folder, mmap_mode, verbose, prewarm=prewarm) forward_reducers[np.ndarray] = forward_reduce_ndarray forward_reducers[np.memmap] = reduce_memmap # Communication from child process to the parent process always # pickles in-memory numpy.ndarray without dumping them as memmap # to avoid confusing the caller and make it tricky to collect the # temporary folder backward_reduce_ndarray = ArrayMemmapReducer(None, pool_folder, mmap_mode, verbose) backward_reducers[np.ndarray] = backward_reduce_ndarray backward_reducers[np.memmap] = reduce_memmap return forward_reducers, backward_reducers, pool_folder
def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ try: should_special_case = obj in _BUILTIN_TYPE_CONSTRUCTORS except TypeError: # Methods of builtin types aren't hashable in python 2. should_special_case = False if should_special_case: # We keep a special-cased cache of built-in type constructors at # global scope, because these functions are structured very # differently in different python versions and implementations (for # example, they're instances of types.BuiltinFunctionType in # CPython, but they're ordinary types.FunctionType instances in # PyPy). # # If the function we've received is in that cache, we just # serialize it as a lookup into the cache. return self.save_reduce(_BUILTIN_TYPE_CONSTRUCTORS[obj], (), obj=obj) write = self.write if name is None: name = obj.__name__ try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = None # print('which gives %s %s %s' % (modname, obj, name)) themodule = None try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' # if modname == '__main__': # themodule = None try: lookedup_by_name = getattr(themodule, name, None) except Exception: lookedup_by_name = None if themodule and modname != '__main__': if lookedup_by_name is obj: return self.save_global(obj, name) # a builtin_function_or_method which comes in as an attribute of some # object (e.g., itertools.chain.from_iterable) will end # up with modname "__main__" and so end up here. But these functions # have no __code__ attribute in CPython, so the handling for # user-defined functions below will fail. # So we pickle them here using save_reduce; have to do it differently # for different python versions. if not hasattr(obj, '__code__'): if PY3: # pragma: no branch rv = obj.__reduce_ex__(self.proto) else: if hasattr(obj, '__self__'): rv = (getattr, (obj.__self__, name)) else: raise pickle.PicklingError("Can't pickle %r" % obj) return self.save_reduce(obj=obj, *rv) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if (islambda(obj) or getattr(obj.__code__, 'co_filename', None) == '<stdin>' or themodule is None): self.save_function_tuple(obj) return else: # func is nested if lookedup_by_name is None or lookedup_by_name is not obj: self.save_function_tuple(obj) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname.encode('utf-8') + b'\n' + name.encode('utf-8') + b'\n') self.memoize(obj)
async def save_global(self, obj, name=None): write = self.write memo = self.memo if name is None: name = getattr(obj, '__qualname__', None) if name is None: name = obj.__name__ module_name = whichmodule(obj, name) try: __import__(module_name, level=0) module = sys.modules[module_name] obj2, parent = _getattribute(module, name) except (ImportError, KeyError, AttributeError): raise PicklingError( "Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) from None else: if obj2 is not obj: raise PicklingError( "Can't pickle %r: it's not the same object as %s.%s" % (obj, module_name, name)) if self.proto >= 2: code = _extension_registry.get((module_name, name)) if code: assert code > 0 if code <= 0xff: await write(EXT1 + pack("<B", code)) elif code <= 0xffff: await write(EXT2 + pack("<H", code)) else: await write(EXT4 + pack("<i", code)) return lastname = name.rpartition('.')[2] if parent is module: name = lastname # Non-ASCII identifiers are supported only with protocols >= 3. if self.proto >= 4: await self.save(module_name) await self.save(name) await write(STACK_GLOBAL) elif parent is not module: await self.save_reduce(getattr, (parent, lastname)) elif self.proto >= 3: await write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + bytes(name, "utf-8") + b'\n') else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING if (module_name, name) in r_name_mapping: module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: await write(GLOBAL + bytes(module_name, "ascii") + b'\n' + bytes(name, "ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, self.proto)) from None await self.memoize(obj)
def save_function(self, obj, name=None, pack=struct.pack): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ write = self.write name = obj.__name__ modname = pickle.whichmodule(obj, name) #print 'which gives %s %s %s' % (modname, obj, name) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) if self.os_env_vars: # recursion hackery os_env_vars = self.os_env_vars self.os_env_vars = [] old_saved_django = self.savedDjangoEnv self.savedDjangoEnv = True write(pickle.MARK) if isinstance(os_env_vars, dict): os_mapping = os_env_vars else: os_mapping = {} for env_var in os_env_vars: if env_var in os.environ: os_mapping[env_var] = os.environ[env_var] self.save_reduce( env_vars_load, (os_mapping, )) # note: nothing sensible to memoize write(pickle.POP_MARK) self.savedDjangoEnv = old_saved_django if modname == 'cloud.shell': # don't save django environment if we are using shell.exec self.savedDjangoEnv = True elif not self.savedDjangoEnv: # hack for django - if we detect the settings module, we transport it # unfortunately this module is dynamically, not statically, resolved, so # dependency analysis never detects it django_settings = os.environ.get('DJANGO_SETTINGS_MODULE', '') if django_settings: django_mod = sys.modules.get(django_settings) if django_mod: logger.debug( 'Transporting django settings %s during save of %s', django_mod, name) self.savedDjangoEnv = True self.modules.add(django_mod) write(pickle.MARK) self.save_reduce(django_settings_load, (django_mod.__name__, ), obj=django_mod) write(pickle.POP_MARK) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if islambda( obj ) or obj.func_code.co_filename == '<stdin>' or themodule == None: #Force server to import modules that have been imported in main modList = None if themodule == None and not self.savedForceImports: mainmod = sys.modules['__main__'] if useForcedImports and hasattr(mainmod, '___pyc_forcedImports__'): modList = list(mainmod.___pyc_forcedImports__) self.savedForceImports = True self.save_function_tuple(obj, modList) return else: # func is nested klass = getattr(themodule, name, None) if klass is None or klass is not obj: self.save_function_tuple(obj, [themodule]) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def save_global(self, obj, name=None, pack=struct.pack): write = self.write memo = self.memo if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) try: __import__(modname) themodule = sys.modules[modname] except (ImportError, KeyError, AttributeError): #should never occur raise pickle.PicklingError( "Can't pickle %r: Module %s cannot be found" % (obj, modname)) if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) sendRef = True typ = type(obj) #print 'saving', obj, typ try: try: #Deal with case when getattribute fails with exceptions klass = getattr(themodule, name) except (AttributeError): if modname == '__builtin__': #new.* are misrepeported modname = 'new' __import__(modname) themodule = sys.modules[modname] try: klass = getattr(themodule, name) except AttributeError, a: # print themodule, name, obj, type(obj) raise pickle.PicklingError("Can't pickle builtin %s" % obj) else: raise except (ImportError, KeyError, AttributeError): if typ == types.TypeType or typ == types.ClassType: sendRef = False else: #we can't deal with this raise else: if klass is not obj and (typ == types.TypeType or typ == types.ClassType): sendRef = False if not sendRef: #note: Third party types might crash this - add better checks! d = dict(obj.__dict__) #copy dict proxy to a dict if not isinstance( d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ self.save_reduce(type(obj), (obj.__name__, obj.__bases__, d), obj=obj) #print 'internal reduce dask %s %s' % (obj, d) return if self.proto >= 2: code = _extension_registry.get((modname, name)) if code: assert code > 0 if code <= 0xff: write(pickle.EXT1 + chr(code)) elif code <= 0xffff: write("%c%c%c" % (pickle.EXT2, code & 0xff, code >> 8)) else: write(pickle.EXT4 + pack("<i", code)) return write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)
def save_global(self, obj, name=None, pack=struct.pack): write = self.write memo = self.memo if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: modname = pickle.whichmodule(obj, name) try: __import__(modname) themodule = sys.modules[modname] except (ImportError, KeyError, AttributeError): #should never occur raise pickle.PicklingError( "Can't pickle %r: Module %s cannot be found" % (obj, modname)) if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) sendRef = True typ = type(obj) #print 'saving', obj, typ try: try: #Deal with case when getattribute fails with exceptions klass = getattr(themodule, name) except (AttributeError): if modname == '__builtin__': #new.* are misrepeported modname = 'new' __import__(modname) themodule = sys.modules[modname] try: klass = getattr(themodule, name) except AttributeError, a: #print themodule, name, obj, type(obj) raise pickle.PicklingError("Can't pickle builtin %s" % obj) else: raise except (ImportError, KeyError, AttributeError): if typ == types.TypeType or typ == types.ClassType: sendRef = False else: #we can't deal with this raise else: if klass is not obj and (typ == types.TypeType or typ == types.ClassType): sendRef = False if not sendRef: self.save_class_obj(obj, name, pack) return if self.proto >= 2: code = _extension_registry.get((modname, name)) if code: assert code > 0 if code <= 0xff: write(pickle.EXT1 + chr(code)) elif code <= 0xffff: write("%c%c%c" % (pickle.EXT2, code & 0xff, code >> 8)) else: write(pickle.EXT4 + pack("<i", code)) return write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj)