def save_global(self, obj, name=None):
assert obj
assert id(obj) not in self.memo
if name is None:
name = obj.__name__
module = getattr(obj, "__module__", None)
if module is None or module == "__main__":
module = pickle.whichmodule(obj, name)
if module is None or module == "__main__":
raise pickle.PicklingError(
"Can't pickle %r: module not found: %s" % (obj, module))
try:
__import__(module)
mod = sys.modules[module]
klass = getattr(mod, name)
except (ImportError, KeyError, AttributeError):
raise pickle.PicklingError(
"Can't pickle %r: it's not found as %s.%s" % (obj, module, name))
else:
if klass is not obj:
raise pickle.PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" % (obj, module, name))
assert "\n" not in module
assert "\n" not in name
self.write(pickle.GLOBAL + bytes(module + '\n' + name + '\n', "utf8"))
self.memoize(obj)
def reduce_function(obj):
name = obj.__name__
if not name or name == "<lambda>":
return load_closure, (dump_closure(obj),)
module = getattr(obj, "__module__", None)
if module is None:
module = whichmodule(obj, name)
if module == "__main__" and name not in (
"load_closure",
"load_module",
"load_method",
"load_local_class",
): # fix for test
return load_closure, (dump_closure(obj),)
try:
f = get_global_function(module, name)
except (ImportError, KeyError, AttributeError):
return load_closure, (dump_closure(obj),)
else:
if f is not obj:
return load_closure, (dump_closure(obj),)
return name
def save_global(coder, obj, name=None):
if name is None:
name = obj.__name__
module = getattr(obj, "__module__", None)
if module is None:
module = whichmodule(obj, name)
try:
__import__ (module)
mod = sys.modules[module]
klass= getattr(mod, name)
except (ImportError, KeyError, AttributeError):
raise PicklingError(
"Can't pickle %r: it's not found as %s.%s" %
(obj, module, name))
else:
if klass is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module, name))
code = copy_reg._extension_registry.get((module, name))
if code:
coder.encodeInt_forKey_(kOP_GLOBAL_EXT, kKIND)
coder.encodeInt_forKey_(code, kCODE)
else:
coder.encodeInt_forKey_(kOP_GLOBAL, kKIND)
coder.encodeObject_forKey_(unicode(module), kMODULE)
coder.encodeObject_forKey_(unicode(name), kNAME)
def save_function(self, obj, name=None):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
write = self.write
if name is None:
name = obj.__name__
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = None
# print('which gives %s %s %s' % (modname, obj, name))
try:
themodule = sys.modules[modname]
except KeyError:
# eval'd items such as namedtuple give invalid items for their
# function __module__
modname = '__main__'
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
if getattr(themodule, name, None) is obj:
return self.save_global(obj, name)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if islambda(obj) or obj.__code__.co_filename == '<stdin>' or \
themodule is None:
# print("save global", islambda(obj), obj.__code__.co_filename,
# modname, themodule)
self.save_function_tuple(obj)
return
else:
# func is nested
klass = getattr(themodule, name, None)
if klass is None or klass is not obj:
self.save_function_tuple(obj)
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def save_global(self, obj, name=None, pack=struct.pack):
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
self.save(_load_class)
self.save_reduce(typ, (obj.__name__, obj.__bases__, {"__doc__": obj.__doc__}), obj=obj)
d.pop('__doc__', None)
# handle property and staticmethod
dd = {}
for k, v in d.items():
if isinstance(v, property):
k = ('property', k)
v = (v.fget, v.fset, v.fdel, v.__doc__)
elif isinstance(v, staticmethod) and hasattr(v, '__func__'):
k = ('staticmethod', k)
v = v.__func__
elif isinstance(v, classmethod) and hasattr(v, '__func__'):
k = ('classmethod', k)
v = v.__func__
dd[k] = v
self.save(dd)
self.write(pickle.TUPLE2)
self.write(pickle.REDUCE)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_function(self, obj, name=None, pack=struct.pack):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
write = self.write
name = obj.__name__
modname = pickle.whichmodule(obj, name)
#print 'which gives %s %s %s' % (modname, obj, name)
try:
themodule = sys.modules[modname]
except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__
modname = '__main__'
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
if getattr(themodule, name, None) is obj:
return self.save_global(obj, name)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if islambda(obj) or obj.func_code.co_filename == '<stdin>' or themodule is None:
#Force server to import modules that have been imported in main
modList = None
if themodule is None and not self.savedForceImports:
mainmod = sys.modules['__main__']
if useForcedImports and hasattr(mainmod,'___pyc_forcedImports__'):
modList = list(mainmod.___pyc_forcedImports__)
self.savedForceImports = True
self.save_function_tuple(obj, modList)
return
else: # func is nested
klass = getattr(themodule, name, None)
if klass is None or klass is not obj:
self.save_function_tuple(obj, [themodule])
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def _jelly_function(self, func):
''' (internal) Jelly a function.
Returns a list of the form (function "name" (module "name"))
'''
name = func.__name__
module = sys.modules[pickle.whichmodule(func, name)]
if self.taster.isModuleAllowed(module.__name__):
jfunc = self._prepare(func)
jfunc.append(function_atom)
jfunc.append(name)
jfunc.append(self.jelly(module))
return self._preserve(func, jfunc)
else:
return self.unpersistable("module %s deemed insecure" % str(module.__name__))
def jelly(self, obj):
if isinstance(obj, Jellyable):
preRef = self._checkMutable(obj)
if preRef:
return preRef
return obj.jellyFor(self)
objType = type(obj)
if self.taster.isTypeAllowed(qual(objType)):
# "Immutable" Types
if ((objType is StringType) or
(objType is IntType) or
(objType is LongType) or
(objType is FloatType)):
return obj
elif objType is MethodType:
return ["method",
obj.im_func.__name__,
self.jelly(obj.im_self),
self.jelly(obj.im_class)]
elif UnicodeType and objType is UnicodeType:
return ['unicode', obj.encode('UTF-8')]
elif objType is NoneType:
return ['None']
elif objType is FunctionType:
name = obj.__name__
return ['function', str(pickle.whichmodule(obj, obj.__name__))
+ '.' +
name]
elif objType is ModuleType:
return ['module', obj.__name__]
elif objType is BooleanType:
return ['boolean', obj and 'true' or 'false']
elif objType is datetime.datetime:
if obj.tzinfo:
raise NotImplementedError(
"Currently can't jelly datetime objects with tzinfo")
return ['datetime', '%s %s %s %s %s %s %s' % (
obj.year, obj.month, obj.day, obj.hour,
obj.minute, obj.second, obj.microsecond)]
elif objType is datetime.time:
if obj.tzinfo:
raise NotImplementedError(
"Currently can't jelly datetime objects with tzinfo")
return ['time', '%s %s %s %s' % (obj.hour, obj.minute,
obj.second, obj.microsecond)]
elif objType is datetime.date:
return ['date', '%s %s %s' % (obj.year, obj.month, obj.day)]
elif objType is datetime.timedelta:
r
def GetServiceClassString(cls, argv = None):
if argv is None:
argv = sys.argv
import pickle
modName = pickle.whichmodule(cls, cls.__name__)
if modName == '__main__':
try:
fname = win32api.GetFullPathName(argv[0])
path = os.path.split(fname)[0]
# Eaaaahhhh - sometimes this will be a short filename, which causes
# problems with 1.5.1 and the silly filename case rule.
# Get the long name
fname = os.path.join(path, win32api.FindFiles(fname)[0][8])
except win32api.error:
raise error("Could not resolve the path name '%s' to a full path" % (argv[0]))
modName = os.path.splitext(fname)[0]
return modName + "." + cls.__name__
def GetServiceClassString(cls, argv=None):
if argv is None:
argv = sys.argv
import pickle
modName = pickle.whichmodule(cls, cls.__name__)
if modName == '__main__':
try:
fname = win32api.GetFullPathName(argv[0])
path = os.path.split(fname)[0]
# Eaaaahhhh - sometimes this will be a short filename, which causes
# problems with 1.5.1 and the silly filename case rule.
# Get the long name
fname = os.path.join(path, win32api.FindFiles(fname)[0][8])
except win32api.error:
raise error("Could not resolve the path name '%s' to a full path" %
(argv[0]))
modName = os.path.splitext(fname)[0]
return modName + "." + cls.__name__
def save_global(coder, obj, name=None):
if name is None:
name = obj.__name__
module = getattr(obj, "__module__", None)
if module is None:
module = whichmodule(obj, name)
try:
__import__(module)
mod = sys.modules[module]
klass = getattr(mod, name)
except (ImportError, KeyError, AttributeError):
raise PicklingError("Can't pickle %r: it's not found as %s.%s" %
(obj, module, name))
else:
if klass is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module, name))
code = copy_reg._extension_registry.get((module, name))
if coder.allowsKeyedCoding():
if code:
coder.encodeInt_forKey_(kOP_GLOBAL_EXT, kKIND)
coder.encodeInt_forKey_(code, kCODE)
else:
coder.encodeInt_forKey_(kOP_GLOBAL, kKIND)
coder.encodeObject_forKey_(unicode(module), kMODULE)
coder.encodeObject_forKey_(unicode(name), kNAME)
else:
if code:
coder.__pyobjc__encodeInt_(kOP_GLOBAL_EXT)
coder.__pyobjc__encodeInt_(code)
else:
coder.__pyobjc__encodeInt_(kOP_GLOBAL)
coder.encodeObject_(unicode(module))
coder.encodeObject_(unicode(name))
def globalname( self, client): """Try to find the name for the client""" name = getattr( client, '__name__', None ) if not name: raise ValueError( """Attempt to put %r in the root cellar, has no __name__ attribute, so can't create a global reference to it"""%( client,)) module = getattr( client, '__module__', None ) if not module: module = whichmodule( client, name ) # sanity check... try: __import__(module) mod = sys.modules[module] klass = getattr(mod, name) except (ImportError, KeyError, AttributeError), error: raise RootCellarError( """Can't store %r: can't import %s.%s: %r"""%( client, module, name, error ) )
def _class_string(cls):
"""Return a string we can use later to find the base class of cls.
Raise TypeError if we can't compute a useful string.
"""
log.debug("_class_string: %r", cls)
try:
return _class_string_cache[cls]
except KeyError:
pass
# Most of this is borrowed from pickle.py
name = getattr(cls, '__qualname__', None)
if name is None:
name = cls.__name__
module_name = whichmodule(cls, name)
try:
__import__(module_name, level=0)
module = sys.modules[module_name]
obj2 = module
for subpath in name.split('.'):
if subpath == '<locals>':
raise AttributeError(
"<locals> in class name {}".format(name))
try:
parent = obj2
obj2 = getattr(obj2, subpath)
except AttributeError:
raise AttributeError(
"Can't get attribute {!r} in {!r}".format(name, obj2))
except (ImportError, KeyError, AttributeError):
raise TypeError("Can't persist {!r} instance, class not found"
" as {}.{}".format(cls, module_name, name))
else:
if obj2 is not cls:
raise TypeError("Can't persist {!r} instance, class is"
" not the same object as {}.{}".format(
cls, module_name, name))
if module_name == '__builtin__':
module_name = 'builtins'
res = _class_string_cache[cls] = "{}:{}".format(module_name, name)
log.debug("new _class_string: %r", res)
return res
def reduce_function(obj):
name = obj.__name__
if not name or name == '<lambda>':
return load_closure, (dump_closure(obj),)
module = getattr(obj, "__module__", None)
if module is None:
module = whichmodule(obj, name)
if module == '__main__' and name != 'load_closure': # fix for test
return load_closure, (dump_closure(obj),)
try:
f = get_global_function(module, name)
except (ImportError, KeyError, AttributeError):
return load_closure, (dump_closure(obj),)
else:
if f is not obj:
return load_closure, (dump_closure(obj),)
return name
def reduce_function(obj):
name = obj.__name__
if not name or name == '<lambda>':
return load_closure, (dump_closure(obj), )
module = getattr(obj, "__module__", None)
if module is None:
module = whichmodule(obj, name)
if module == '__main__' and name != 'load_closure': # fix for test
return load_closure, (dump_closure(obj), )
try:
f = get_global_function(module, name)
except (ImportError, KeyError, AttributeError):
return load_closure, (dump_closure(obj), )
else:
if f is not obj:
return load_closure, (dump_closure(obj), )
return name
def _class_string(cls):
"""Return a string we can use later to find the base class of cls.
Raise TypeError if we can't compute a useful string.
"""
log.debug("_class_string: %r", cls)
try:
return _class_string_cache[cls]
except KeyError:
pass
# Most of this is borrowed from pickle.py
name = getattr(cls, '__qualname__', None)
if name is None:
name = cls.__name__
module_name = whichmodule(cls, name)
try:
__import__(module_name, level=0)
module = sys.modules[module_name]
obj2 = module
for subpath in name.split('.'):
if subpath == '<locals>':
raise AttributeError("<locals> in class name {}".format(name))
try:
parent = obj2
obj2 = getattr(obj2, subpath)
except AttributeError:
raise AttributeError("Can't get attribute {!r} in {!r}".format(
name, obj2))
except (ImportError, KeyError, AttributeError):
raise TypeError("Can't persist {!r} instance, class not found"
" as {}.{}".format(cls, module_name, name))
else:
if obj2 is not cls:
raise TypeError("Can't persist {!r} instance, class is"
" not the same object as {}.{}".format(
cls, module_name, name))
if module_name == '__builtin__':
module_name = 'builtins'
res = _class_string_cache[cls] = "{}:{}".format(module_name, name)
log.debug("new _class_string: %r", res)
return res
def GetServiceClassString(cls, argv = None):
if argv is None:
argv = sys.argv
import pickle
modName = pickle.whichmodule(cls, cls.__name__)
if modName == '__main__':
try:
fname = win32api.GetFullPathName(argv[0])
path = os.path.split(fname)[0]
# Eaaaahhhh - sometimes this will be a short filename, which causes
# problems with 1.5.1 and the silly filename case rule.
filelist = win32api.FindFiles(fname)
# win32api.FindFiles will not detect files in a zip or exe. If list is empty,
# skip the test and hope the file really exists.
if len(filelist) != 0:
# Get the long name
fname = os.path.join(path, filelist[0][8])
except win32api.error:
raise error("Could not resolve the path name '%s' to a full path" % (argv[0]))
modName = os.path.splitext(fname)[0]
return modName + "." + cls.__name__
def save_global(self, obj, name=None, pack=struct.pack):
"""
Save a "global".
The name of this method is somewhat misleading: all types get
dispatched here.
"""
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type,
(_BUILTIN_TYPE_NAMES[obj], ),
obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = '__main__'
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, _class_type)):
self.save_dynamic_class(obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack):
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type,
(_BUILTIN_TYPE_NAMES[obj], ),
obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def _pickle_global(obj):
name = obj.__name__
module = getattr(obj, "__module__", None)
if module is None:
module = pickle.whichmodule(obj, name)
try:
__import__(module)
mod = sys.modules[module]
klass = getattr(mod, name)
except (ImportError, KeyError, AttributeError):
raise pickle.PicklingError(
"Can't pickle %r: it's not found as %s.%s" %
(obj, module, name))
else:
if klass is not obj:
raise pickle.PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module, name))
return (_unpickle_global, (module, name))
def save_global(self, obj, name=None, pack=struct.pack):
"""
Save a "global".
The name of this method is somewhat misleading: all types get
dispatched here.
"""
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = '__main__'
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
self.save_dynamic_class(obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global(self, obj, name=None, pack=struct.pack):
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def save_global_py2(obj, name=None, proto=2):
if name is None:
name = obj.__name__
module = getattr(obj, "__module__", None)
if module is None:
module = whichmodule(obj, name)
try:
__import__(module)
mod = sys.modules[module]
klass = getattr(mod, name)
except (ImportError, KeyError, AttributeError):
raise PicklingError(
"Can't pickle %r: it's not found as %s.%s" %
(obj, module, name))
else:
if klass is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module, name))
if proto >= 2:
code = _extension_registry.get((module, name))
if code:
# assert code > 0
# if code <= 0xff:
# write(EXT1 + chr(code))
# elif code <= 0xffff:
# write("%c%c%c" % (EXT2, code&0xff, code>>8))
# else:
# write(EXT4 + pack("<i", code))
# return
raise H5itPicklingError(
"h5it Can't pickle %r: extension codes "
"are not supported yet." % obj)
return GlobalTuple(module, name)
def _ModuleElementType(self, obj, obj_type, obj_id, name=None):
self._seen.add(obj_id)
if name is None:
name = obj.__name__
modulename = getattr(obj, "__module__", None)
if modulename is None:
modulename = pickle.whichmodule(obj, name)
try:
__import__(modulename)
module = sys.modules[modulename]
same_as_obj = getattr(module, name)
except (ImportError, KeyError, AttributeError):
raise pickle.PicklingError(
"Can't pickle %r: it's not found as %s.%s" %
(obj, modulename, name))
else:
if same_as_obj is not obj:
raise pickle.PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, modulename, name))
code = copy_reg._extension_registry.get((modulename, name))
if code:
assert code > 0
if code <= 0xFF:
size = 2
elif code <= 0xFFFF:
size = 3
else:
size = 5
else:
size = 3 + len(modulename) + len(name)
return size
def _save_global(self, path, obj, name=None, pack=struct.pack):
if name is None:
name = obj.__name__
module = getattr(obj, "__module__", None)
if module is None:
module = whichmodule(obj, name)
try:
__import__(module)
mod = sys.modules[module]
klass = getattr(mod, name)
except (ImportError, KeyError, AttributeError):
raise PicklingError(
"Can't pickle %r: it's not found as %s.%s" %
(obj, module, name))
else:
if klass is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module, name))
pickletype = None
code = _extension_registry.get((module, name))
if code:
assert code > 0
pickletype = EXT4
stuff = pack("<i", code)
if not pickletype:
stuff = module + '\n' + name
pickletype = GLOBAL
array = self.file.save_array(path, str(stuff))
self.file.set_attr(array, 'pickletype', pickletype)
def _ufunc_reduce(func):
from pickle import whichmodule
name = func.__name__
return _ufunc_reconstruct, (whichmodule(func, name), name)
def save_function(self, obj, name=None, pack=struct.pack):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
write = self.write
name = obj.__name__
modname = pickle.whichmodule(obj, name)
#print 'which gives %s %s %s' % (modname, obj, name)
try:
themodule = sys.modules[modname]
except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__
modname = '__main__'
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
if self.os_env_vars:
# recursion hackery
os_env_vars = self.os_env_vars
self.os_env_vars = []
old_saved_django = self.savedDjangoEnv
self.savedDjangoEnv = True
write(pickle.MARK)
if isinstance(os_env_vars, dict):
os_mapping = os_env_vars
else:
os_mapping = {}
for env_var in os_env_vars:
if env_var in os.environ:
os_mapping[env_var] = os.environ[env_var]
self.save_reduce(env_vars_load, (os_mapping,)) # note: nothing sensible to memoize
write(pickle.POP_MARK)
self.savedDjangoEnv = old_saved_django
if modname == 'cloud.shell':
# don't save django environment if we are using shell.exec
self.savedDjangoEnv = True
elif not self.savedDjangoEnv:
# hack for django - if we detect the settings module, we transport it
# unfortunately this module is dynamically, not statically, resolved, so
# dependency analysis never detects it
django_settings = os.environ.get('DJANGO_SETTINGS_MODULE', '')
if django_settings:
django_mod = sys.modules.get(django_settings)
if django_mod:
cloudLog.debug('Transporting django settings %s during save of %s', django_mod, name)
self.savedDjangoEnv = True
self.modules.add(django_mod)
write(pickle.MARK)
self.save_reduce(django_settings_load, (django_mod.__name__,), obj=django_mod)
write(pickle.POP_MARK)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if islambda(obj) or obj.__code__.co_filename == '<stdin>' or themodule == None:
#Force server to import modules that have been imported in main
modList = None
if themodule == None and not self.savedForceImports:
mainmod = sys.modules['__main__']
if useForcedImports and hasattr(mainmod,'___pyc_forcedImports__'):
modList = list(mainmod.___pyc_forcedImports__)
self.savedForceImports = True
self.save_function_tuple(obj, modList)
return
else: # func is nested
klass = getattr(themodule, name, None)
if klass is None or klass is not obj:
self.save_function_tuple(obj, [themodule])
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
if six.PY3:
write(pickle.MARK + pickle.GLOBAL + (modname + '\n' + name + '\n').encode('utf-8'))
else:
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
if six.PY3:
write(pickle.GLOBAL + (modname + '\n' + name + '\n').encode('utf-8'))
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def save_global(self, obj, name=None, pack=struct.pack):
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type,
(_BUILTIN_TYPE_NAMES[obj], ),
obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
self.save(_load_class)
self.save_reduce(typ, (obj.__name__, obj.__bases__, {
"__doc__": obj.__doc__
}),
obj=obj)
d.pop('__doc__', None)
# handle property and staticmethod
dd = {}
for k, v in d.items():
if isinstance(v, property):
k = ('property', k)
v = (v.fget, v.fset, v.fdel, v.__doc__)
elif isinstance(v, staticmethod) and hasattr(v, '__func__'):
k = ('staticmethod', k)
v = v.__func__
elif isinstance(v, classmethod) and hasattr(v, '__func__'):
k = ('classmethod', k)
v = v.__func__
dd[k] = v
self.save(dd)
self.write(pickle.TUPLE2)
self.write(pickle.REDUCE)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def get_memmapping_reducers(
pool_id, forward_reducers=None, backward_reducers=None,
temp_folder=None, max_nbytes=1e6, mmap_mode='r', verbose=0,
prewarm=False, **kwargs):
"""Construct a pair of memmapping reducer linked to a tmpdir.
This function manage the creation and the clean up of the temporary folders
underlying the memory maps and should be use to get the reducers necessary
to construct joblib pool or executor.
"""
if forward_reducers is None:
forward_reducers = dict()
if backward_reducers is None:
backward_reducers = dict()
# Prepare a sub-folder name for the serialization of this particular
# pool instance (do not create in advance to spare FS write access if
# no array is to be dumped):
pool_folder_name = "joblib_memmapping_folder_{}_{}".format(
os.getpid(), pool_id)
pool_folder, use_shared_mem = _get_temp_dir(pool_folder_name,
temp_folder)
# Register the garbage collector at program exit in case caller forgets
# to call terminate explicitly: note we do not pass any reference to
# self to ensure that this callback won't prevent garbage collection of
# the pool instance and related file handler resources such as POSIX
# semaphores and pipes
pool_module_name = whichmodule(delete_folder, 'delete_folder')
def _cleanup():
# In some cases the Python runtime seems to set delete_folder to
# None just before exiting when accessing the delete_folder
# function from the closure namespace. So instead we reimport
# the delete_folder function explicitly.
# https://github.com/joblib/joblib/issues/328
# We cannot just use from 'joblib.pool import delete_folder'
# because joblib should only use relative imports to allow
# easy vendoring.
delete_folder = __import__(
pool_module_name, fromlist=['delete_folder']).delete_folder
try:
delete_folder(pool_folder)
except WindowsError:
warnings.warn("Failed to clean temporary folder: {}"
.format(pool_folder))
atexit.register(_cleanup)
if np is not None:
# Register smart numpy.ndarray reducers that detects memmap backed
# arrays and that is also able to dump to memmap large in-memory
# arrays over the max_nbytes threshold
if prewarm == "auto":
prewarm = not use_shared_mem
forward_reduce_ndarray = ArrayMemmapReducer(
max_nbytes, pool_folder, mmap_mode, verbose,
prewarm=prewarm)
forward_reducers[np.ndarray] = forward_reduce_ndarray
forward_reducers[np.memmap] = reduce_memmap
# Communication from child process to the parent process always
# pickles in-memory numpy.ndarray without dumping them as memmap
# to avoid confusing the caller and make it tricky to collect the
# temporary folder
backward_reduce_ndarray = ArrayMemmapReducer(
None, pool_folder, mmap_mode, verbose)
backward_reducers[np.ndarray] = backward_reduce_ndarray
backward_reducers[np.memmap] = reduce_memmap
return forward_reducers, backward_reducers, pool_folder
def save_global(self, obj, name=None):
# unfortunately the pickler code is factored in a way that
# forces us to copy/paste this function. The only change is marked
# CHANGED below.
write = self.write
memo = self.memo
if name is None:
name = getattr(obj, '__qualname__', None)
if name is None:
name = obj.__name__
orig_module_name = whichmodule(obj, name)
# CHANGED: demangle the module name before importing. If this obj came
# out of a PackageImporter, `__module__` will be mangled. See
# mangling.md for details.
module_name = demangle(orig_module_name)
try:
# CHANGED: self.import_module rather than
# __import__
module = self.import_module(module_name)
obj2, parent = _getattribute(module, name)
except (ImportError, KeyError, AttributeError):
raise PicklingError("Can't pickle %r: it's not found as %s.%s" %
(obj, module_name, name)) from None
else:
if obj2 is not obj:
# CHANGED: More specific error message in the case of mangling.
obj_module_name = getattr(obj, "__module__", orig_module_name)
obj2_module_name = getattr(obj2, "__module__",
orig_module_name)
msg = f"Can't pickle {obj}: it's not the same object as {obj2_module_name}.{name}."
is_obj_mangled = is_mangled(obj_module_name)
is_obj2_mangled = is_mangled(obj2_module_name)
if is_obj_mangled or is_obj2_mangled:
obj_location = (get_mangle_prefix(obj_module_name)
if is_obj_mangled else
"the current Python environment")
obj2_location = (get_mangle_prefix(obj2_module_name)
if is_obj2_mangled else
"the current Python environment")
obj_importer_name = (
f"the importer for {get_mangle_prefix(obj_module_name)}"
if is_obj_mangled else "'importlib.import_module'")
obj2_importer_name = (
f"the importer for {get_mangle_prefix(obj2_module_name)}"
if is_obj2_mangled else "'importlib.import_module'")
msg += (
f"\n\nThe object being pickled is from '{orig_module_name}', "
f"which is coming from {obj_location}."
f"\nHowever, when we import '{orig_module_name}', it's coming from {obj2_location}."
"\nTo fix this, make sure 'PackageExporter.importers' lists "
f"{obj_importer_name} before {obj2_importer_name}")
raise PicklingError(msg)
if self.proto >= 2:
code = _extension_registry.get((module_name, name))
if code:
assert code > 0
if code <= 0xff:
write(EXT1 + pack("<B", code))
elif code <= 0xffff:
write(EXT2 + pack("<H", code))
else:
write(EXT4 + pack("<i", code))
return
lastname = name.rpartition('.')[2]
if parent is module:
name = lastname
# Non-ASCII identifiers are supported only with protocols >= 3.
if self.proto >= 4:
self.save(module_name)
self.save(name)
write(STACK_GLOBAL)
elif parent is not module:
self.save_reduce(getattr, (parent, lastname))
elif self.proto >= 3:
write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
bytes(name, "utf-8") + b'\n')
else:
if self.fix_imports:
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
if (module_name, name) in r_name_mapping:
module_name, name = r_name_mapping[(module_name, name)]
elif module_name in r_import_mapping:
module_name = r_import_mapping[module_name]
try:
write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
bytes(name, "ascii") + b'\n')
except UnicodeEncodeError:
raise PicklingError(
"can't pickle global identifier '%s.%s' using "
"pickle protocol %i" %
(module, name, self.proto)) from None
self.memoize(obj)
def __init__(self, processes=None, temp_folder=None, max_nbytes=1e6,
mmap_mode='r', forward_reducers=None, backward_reducers=None,
verbose=0, context_id=None, prewarm=False, **kwargs):
if forward_reducers is None:
forward_reducers = dict()
if backward_reducers is None:
backward_reducers = dict()
if context_id is not None:
warnings.warn('context_id is deprecated and ignored in joblib'
' 0.9.4 and will be removed in 0.11',
DeprecationWarning)
# Prepare a sub-folder name for the serialization of this particular
# pool instance (do not create in advance to spare FS write access if
# no array is to be dumped):
use_shared_mem = False
pool_folder_name = "joblib_memmaping_pool_%d_%d" % (
os.getpid(), id(self))
if temp_folder is None:
temp_folder = os.environ.get('JOBLIB_TEMP_FOLDER', None)
if temp_folder is None:
if os.path.exists(SYSTEM_SHARED_MEM_FS):
try:
temp_folder = SYSTEM_SHARED_MEM_FS
pool_folder = os.path.join(temp_folder, pool_folder_name)
if not os.path.exists(pool_folder):
os.makedirs(pool_folder)
use_shared_mem = True
except IOError:
# Missing rights in the the /dev/shm partition,
# fallback to regular temp folder.
temp_folder = None
if temp_folder is None:
# Fallback to the default tmp folder, typically /tmp
temp_folder = tempfile.gettempdir()
temp_folder = os.path.abspath(os.path.expanduser(temp_folder))
pool_folder = os.path.join(temp_folder, pool_folder_name)
self._temp_folder = pool_folder
# Register the garbage collector at program exit in case caller forgets
# to call terminate explicitly: note we do not pass any reference to
# self to ensure that this callback won't prevent garbage collection of
# the pool instance and related file handler resources such as POSIX
# semaphores and pipes
pool_module_name = whichmodule(delete_folder, 'delete_folder')
def _cleanup():
# In some cases the Python runtime seems to set delete_folder to
# None just before exiting when accessing the delete_folder
# function from the closure namespace. So instead we reimport
# the delete_folder function explicitly.
# https://github.com/joblib/joblib/issues/328
# We cannot just use from 'joblib.pool import delete_folder'
# because joblib should only use relative imports to allow
# easy vendoring.
delete_folder = __import__(
pool_module_name, fromlist=['delete_folder']).delete_folder
delete_folder(pool_folder)
atexit.register(_cleanup)
if np is not None:
# Register smart numpy.ndarray reducers that detects memmap backed
# arrays and that is alse able to dump to memmap large in-memory
# arrays over the max_nbytes threshold
if prewarm == "auto":
prewarm = not use_shared_mem
forward_reduce_ndarray = ArrayMemmapReducer(
max_nbytes, pool_folder, mmap_mode, verbose,
prewarm=prewarm)
forward_reducers[np.ndarray] = forward_reduce_ndarray
forward_reducers[np.memmap] = reduce_memmap
# Communication from child process to the parent process always
# pickles in-memory numpy.ndarray without dumping them as memmap
# to avoid confusing the caller and make it tricky to collect the
# temporary folder
backward_reduce_ndarray = ArrayMemmapReducer(
None, pool_folder, mmap_mode, verbose)
backward_reducers[np.ndarray] = backward_reduce_ndarray
backward_reducers[np.memmap] = reduce_memmap
poolargs = dict(
processes=processes,
forward_reducers=forward_reducers,
backward_reducers=backward_reducers)
poolargs.update(kwargs)
super(MemmapingPool, self).__init__(**poolargs)
def save_function(self, obj, name=None):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
write = self.write
if name is None:
name = obj.__name__
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = None
# print('which gives %s %s %s' % (modname, obj, name))
try:
themodule = sys.modules[modname]
except KeyError:
# eval'd items such as namedtuple give invalid items for their function __module__
modname = '__main__'
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
if getattr(themodule, name, None) is obj:
return self.save_global(obj, name)
# a builtin_function_or_method which comes in as an attribute of some
# object (e.g., object.__new__, itertools.chain.from_iterable) will end
# up with modname "__main__" and so end up here. But these functions
# have no __code__ attribute in CPython, so the handling for
# user-defined functions below will fail.
# So we pickle them here using save_reduce; have to do it differently
# for different python versions.
if not hasattr(obj, '__code__'):
if PY3:
if sys.version_info < (3, 4):
raise pickle.PicklingError("Can't pickle %r" % obj)
else:
rv = obj.__reduce_ex__(self.proto)
else:
if hasattr(obj, '__self__'):
rv = (getattr, (obj.__self__, name))
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
return Pickler.save_reduce(self, obj=obj, *rv)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if (islambda(obj)
or getattr(obj.__code__, 'co_filename', None) == '<stdin>'
or themodule is None):
self.save_function_tuple(obj)
return
else:
# func is nested
klass = getattr(themodule, name, None)
if klass is None or klass is not obj:
self.save_function_tuple(obj)
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def jelly(self, obj):
if isinstance(obj, Jellyable):
preRef = self._checkMutable(obj)
if preRef:
return preRef
return obj.jellyFor(self)
objType = type(obj)
if self.taster.isTypeAllowed(qual(objType)):
# "Immutable" Types
if ((objType is StringType) or
(objType is IntType) or
(objType is LongType) or
(objType is FloatType)):
return obj
elif objType is MethodType:
return ["method",
obj.im_func.__name__,
self.jelly(obj.im_self),
self.jelly(obj.im_class)]
elif UnicodeType and objType is UnicodeType:
return ['unicode', obj.encode('UTF-8')]
elif objType is NoneType:
return ['None']
elif objType is FunctionType:
name = obj.__name__
return ['function', str(pickle.whichmodule(obj, obj.__name__))
+ '.' +
name]
elif objType is ModuleType:
return ['module', obj.__name__]
elif objType is BooleanType:
return ['boolean', obj and 'true' or 'false']
elif objType is datetime.datetime:
if obj.tzinfo:
raise NotImplementedError(
"Currently can't jelly datetime objects with tzinfo")
return ['datetime', '%s %s %s %s %s %s %s' % (
obj.year, obj.month, obj.day, obj.hour,
obj.minute, obj.second, obj.microsecond)]
elif objType is datetime.time:
if obj.tzinfo:
raise NotImplementedError(
"Currently can't jelly datetime objects with tzinfo")
return ['time', '%s %s %s %s' % (obj.hour, obj.minute,
obj.second, obj.microsecond)]
elif objType is datetime.date:
return ['date', '%s %s %s' % (obj.year, obj.month, obj.day)]
elif objType is datetime.timedelta:
return ['timedelta', '%s %s %s' % (obj.days, obj.seconds,
obj.microseconds)]
elif objType is ClassType or issubclass(objType, type):
return ['class', qual(obj)]
elif decimal is not None and objType is decimal.Decimal:
return self.jelly_decimal(obj)
else:
preRef = self._checkMutable(obj)
if preRef:
return preRef
# "Mutable" Types
sxp = self.prepare(obj)
if objType is ListType:
sxp.extend(self._jellyIterable(list_atom, obj))
elif objType is TupleType:
sxp.extend(self._jellyIterable(tuple_atom, obj))
elif objType in DictTypes:
sxp.append(dictionary_atom)
for key, val in obj.items():
sxp.append([self.jelly(key), self.jelly(val)])
elif (_set is not None and objType is set or
objType is _sets.Set):
sxp.extend(self._jellyIterable(set_atom, obj))
elif (_set is not None and objType is frozenset or
objType is _sets.ImmutableSet):
sxp.extend(self._jellyIterable(frozenset_atom, obj))
else:
className = qual(obj.__class__)
persistent = None
if self.persistentStore:
persistent = self.persistentStore(obj, self)
if persistent is not None:
sxp.append(persistent_atom)
sxp.append(persistent)
elif self.taster.isClassAllowed(obj.__class__):
sxp.append(className)
if hasattr(obj, "__getstate__"):
state = obj.__getstate__()
else:
state = obj.__dict__
sxp.append(self.jelly(state))
else:
self.unpersistable(
"instance of class %s deemed insecure" %
qual(obj.__class__), sxp)
return self.preserve(obj, sxp)
else:
if objType is InstanceType:
raise InsecureJelly("Class not allowed for instance: %s %s" %
(obj.__class__, obj))
raise InsecureJelly("Type not allowed for object: %s %s" %
(objType, obj))
def sliceBody(self, streamable, banana):
name = self.obj.__name__
fullname = str(whichmodule(self.obj, self.obj.__name__)) + '.' + name
yield fullname
def save_global(self, obj, name=None, pack=struct.pack): # pylint: disable=too-many-branches
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type,
(_BUILTIN_TYPE_NAMES[obj], ),
obj=obj)
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = '__main__'
if modname == '__main__':
themodule = None
else:
__import__(modname)
themodule = sys.modules[modname]
self.modules.add(themodule)
if hasattr(themodule, name) and getattr(themodule, name) is obj:
return Pickler.save_global(self, obj, name)
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
d = dict(obj.__dict__) # copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property):
# don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
# workaround for namedtuple (hijacked by PySpark)
if getattr(obj, '_is_namedtuple_', False):
self.save_reduce(_load_namedtuple, (obj.__name__, obj._fields))
return
self.save(_load_class)
self.save_reduce(typ, (obj.__name__, obj.__bases__, {
"__doc__": obj.__doc__
}),
obj=obj)
d.pop('__doc__', None)
# handle property and staticmethod
dd = {}
for k, v in d.items():
if isinstance(v, property):
k = ('property', k)
v = (v.fget, v.fset, v.fdel, v.__doc__)
elif isinstance(v, staticmethod) and hasattr(v, '__func__'):
k = ('staticmethod', k)
v = v.__func__
elif isinstance(v, classmethod) and hasattr(v, '__func__'):
k = ('classmethod', k)
v = v.__func__
dd[k] = v
self.save(dd)
self.write(pickle.TUPLE2)
self.write(pickle.REDUCE)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
def get_class_path(cls):
module_name = pickle.whichmodule(cls, cls.__name__)
return module_name + "." + cls.__name__
def save_function(self, obj, name=None, pack=struct.pack):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
write = self.write
name = obj.__name__
modname = pickle.whichmodule(obj, name)
#print 'which gives %s %s %s' % (modname, obj, name)
try:
themodule = sys.modules[modname]
except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__
modname = '__main__'
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
if getattr(themodule, name, None) is obj:
return self.save_global(obj, name)
if not self.savedDjangoEnv:
#hack for django - if we detect the settings module, we transport it
django_settings = os.environ.get('DJANGO_SETTINGS_MODULE', '')
if django_settings:
django_mod = sys.modules.get(django_settings)
if django_mod:
cloudLog.debug(
'Transporting django settings %s during save of %s',
django_mod, name)
self.savedDjangoEnv = True
self.modules.add(django_mod)
write(pickle.MARK)
self.save_reduce(django_settings_load,
(django_mod.__name__, ),
obj=django_mod)
write(pickle.POP_MARK)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if islambda(
obj
) or obj.func_code.co_filename == '<stdin>' or themodule is None:
#Force server to import modules that have been imported in main
modList = None
if themodule is None and not self.savedForceImports:
mainmod = sys.modules['__main__']
if useForcedImports and hasattr(mainmod,
'___pyc_forcedImports__'):
modList = list(mainmod.___pyc_forcedImports__)
self.savedForceImports = True
self.save_function_tuple(obj, modList)
return
else: # func is nested
klass = getattr(themodule, name, None)
if klass is None or klass is not obj:
self.save_function_tuple(obj, [themodule])
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def fullFuncName(func):
qualName = str(pickle.whichmodule(func, func.__name__)) + "." + func.__name__
if namedObject(qualName) is not func:
raise Exception("Couldn't find {} as {}.".format(func, qualName))
return qualName
def save_global(self, obj, name=None, pack=struct.pack):
write = self.write
memo = self.memo
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
try:
__import__(modname)
themodule = sys.modules[modname]
except (ImportError, KeyError, AttributeError): #should never occur
raise pickle.PicklingError(
"Can't pickle %r: Module %s cannot be found" %
(obj, modname))
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
sendRef = True
typ = type(obj)
#print 'saving', obj, typ
try:
try: #Deal with case when getattribute fails with exceptions
klass = getattr(themodule, name)
except (AttributeError):
if modname == '__builtin__': #new.* are misrepeported
modname = 'new'
__import__(modname)
themodule = sys.modules[modname]
try:
klass = getattr(themodule, name)
except AttributeError as a:
#print themodule, name, obj, type(obj)
raise pickle.PicklingError("Can't pickle builtin %s" % obj)
else:
raise
except (ImportError, KeyError, AttributeError):
if typ == type:
sendRef = False
else: #we can't deal with this
raise
else:
if klass is not obj and (typ == type):
sendRef = False
if not sendRef:
self.save_class_obj(obj, name, pack)
return
if self.proto >= 2:
code = _extension_registry.get((modname, name))
if code:
assert code > 0
if code <= 0xff:
write(pickle.EXT1 + chr(code))
elif code <= 0xffff:
write("%c%c%c" % (pickle.EXT2, code&0xff, code>>8))
else:
write(pickle.EXT4 + pack("<i", code))
return
if six.PY3:
write(pickle.GLOBAL + (modname + '\n' + name + '\n').encode('utf-8'))
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def save_function(self, obj, name=None):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
try:
should_special_case = obj in _BUILTIN_TYPE_CONSTRUCTORS
except TypeError:
# Methods of builtin types aren't hashable in python 2.
should_special_case = False
if should_special_case:
# We keep a special-cased cache of built-in type constructors at
# global scope, because these functions are structured very
# differently in different python versions and implementations (for
# example, they're instances of types.BuiltinFunctionType in
# CPython, but they're ordinary types.FunctionType instances in
# PyPy).
#
# If the function we've received is in that cache, we just
# serialize it as a lookup into the cache.
return self.save_reduce(_BUILTIN_TYPE_CONSTRUCTORS[obj], (), obj=obj)
write = self.write
if name is None:
name = obj.__name__
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = None
# print('which gives %s %s %s' % (modname, obj, name))
try:
themodule = sys.modules[modname]
except KeyError:
# eval'd items such as namedtuple give invalid items for their function __module__
modname = '__main__'
if modname == '__main__':
themodule = None
try:
lookedup_by_name = getattr(themodule, name, None)
except Exception:
lookedup_by_name = None
if themodule:
if lookedup_by_name is obj:
return self.save_global(obj, name)
# a builtin_function_or_method which comes in as an attribute of some
# object (e.g., itertools.chain.from_iterable) will end
# up with modname "__main__" and so end up here. But these functions
# have no __code__ attribute in CPython, so the handling for
# user-defined functions below will fail.
# So we pickle them here using save_reduce; have to do it differently
# for different python versions.
if not hasattr(obj, '__code__'):
if PY3: # pragma: no branch
rv = obj.__reduce_ex__(self.proto)
else:
if hasattr(obj, '__self__'):
rv = (getattr, (obj.__self__, name))
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
return self.save_reduce(obj=obj, *rv)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if (islambda(obj)
or getattr(obj.__code__, 'co_filename', None) == '<stdin>'
or themodule is None):
self.save_function_tuple(obj)
return
else:
# func is nested
if lookedup_by_name is None or lookedup_by_name is not obj:
self.save_function_tuple(obj)
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def save_global(self, obj, name=None, pack=struct.pack):
write = self.write
memo = self.memo
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
try:
__import__(modname)
themodule = sys.modules[modname]
except (ImportError, KeyError, AttributeError): #should never occur
raise pickle.PicklingError(
"Can't pickle %r: Module %s cannot be found" %
(obj, modname))
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
sendRef = True
typ = type(obj)
#print 'saving', obj, typ
try:
try: #Deal with case when getattribute fails with exceptions
klass = getattr(themodule, name)
except (AttributeError):
if modname == '__builtin__': #new.* are misrepeported
modname = 'new'
__import__(modname)
themodule = sys.modules[modname]
try:
klass = getattr(themodule, name)
except AttributeError, a:
#print themodule, name, obj, type(obj)
raise pickle.PicklingError("Can't pickle builtin %s" % obj)
else:
raise
except (ImportError, KeyError, AttributeError):
if typ == types.TypeType or typ == types.ClassType:
sendRef = False
else: #we can't deal with this
raise
else:
if klass is not obj and (typ == types.TypeType or typ == types.ClassType):
sendRef = False
if not sendRef:
#note: Third party types might crash this - add better checks!
d = dict(obj.__dict__) #copy dict proxy to a dict
if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties
d.pop('__dict__',None)
d.pop('__weakref__',None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
self.save_reduce(type(obj),(obj.__name__,obj.__bases__,
d),obj=obj)
#print 'internal reduce dask %s %s' % (obj, d)
return
if self.proto >= 2:
code = _extension_registry.get((modname, name))
if code:
assert code > 0
if code <= 0xff:
write(pickle.EXT1 + chr(code))
elif code <= 0xffff:
write("%c%c%c" % (pickle.EXT2, code&0xff, code>>8))
else:
write(pickle.EXT4 + pack("<i", code))
return
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def save_function(self, obj, name=None):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
write = self.write
if name is None:
name = getattr(obj, '__qualname__', None)
if name is None:
name = getattr(obj, '__name__', None)
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = None
# print('which gives %s %s %s' % (modname, obj, name))
try:
themodule = sys.modules[modname]
except KeyError:
# eval'd items such as namedtuple give invalid items for their function __module__
modname = '__main__'
if modname == '__main__':
themodule = None
try:
lookedup_by_name, _ = _getattribute(themodule, name)
except Exception:
lookedup_by_name = None
if themodule:
if lookedup_by_name is obj:
return self.save_global(obj, name)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if (islambda(obj)
or getattr(obj.__code__, 'co_filename', None) == '<stdin>'
or themodule is None):
self.save_function_tuple(obj)
return
else:
# func is nested
if lookedup_by_name is None or lookedup_by_name is not obj:
self.save_function_tuple(obj)
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def save_global_py3(obj, name=None, proto=2, fix_imports=True):
if name is None and proto >= 4:
name = getattr(obj, '__qualname__', None)
if name is None:
name = obj.__name__
module_name = whichmodule(obj, name, allow_qualname=proto >= 4)
try:
__import__(module_name, level=0)
module = sys.modules[module_name]
obj2 = _getattribute(module, name, allow_qualname=proto >= 4)
except (ImportError, KeyError, AttributeError):
raise PicklingError(
"Can't pickle %r: it's not found as %s.%s" %
(obj, module_name, name))
else:
if obj2 is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module_name, name))
if proto >= 2:
code = _extension_registry.get((module_name, name))
if code:
# assert code > 0
# if code <= 0xff:
# write(EXT1 + pack("<B", code))
# elif code <= 0xffff:
# write(EXT2 + pack("<H", code))
# else:
# write(EXT4 + pack("<i", code))
# return
raise H5itPicklingError("h5it Can't pickle %r: extension codes are not"
" supported yet." % obj)
# Non-ASCII identifiers are supported only with protocols >= 3.
if proto >= 4:
# self.save(module_name)
# self.save(name)
# write(STACK_GLOBAL)
raise H5itPicklingError("h5it Can't pickle %r: protocol %i is not "
"supported yet." % (obj, proto))
elif proto >= 3:
# write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
# bytes(name, "utf-8") + b'\n')
raise H5itPicklingError("h5it Can't pickle %r: protocol %i is not "
"supported yet." % (obj, proto))
else:
if fix_imports:
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
if (module_name, name) in r_name_mapping:
module_name, name = r_name_mapping[(module_name, name)]
if module_name in r_import_mapping:
module_name = r_import_mapping[module_name]
try:
return GlobalTuple(bytes(module_name, "ascii"),
bytes(name, "ascii"))
except UnicodeEncodeError:
raise PicklingError(
"can't pickle global identifier '%s.%s' using "
"pickle protocol %i" % (module, name, proto))
def jelly(self, obj):
if isinstance(obj, Jellyable):
preRef = self._checkMutable(obj)
if preRef:
return preRef
return obj.jellyFor(self)
objType = type(obj)
if self.taster.isTypeAllowed(qual(objType)):
# "Immutable" Types
if ((objType is StringType) or (objType is IntType)
or (objType is LongType) or (objType is FloatType)):
return obj
elif objType is MethodType:
return [
"method", obj.im_func.__name__,
self.jelly(obj.im_self),
self.jelly(obj.im_class)
]
elif UnicodeType and objType is UnicodeType:
return ['unicode', obj.encode('UTF-8')]
elif objType is NoneType:
return ['None']
elif objType is FunctionType:
name = obj.__name__
return [
'function',
str(pickle.whichmodule(obj, obj.__name__)) + '.' + name
]
elif objType is ModuleType:
return ['module', obj.__name__]
elif objType is BooleanType:
return ['boolean', obj and 'true' or 'false']
elif objType is datetime.datetime:
if obj.tzinfo:
raise NotImplementedError(
"Currently can't jelly datetime objects with tzinfo")
return [
'datetime',
'%s %s %s %s %s %s %s' %
(obj.year, obj.month, obj.day, obj.hour, obj.minute,
obj.second, obj.microsecond)
]
elif objType is datetime.time:
if obj.tzinfo:
raise NotImplementedError(
"Currently can't jelly datetime objects with tzinfo")
return [
'time',
'%s %s %s %s' %
(obj.hour, obj.minute, obj.second, obj.microsecond)
]
elif objType is datetime.date:
return ['date', '%s %s %s' % (obj.year, obj.month, obj.day)]
elif objType is datetime.timedelta:
return [
'timedelta',
'%s %s %s' % (obj.days, obj.seconds, obj.microseconds)
]
elif objType is ClassType or issubclass(objType, type):
return ['class', qual(obj)]
elif decimal is not None and objType is decimal.Decimal:
return self.jelly_decimal(obj)
else:
preRef = self._checkMutable(obj)
if preRef:
return preRef
# "Mutable" Types
sxp = self.prepare(obj)
if objType is ListType:
sxp.extend(self._jellyIterable(list_atom, obj))
elif objType is TupleType:
sxp.extend(self._jellyIterable(tuple_atom, obj))
elif objType in DictTypes:
sxp.append(dictionary_atom)
for key, val in obj.items():
sxp.append([self.jelly(key), self.jelly(val)])
elif (_set is not None and objType is set
or objType is _sets.Set):
sxp.extend(self._jellyIterable(set_atom, obj))
elif (_set is not None and objType is frozenset
or objType is _sets.ImmutableSet):
sxp.extend(self._jellyIterable(frozenset_atom, obj))
else:
className = qual(obj.__class__)
persistent = None
if self.persistentStore:
persistent = self.persistentStore(obj, self)
if persistent is not None:
sxp.append(persistent_atom)
sxp.append(persistent)
elif self.taster.isClassAllowed(obj.__class__):
sxp.append(className)
if hasattr(obj, "__getstate__"):
state = obj.__getstate__()
else:
state = obj.__dict__
sxp.append(self.jelly(state))
else:
self.unpersistable(
"instance of class %s deemed insecure" %
qual(obj.__class__), sxp)
return self.preserve(obj, sxp)
else:
if objType is InstanceType:
raise InsecureJelly("Class not allowed for instance: %s %s" %
(obj.__class__, obj))
raise InsecureJelly("Type not allowed for object: %s %s" %
(objType, obj))
self.__listenTask.servListen()
if self.__listenTask.hasStopServs():
time.sleep(4)
self.__listenTask.servReStart()
time.sleep(4)
def SvcStop(self):
self.__run = False
time.sleep(2)
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
win32event.SetEvent(self.hWaitStop)
if __name__ == '__main__':
if len(sys.argv) == 1:
try:
evtsrc_dll = os.path.abspath(servicemanager.__file__)
servicemanager.PrepareToHostSingle(AIZYingListenService)
servicemanager.Initialize('AIZYingListenService', evtsrc_dll)
servicemanager.StartServiceCtrlDispatcher()
except win32service.error as details:
if details[0] == winerror.ERROR_FAILED_SERVICE_CONTROLLER_CONNECT:
win32serviceutil.usage()
else:
import pickle
modName = pickle.whichmodule(AIZYingListenService,
AIZYingListenService.__name__)
win32serviceutil.HandleCommandLine(AIZYingListenService)
def save_function(self, obj, name=None):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
write = self.write
if name is None:
name = obj.__name__
modname = pickle.whichmodule(obj, name)
# print('which gives %s %s %s' % (modname, obj, name))
try:
themodule = sys.modules[modname]
except KeyError:
# eval'd items such as namedtuple give invalid items for their function __module__
modname = '__main__'
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
if getattr(themodule, name, None) is obj:
return self.save_global(obj, name)
# a builtin_function_or_method which comes in as an attribute of some
# object (e.g., object.__new__, itertools.chain.from_iterable) will end
# up with modname "__main__" and so end up here. But these functions
# have no __code__ attribute in CPython, so the handling for
# user-defined functions below will fail.
# So we pickle them here using save_reduce; have to do it differently
# for different python versions.
if not hasattr(obj, '__code__'):
if sys.version_info >= (3, 4):
rv = obj.__reduce_ex__(self.proto)
elif hasattr(obj, '__self__'):
rv = (getattr, (obj.__self__, name))
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
return Pickler.save_reduce(self, obj=obj, *rv)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if (islambda(obj)
or getattr(obj.__code__, 'co_filename', None) == '<stdin>'
or themodule is None):
self.save_function_tuple(obj)
return
else:
# func is nested
klass = getattr(themodule, name, None)
if klass is None or klass is not obj:
self.save_function_tuple(obj)
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def save_global(self, obj, name=None):
# unfortunately the pickler code is factored in a way that
# forces us to copy/paste this function. The only change is marked
# CHANGED below.
write = self.write
memo = self.memo
if name is None:
name = getattr(obj, '__qualname__', None)
if name is None:
name = obj.__name__
module_name = whichmodule(obj, name)
try:
# CHANGED: self.import_module rather than
# __import__
module = self.import_module(module_name)
obj2, parent = _getattribute(module, name)
except (ImportError, KeyError, AttributeError):
raise PicklingError("Can't pickle %r: it's not found as %s.%s" %
(obj, module_name, name)) from None
else:
if obj2 is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module_name, name))
if self.proto >= 2:
code = _extension_registry.get((module_name, name))
if code:
assert code > 0
if code <= 0xff:
write(EXT1 + pack("<B", code))
elif code <= 0xffff:
write(EXT2 + pack("<H", code))
else:
write(EXT4 + pack("<i", code))
return
lastname = name.rpartition('.')[2]
if parent is module:
name = lastname
# Non-ASCII identifiers are supported only with protocols >= 3.
if self.proto >= 4:
self.save(module_name)
self.save(name)
write(STACK_GLOBAL)
elif parent is not module:
self.save_reduce(getattr, (parent, lastname))
elif self.proto >= 3:
write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
bytes(name, "utf-8") + b'\n')
else:
if self.fix_imports:
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
if (module_name, name) in r_name_mapping:
module_name, name = r_name_mapping[(module_name, name)]
elif module_name in r_import_mapping:
module_name = r_import_mapping[module_name]
try:
write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
bytes(name, "ascii") + b'\n')
except UnicodeEncodeError:
raise PicklingError(
"can't pickle global identifier '%s.%s' using "
"pickle protocol %i" %
(module, name, self.proto)) from None
self.memoize(obj)
def sliceBody(self, streamable, banana):
name = self.obj.__name__
fullname = str(whichmodule(self.obj, self.obj.__name__)) + '.' + name
yield fullname
def fullFuncName(func):
qualName = (str(pickle.whichmodule(func, func.__name__)) + '.' + func.__name__)
if namedObject(qualName) is not func:
raise Exception("Couldn't find %s as %s." % (func, qualName))
return qualName
def jelly(self, obj):
if isinstance(obj, Jellyable):
refId = self._ref_id
self._ref_id += 1
preRef = self._checkMutable(obj, refId)
if preRef:
return preRef
return obj.jellyFor(self)
objType = type(obj)
if self.taster.isTypeAllowed(
string.replace(objType.__name__, ' ', '_')):
if ((objType is StringType) or
(objType is IntType) or
(objType is LongType) or
(objType is FloatType)):
return obj
refId = self._ref_id
self._ref_id += 1
if objType is MethodType:
return ["method",
obj.im_func.__name__,
self.jelly(obj.im_self),
self.jelly(obj.im_class)]
elif UnicodeType and objType is UnicodeType:
return ['unicode', obj.encode('UTF-8')]
elif objType is NoneType:
return ['None']
elif objType is FunctionType:
name = obj.__name__
return ['function', str(pickle.whichmodule(obj, obj.__name__))
+ '.' +
name]
elif objType is ModuleType:
return ['module', obj.__name__]
elif objType is BooleanType:
return ['boolean', obj and 'true' or 'false']
elif objType is ClassType or issubclass(objType, type):
return ['class', qual(obj)]
else:
preRef = self._checkMutable(obj, refId)
if preRef:
return preRef
sxp = []
if objType is ListType:
sxp.append(list_atom)
for item in obj:
sxp.append(self.jelly(item))
elif objType is TupleType:
sxp.append(tuple_atom)
for item in obj:
sxp.append(self.jelly(item))
elif objType in DictTypes:
sxp.append(dictionary_atom)
for key, val in obj.items():
self._ref_id += 1
sxp.append([self.jelly(key), self.jelly(val)])
elif objType is InstanceType:
className = qual(obj.__class__)
if self.taster.isClassAllowed(obj.__class__):
sxp.append(className)
if hasattr(obj, "__getstate__"):
state = obj.__getstate__()
else:
state = obj.__dict__
sxp.append(self.jelly(state))
else:
self.unpersistable(
"instance of class %s deemed insecure" %
qual(obj.__class__), sxp)
else:
raise NotImplementedError("Don't know the type: %s" % objType)
return sxp
else:
if objType is types.InstanceType:
raise InsecureJelly("Class not allowed for instance: %s %s" %
(obj.__class__, obj))
raise InsecureJelly("Type not allowed for object: %s %s" %
(objType, obj))
def __init__(self,
processes=None,
temp_folder=None,
max_nbytes=1e6,
mmap_mode='r',
forward_reducers=None,
backward_reducers=None,
verbose=0,
context_id=None,
prewarm=False,
**kwargs):
if forward_reducers is None:
forward_reducers = dict()
if backward_reducers is None:
backward_reducers = dict()
if context_id is not None:
warnings.warn(
'context_id is deprecated and ignored in joblib'
' 0.9.4 and will be removed in 0.11', DeprecationWarning)
# Prepare a sub-folder name for the serialization of this particular
# pool instance (do not create in advance to spare FS write access if
# no array is to be dumped):
use_shared_mem = False
pool_folder_name = "joblib_memmaping_pool_%d_%d" % (os.getpid(),
id(self))
if temp_folder is None:
temp_folder = os.environ.get('JOBLIB_TEMP_FOLDER', None)
if temp_folder is None:
if os.path.exists(SYSTEM_SHARED_MEM_FS):
try:
temp_folder = SYSTEM_SHARED_MEM_FS
pool_folder = os.path.join(temp_folder, pool_folder_name)
if not os.path.exists(pool_folder):
os.makedirs(pool_folder)
use_shared_mem = True
except IOError:
# Missing rights in the the /dev/shm partition,
# fallback to regular temp folder.
temp_folder = None
if temp_folder is None:
# Fallback to the default tmp folder, typically /tmp
temp_folder = tempfile.gettempdir()
temp_folder = os.path.abspath(os.path.expanduser(temp_folder))
pool_folder = os.path.join(temp_folder, pool_folder_name)
self._temp_folder = pool_folder
# Register the garbage collector at program exit in case caller forgets
# to call terminate explicitly: note we do not pass any reference to
# self to ensure that this callback won't prevent garbage collection of
# the pool instance and related file handler resources such as POSIX
# semaphores and pipes
pool_module_name = whichmodule(delete_folder, 'delete_folder')
def _cleanup():
# In some cases the Python runtime seems to set delete_folder to
# None just before exiting when accessing the delete_folder
# function from the closure namespace. So instead we reimport
# the delete_folder function explicitly.
# https://github.com/joblib/joblib/issues/328
# We cannot just use from 'joblib.pool import delete_folder'
# because joblib should only use relative imports to allow
# easy vendoring.
delete_folder = __import__(pool_module_name,
fromlist=['delete_folder'
]).delete_folder
delete_folder(pool_folder)
atexit.register(_cleanup)
if np is not None:
# Register smart numpy.ndarray reducers that detects memmap backed
# arrays and that is alse able to dump to memmap large in-memory
# arrays over the max_nbytes threshold
if prewarm == "auto":
prewarm = not use_shared_mem
forward_reduce_ndarray = ArrayMemmapReducer(max_nbytes,
pool_folder,
mmap_mode,
verbose,
prewarm=prewarm)
forward_reducers[np.ndarray] = forward_reduce_ndarray
forward_reducers[np.memmap] = reduce_memmap
# Communication from child process to the parent process always
# pickles in-memory numpy.ndarray without dumping them as memmap
# to avoid confusing the caller and make it tricky to collect the
# temporary folder
backward_reduce_ndarray = ArrayMemmapReducer(
None, pool_folder, mmap_mode, verbose)
backward_reducers[np.ndarray] = backward_reduce_ndarray
backward_reducers[np.memmap] = reduce_memmap
poolargs = dict(processes=processes,
forward_reducers=forward_reducers,
backward_reducers=backward_reducers)
poolargs.update(kwargs)
super(MemmapingPool, self).__init__(**poolargs)
def get_memmapping_reducers(pool_id,
forward_reducers=None,
backward_reducers=None,
temp_folder=None,
max_nbytes=1e6,
mmap_mode='r',
verbose=0,
prewarm=False,
**kwargs):
"""Construct a pair of memmapping reducer linked to a tmpdir.
This function manage the creation and the clean up of the temporary folders
underlying the memory maps and should be use to get the reducers necessary
to construct joblib pool or executor.
"""
if forward_reducers is None:
forward_reducers = dict()
if backward_reducers is None:
backward_reducers = dict()
# Prepare a sub-folder name for the serialization of this particular
# pool instance (do not create in advance to spare FS write access if
# no array is to be dumped):
pool_folder_name = "joblib_memmapping_folder_{}_{}".format(
os.getpid(), pool_id)
pool_folder, use_shared_mem = _get_temp_dir(pool_folder_name, temp_folder)
# Register the garbage collector at program exit in case caller forgets
# to call terminate explicitly: note we do not pass any reference to
# self to ensure that this callback won't prevent garbage collection of
# the pool instance and related file handler resources such as POSIX
# semaphores and pipes
pool_module_name = whichmodule(delete_folder, 'delete_folder')
def _cleanup():
# In some cases the Python runtime seems to set delete_folder to
# None just before exiting when accessing the delete_folder
# function from the closure namespace. So instead we reimport
# the delete_folder function explicitly.
# https://github.com/joblib/joblib/issues/328
# We cannot just use from 'joblib.pool import delete_folder'
# because joblib should only use relative imports to allow
# easy vendoring.
delete_folder = __import__(pool_module_name,
fromlist=['delete_folder']).delete_folder
try:
delete_folder(pool_folder)
except WindowsError:
warnings.warn(
"Failed to clean temporary folder: {}".format(pool_folder))
atexit.register(_cleanup)
if np is not None:
# Register smart numpy.ndarray reducers that detects memmap backed
# arrays and that is also able to dump to memmap large in-memory
# arrays over the max_nbytes threshold
if prewarm == "auto":
prewarm = not use_shared_mem
forward_reduce_ndarray = ArrayMemmapReducer(max_nbytes,
pool_folder,
mmap_mode,
verbose,
prewarm=prewarm)
forward_reducers[np.ndarray] = forward_reduce_ndarray
forward_reducers[np.memmap] = reduce_memmap
# Communication from child process to the parent process always
# pickles in-memory numpy.ndarray without dumping them as memmap
# to avoid confusing the caller and make it tricky to collect the
# temporary folder
backward_reduce_ndarray = ArrayMemmapReducer(None, pool_folder,
mmap_mode, verbose)
backward_reducers[np.ndarray] = backward_reduce_ndarray
backward_reducers[np.memmap] = reduce_memmap
return forward_reducers, backward_reducers, pool_folder
def save_function(self, obj, name=None):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
try:
should_special_case = obj in _BUILTIN_TYPE_CONSTRUCTORS
except TypeError:
# Methods of builtin types aren't hashable in python 2.
should_special_case = False
if should_special_case:
# We keep a special-cased cache of built-in type constructors at
# global scope, because these functions are structured very
# differently in different python versions and implementations (for
# example, they're instances of types.BuiltinFunctionType in
# CPython, but they're ordinary types.FunctionType instances in
# PyPy).
#
# If the function we've received is in that cache, we just
# serialize it as a lookup into the cache.
return self.save_reduce(_BUILTIN_TYPE_CONSTRUCTORS[obj], (),
obj=obj)
write = self.write
if name is None:
name = obj.__name__
try:
# whichmodule() could fail, see
# https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
modname = pickle.whichmodule(obj, name)
except Exception:
modname = None
# print('which gives %s %s %s' % (modname, obj, name))
themodule = None
try:
themodule = sys.modules[modname]
except KeyError:
# eval'd items such as namedtuple give invalid items for their function __module__
modname = '__main__'
# if modname == '__main__':
# themodule = None
try:
lookedup_by_name = getattr(themodule, name, None)
except Exception:
lookedup_by_name = None
if themodule and modname != '__main__':
if lookedup_by_name is obj:
return self.save_global(obj, name)
# a builtin_function_or_method which comes in as an attribute of some
# object (e.g., itertools.chain.from_iterable) will end
# up with modname "__main__" and so end up here. But these functions
# have no __code__ attribute in CPython, so the handling for
# user-defined functions below will fail.
# So we pickle them here using save_reduce; have to do it differently
# for different python versions.
if not hasattr(obj, '__code__'):
if PY3: # pragma: no branch
rv = obj.__reduce_ex__(self.proto)
else:
if hasattr(obj, '__self__'):
rv = (getattr, (obj.__self__, name))
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
return self.save_reduce(obj=obj, *rv)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if (islambda(obj)
or getattr(obj.__code__, 'co_filename', None) == '<stdin>'
or themodule is None):
self.save_function_tuple(obj)
return
else:
# func is nested
if lookedup_by_name is None or lookedup_by_name is not obj:
self.save_function_tuple(obj)
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
write(pickle.GLOBAL + modname.encode('utf-8') + b'\n' +
name.encode('utf-8') + b'\n')
self.memoize(obj)
async def save_global(self, obj, name=None):
write = self.write
memo = self.memo
if name is None:
name = getattr(obj, '__qualname__', None)
if name is None:
name = obj.__name__
module_name = whichmodule(obj, name)
try:
__import__(module_name, level=0)
module = sys.modules[module_name]
obj2, parent = _getattribute(module, name)
except (ImportError, KeyError, AttributeError):
raise PicklingError(
"Can't pickle %r: it's not found as %s.%s" %
(obj, module_name, name)) from None
else:
if obj2 is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module_name, name))
if self.proto >= 2:
code = _extension_registry.get((module_name, name))
if code:
assert code > 0
if code <= 0xff:
await write(EXT1 + pack("<B", code))
elif code <= 0xffff:
await write(EXT2 + pack("<H", code))
else:
await write(EXT4 + pack("<i", code))
return
lastname = name.rpartition('.')[2]
if parent is module:
name = lastname
# Non-ASCII identifiers are supported only with protocols >= 3.
if self.proto >= 4:
await self.save(module_name)
await self.save(name)
await write(STACK_GLOBAL)
elif parent is not module:
await self.save_reduce(getattr, (parent, lastname))
elif self.proto >= 3:
await write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
bytes(name, "utf-8") + b'\n')
else:
if self.fix_imports:
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
if (module_name, name) in r_name_mapping:
module_name, name = r_name_mapping[(module_name, name)]
elif module_name in r_import_mapping:
module_name = r_import_mapping[module_name]
try:
await write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
bytes(name, "ascii") + b'\n')
except UnicodeEncodeError:
raise PicklingError(
"can't pickle global identifier '%s.%s' using "
"pickle protocol %i" % (module, name, self.proto)) from None
await self.memoize(obj)
def _ufunc_reduce(func):
from pickle import whichmodule
name = func.__name__
return _ufunc_reconstruct, (whichmodule(func, name), name)
def save_function(self, obj, name=None, pack=struct.pack):
""" Registered with the dispatch to handle all function types.
Determines what kind of function obj is (e.g. lambda, defined at
interactive prompt, etc) and handles the pickling appropriately.
"""
write = self.write
name = obj.__name__
modname = pickle.whichmodule(obj, name)
#print 'which gives %s %s %s' % (modname, obj, name)
try:
themodule = sys.modules[modname]
except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__
modname = '__main__'
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
if self.os_env_vars:
# recursion hackery
os_env_vars = self.os_env_vars
self.os_env_vars = []
old_saved_django = self.savedDjangoEnv
self.savedDjangoEnv = True
write(pickle.MARK)
if isinstance(os_env_vars, dict):
os_mapping = os_env_vars
else:
os_mapping = {}
for env_var in os_env_vars:
if env_var in os.environ:
os_mapping[env_var] = os.environ[env_var]
self.save_reduce(
env_vars_load,
(os_mapping, )) # note: nothing sensible to memoize
write(pickle.POP_MARK)
self.savedDjangoEnv = old_saved_django
if modname == 'cloud.shell':
# don't save django environment if we are using shell.exec
self.savedDjangoEnv = True
elif not self.savedDjangoEnv:
# hack for django - if we detect the settings module, we transport it
# unfortunately this module is dynamically, not statically, resolved, so
# dependency analysis never detects it
django_settings = os.environ.get('DJANGO_SETTINGS_MODULE', '')
if django_settings:
django_mod = sys.modules.get(django_settings)
if django_mod:
logger.debug(
'Transporting django settings %s during save of %s',
django_mod, name)
self.savedDjangoEnv = True
self.modules.add(django_mod)
write(pickle.MARK)
self.save_reduce(django_settings_load,
(django_mod.__name__, ),
obj=django_mod)
write(pickle.POP_MARK)
# if func is lambda, def'ed at prompt, is in main, or is nested, then
# we'll pickle the actual function object rather than simply saving a
# reference (as is done in default pickler), via save_function_tuple.
if islambda(
obj
) or obj.func_code.co_filename == '<stdin>' or themodule == None:
#Force server to import modules that have been imported in main
modList = None
if themodule == None and not self.savedForceImports:
mainmod = sys.modules['__main__']
if useForcedImports and hasattr(mainmod,
'___pyc_forcedImports__'):
modList = list(mainmod.___pyc_forcedImports__)
self.savedForceImports = True
self.save_function_tuple(obj, modList)
return
else: # func is nested
klass = getattr(themodule, name, None)
if klass is None or klass is not obj:
self.save_function_tuple(obj, [themodule])
return
if obj.__dict__:
# essentially save_reduce, but workaround needed to avoid recursion
self.save(_restore_attr)
write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
self.save(obj.__dict__)
write(pickle.TUPLE + pickle.REDUCE)
else:
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def save_global(self, obj, name=None, pack=struct.pack):
write = self.write
memo = self.memo
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
try:
__import__(modname)
themodule = sys.modules[modname]
except (ImportError, KeyError, AttributeError): #should never occur
raise pickle.PicklingError(
"Can't pickle %r: Module %s cannot be found" % (obj, modname))
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
sendRef = True
typ = type(obj)
#print 'saving', obj, typ
try:
try: #Deal with case when getattribute fails with exceptions
klass = getattr(themodule, name)
except (AttributeError):
if modname == '__builtin__': #new.* are misrepeported
modname = 'new'
__import__(modname)
themodule = sys.modules[modname]
try:
klass = getattr(themodule, name)
except AttributeError, a:
# print themodule, name, obj, type(obj)
raise pickle.PicklingError("Can't pickle builtin %s" %
obj)
else:
raise
except (ImportError, KeyError, AttributeError):
if typ == types.TypeType or typ == types.ClassType:
sendRef = False
else: #we can't deal with this
raise
else:
if klass is not obj and (typ == types.TypeType
or typ == types.ClassType):
sendRef = False
if not sendRef:
#note: Third party types might crash this - add better checks!
d = dict(obj.__dict__) #copy dict proxy to a dict
if not isinstance(
d.get('__dict__', None),
property): # don't extract dict that are properties
d.pop('__dict__', None)
d.pop('__weakref__', None)
# hack as __new__ is stored differently in the __dict__
new_override = d.get('__new__', None)
if new_override:
d['__new__'] = obj.__new__
self.save_reduce(type(obj), (obj.__name__, obj.__bases__, d),
obj=obj)
#print 'internal reduce dask %s %s' % (obj, d)
return
if self.proto >= 2:
code = _extension_registry.get((modname, name))
if code:
assert code > 0
if code <= 0xff:
write(pickle.EXT1 + chr(code))
elif code <= 0xffff:
write("%c%c%c" % (pickle.EXT2, code & 0xff, code >> 8))
else:
write(pickle.EXT4 + pack("<i", code))
return
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
def save_global(self, obj, name=None, pack=struct.pack):
write = self.write
memo = self.memo
if name is None:
name = obj.__name__
modname = getattr(obj, "__module__", None)
if modname is None:
modname = pickle.whichmodule(obj, name)
try:
__import__(modname)
themodule = sys.modules[modname]
except (ImportError, KeyError, AttributeError): #should never occur
raise pickle.PicklingError(
"Can't pickle %r: Module %s cannot be found" % (obj, modname))
if modname == '__main__':
themodule = None
if themodule:
self.modules.add(themodule)
sendRef = True
typ = type(obj)
#print 'saving', obj, typ
try:
try: #Deal with case when getattribute fails with exceptions
klass = getattr(themodule, name)
except (AttributeError):
if modname == '__builtin__': #new.* are misrepeported
modname = 'new'
__import__(modname)
themodule = sys.modules[modname]
try:
klass = getattr(themodule, name)
except AttributeError, a:
#print themodule, name, obj, type(obj)
raise pickle.PicklingError("Can't pickle builtin %s" %
obj)
else:
raise
except (ImportError, KeyError, AttributeError):
if typ == types.TypeType or typ == types.ClassType:
sendRef = False
else: #we can't deal with this
raise
else:
if klass is not obj and (typ == types.TypeType
or typ == types.ClassType):
sendRef = False
if not sendRef:
self.save_class_obj(obj, name, pack)
return
if self.proto >= 2:
code = _extension_registry.get((modname, name))
if code:
assert code > 0
if code <= 0xff:
write(pickle.EXT1 + chr(code))
elif code <= 0xffff:
write("%c%c%c" % (pickle.EXT2, code & 0xff, code >> 8))
else:
write(pickle.EXT4 + pack("<i", code))
return
write(pickle.GLOBAL + modname + '\n' + name + '\n')
self.memoize(obj)
