def initialize(self, max_pass):
# List of (object, generator) tuples that initialize objects.
generators = []
# Execute object's initialize to setup the generators.
for obj in self.objects:
if hasattr(obj, '__initialize__') and \
callable(obj.__initialize__):
ret = obj.__initialize__()
if isinstance(ret, GeneratorType):
generators.append((obj, ret))
elif ret is not None:
raise UnpicklingError('Unexpected return value from '
'__initialize__. %s returned %s' %
(obj, ret))
# Ensure a maximum number of passes
if max_pass < 0:
max_pass = len(generators)
# Now run the generators.
count = 0
while len(generators) > 0:
count += 1
if count > max_pass:
not_done = [x[0] for x in generators]
msg = """Reached maximum pass count %s. You may have
a deadlock! The following objects are
uninitialized: %s""" % (max_pass, not_done)
raise UnpicklingError(msg)
for o, g in generators[:]:
try:
g.next()
except StopIteration:
generators.remove((o, g))
async def load_long4(self):
n, = unpack('<i', await self.read(4))
if n < 0:
# Corrupt or hostile pickle -- we never write one like this
raise UnpicklingError("LONG pickle has negative byte count")
data = await self.read(n)
self.append(decode_long(data))
def _create_lock(locked, *args):
from threading import Lock
lock = Lock()
if locked:
if not lock.acquire(False):
raise UnpicklingError("Cannot acquire lock")
return lock
def _get_target_class(self, module, name):
""" Returns the class info that the class, within the specified module
and with the specified name, should be instantiated as according to
our associated updater.
This is done in a manner that allows for chaining of class mappings
but is tolerant of the fact that a mapping away from an
intermediate class may not be registered until an attempt is made
to load that class.
"""
# Keep a record of the original class asked for.
original_module, original_name = module, name
# Iterate through any mappings in a manner that allows us to detect any
# infinite loops.
visited = []
while self.updater.has_class_mapping(module, name):
if (module, name) in visited:
from enthought.sweet_pickle import UnpicklingError
raise UnpicklingError('Detected infinite loop in class ' + \
'mapping from [%s.%s] to [%s.%s] within Updater [%s]' % \
(original_module, original_name, module, name,
self.updater))
visited.append( (module, name) )
# Get the mapping for the current class and try loading the class
# to ensure any mappings away from it are registered.
module, name = self.updater.class_map[(module, name)]
try:
super(VersionedUnpickler, self).find_class(module, name)
except:
logger.exception("_get_target_class can't find: %s" % (module, name))
pass
return module, name
def loads(self, s):
up = Unpickler(BytesIO(s))
up.persistent_load = self._get_object
try:
return up.load()
except KeyError as e:
raise UnpicklingError("Could not find Node class for %s" % e)
def pyobjectDecode(coder, setValue):
tp = coder.decodeIntForKey_(kKIND)
f = decode_dispatch.get(tp)
if f is None:
raise UnpicklingError("Unknown object kind: %s" % (tp, ))
return f(coder, setValue)
def _create_filehandle(name, mode, position, closed, open=open): # buffering=0
# only pickles the handle, not the file contents... good? or StringIO(data)?
# (for file contents see: http://effbot.org/librarybook/copy-reg.htm)
# NOTE: handle special cases first (are there more special cases?)
names = {
'<stdin>': sys.__stdin__,
'<stdout>': sys.__stdout__,
'<stderr>': sys.__stderr__
} #XXX: better fileno=(0,1,2) ?
if name in list(names.keys()): f = names[name] #XXX: safer "f=sys.stdin"
elif name == '<tmpfile>':
import os
f = os.tmpfile()
elif name == '<fdopen>':
import tempfile
f = tempfile.TemporaryFile(mode)
else:
try: # try to open the file by name # NOTE: has different fileno
f = open(name,
mode) #FIXME: missing: *buffering*, encoding,softspace
except IOError:
err = sys.exc_info()[1]
try: # failing, then use /dev/null #XXX: better to just fail here?
import os
f = open(os.devnull, mode)
except IOError:
raise UnpicklingError(err)
#XXX: python default is closed '<uninitialized file>' file/mode
if closed: f.close()
elif position >= 0: f.seek(position)
return f
async def load(self):
"""Read a pickled object representation from the open file.
Return the reconstituted object hierarchy specified in the file.
"""
# Check whether Unpickler was initialized correctly. This is
# only needed to mimic the behavior of _pickle.Unpickler.dump().
if not hasattr(self, "_file_read"):
raise UnpicklingError("Unpickler.__init__() was not called by "
"%s.__init__()" %
(self.__class__.__name__, ))
self._unframer = _Unframer(self._file_read, self._file_readline)
self.read = self._unframer.read
self.readline = self._unframer.readline
self.metastack = []
self.stack = []
self.append = self.stack.append
self.proto = 0
read = self.read
dispatch = self.dispatch
try:
while True:
key = await read(1)
if not key:
raise EOFError
assert isinstance(key, bytes_types)
await dispatch[key[0]](self)
except _Stop as stopinst:
return stopinst.value
def persistent_load(self, pid: Tuple[str, str]) -> object:
type_tag, key_id = pid
if type_tag == "Namespace":
assert self.namespace_name == key_id
return self.namespace
else:
raise UnpicklingError("unsupported persistent object")
async def load_binstring(self):
# Deprecated BINSTRING uses signed 32-bit length
len, = unpack('<i', await self.read(4))
if len < 0:
raise UnpicklingError("BINSTRING pickle has negative byte count")
data = await self.read(len)
self.append(self._decode_string(data))
def _Load(self):
global BOOKMARKS
global BOOKMARKS_MODE
global UID
Log("LOADING BOOKMARKS")
try:
savefile = open(self.SAVE_PATH, "rb")
saveVersion = load(savefile)
if saveVersion != VERSION:
raise UnpicklingError("version difference in files")
BOOKMARKS_MODE = load(savefile)
UID = load(savefile)
BOOKMARKS = load(savefile)
except (OSError, IOError, UnpicklingError, EOFError, BaseException) as e:
print ("\nEXCEPTION:------- ")
print (e)
print("\nUNABLE TO LOAD BOOKMARKS. NUKING LOAD FILE")
#clear the load file :]
open(self.SAVE_PATH, "wb").close()
#if you can't load, try and save a "default" state
self._Save()
async def load_persid(self):
try:
pid = (await self.readline())[:-1].decode("ascii")
except UnicodeDecodeError:
raise UnpicklingError(
"persistent IDs in protocol 0 must be ASCII strings")
self.append(self.persistent_load(pid))
def restricted_find_global(mod_name, fn_name):
__import__(mod_name)
mod = sys.modules[mod_name]
fn = getattr(mod, fn_name)
if not (fn in JOB_REGISTRY or fn in _UNPICKLE_WHITELIST):
raise UnpicklingError('{}.{} is not allowed in jobs'.format(
mod_name, fn_name))
return fn
async def load_string(self):
data = (await self.readline())[:-1]
# Strip outermost quotes
if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
data = data[1:-1]
else:
raise UnpicklingError("the STRING opcode argument must be quoted")
self.append(self._decode_string(codecs.escape_decode(data)[0]))
def pickle_load(file, lookup_func=None):
"""Allows unpickling with manual control over class lookup on both Python
2 and Python 3.
Will unpickle from the current position to the final stop marker.
lookup_func gets passed a function for global lookup, the mod name
to import and the attribute name to return from the module
The lookup function passed to the callback can raise ImportError
or AttributeError.
Args:
file (fileobj)
lookup_func (callable or None)
Returns:
The unpickled objects
Raises:
pickle.UnpicklingError
"""
if PY2:
inst = cPickle.Unpickler(file)
if lookup_func is not None:
# this is just a dummy unpickler we use for fallback class lookup
unpickler = pickle.Unpickler(cBytesIO())
def find_global(mod, name):
return lookup_func(unpickler.find_class, mod, name)
inst.find_global = find_global
else:
if lookup_func is not None:
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
func = super(CustomUnpickler, self).find_class
return lookup_func(func, module, name)
unpickler_type = CustomUnpickler
else:
unpickler_type = pickle.Unpickler
# helps a lot, but only on py3
if isinstance(file, BytesIO):
file = BufferedReader(file)
inst = unpickler_type(file, encoding="bytes")
try:
return inst.load()
except UnpicklingError:
raise
except Exception as e:
# unpickle can fail in many ways
raise UnpicklingError(e)
def findGlobal(self, module, klass):
"""Find class name."""
if (module, klass) not in self.allowedGlobals():
raise UnpicklingError(
"For security reasons, you can\'t unpickle"
f' objects from module {module} with type {klass}.')
g = {}
exec(f'from {module} import {klass} as theClass', g)
return g['theClass']
def __check_allowed(module):
# check if we are allowed to unpickle from these modules.
i = module.find('.')
if i > 0:
package = module[:i]
else:
package = module
if package not in HDF5PersistentUnpickler.__allowed_packages:
raise UnpicklingError(
'{mod} not allowed to unpickle'.format(mod=module))
def pyobjectDecode(coder, setValue):
if coder.allowsKeyedCoding():
tp = coder.decodeIntForKey_(kKIND)
else:
tp = coder.decodeValueOfObjCType_at_(objc._C_INT, None)
f = decode_dispatch.get(tp)
if f is None:
raise UnpicklingError("Unknown object kind: %s" % (tp,))
return f(coder, setValue)
def persistent_load(self, pid):
# This method is invoked whenever a persistent ID is encountered.
# Here, pid is the type and the dataset id.
type_tag, key_id = pid
if type_tag == "np_array":
return self.group[str(key_id)][:]
else:
# Always raises an error if you cannot return the correct object.
# Otherwise, the unpickler will think None is the object referenced
# by the persistent ID.
raise UnpicklingError("unsupported persistent object")
def drop(cls, path):
"""Remove db file."""
if os.path.exists(path):
try:
# verify filetype before deleting
_ = cls.read(path)
os.remove(path)
except UnpicklingError("Aborted: not a database file."):
return
except FileNotFoundError:
logger.exception(f"nothing to delete at {path}")
def _recv(self):
"""
wrap recv method to provide additional features:
unpickle the message
and make sure receive one and only one message
:rtype : BaseClientMessage
"""
def _receive_len_header(sock):
"""
return then length of the message
return 0 if connection broken
:rtype : int
"""
buf = b''
while not buf.endswith(b'\n'):
temp_buf = sock.recv(1)
if len(temp_buf) == 0: # client disconnected
return 0
buf += temp_buf
length = int(buf)
return length
def _recv_real_message(sock, length):
"""
receive data until size of length reached
:rtype : BaseClientMessage
:type socket.socket
:type length: int
"""
buf = b''
while length > 0:
temp_buf = sock.recv(length)
if len(temp_buf) == 0: # client disconnected
return b''
length -= len(temp_buf)
buf += temp_buf
return buf
try:
message_len = _receive_len_header(self.socket)
if not message_len:
raise Exception("connection broken")
new_pmsg = _recv_real_message(self.socket, message_len) # pickled
if not new_pmsg:
raise Exception("connection broken")
except Exception: # connection broken
raise Exception("connection broken")
try:
new_msg = pickle.loads(new_pmsg)
except Exception:
raise UnpicklingError(new_pmsg)
else:
return new_msg
def find_class(self, module, name):
""" Returns the class definition for the named class within the
specified module.
Overridden here to:
- Allow updaters to redirect to a different class, possibly
within a different module.
- Ensure that any setstate hooks for the class are called
when the instance of this class is unpickled.
"""
# Remove any extraneous characters that an Unpickler might handle
# but a user wouldn't have included in their mapping definitions.
module = module.strip()
name = name.strip()
# Attempt to find the class, this may cause a new mapping for that
# very class to be introduced. That's why we ignore the result.
try:
klass = super(VersionedUnpickler, self).find_class(module, name)
except:
pass
# Determine the target class that the requested class should be
# mapped to according to our updater. The target class is the one
# at the end of any chain of mappings.
original_module, original_name = module, name
if self.updater is not None and \
self.updater.has_class_mapping(module, name):
module, name = self._get_target_class(module, name)
if module != original_module or name != original_name:
logger.debug('Unpickling [%s.%s] as [%s.%s]', original_module,
original_name, module, name)
# Retrieve the target class definition
try:
klass = super(VersionedUnpickler, self).find_class(module, name)
except Exception as e:
from apptools.sweet_pickle import UnpicklingError
logger.debug('Traceback when finding class [%s.%s]:' \
% (module, name), exc_info=True)
raise UnpicklingError(
'Unable to load class [%s.%s]. '
'Original exception was, "%s". map:%s' %
(module, name, str(e), self.updater.class_map))
# Make sure we run the updater's state functions if any are declared
# for the target class.
if self.updater is not None \
and self._has_state_function(original_module, original_name):
self._add_unpickler(klass, original_module, original_name)
return klass
def load_package(path):
try:
inputFile = open(path, 'rb')
unpickledObj = pickle.load(inputFile)
inputFile.close()
except UnpicklingError:
raise UnpicklingError(
"Pickle has failed to load a Package object from disk.")
else:
return unpickledObj
return unpickledObj
def load(file):
""" Read a json object representation from the opened file object file.
Args:
file: file object
Returns:
object representation as a dict
"""
try:
return json.load(file, object_hook=decode)
except (TypeError, ValueError) as e:
raise UnpicklingError(str(e))
def get_extension(self, code):
nil = []
obj = _extension_cache.get(code, nil)
if obj is not nil:
self.append(obj)
return
key = _inverted_registry.get(code)
if not key:
if code <= 0: # note that 0 is forbidden
# Corrupt or hostile pickle.
raise UnpicklingError("EXT specifies code <= 0")
raise ValueError("unregistered extension code %d" % code)
obj = self.find_class(*key)
_extension_cache[code] = obj
self.append(obj)
def _load_cache(self):
"""
Load existing cache or create new. Working only for UNIX-like systems and local files (not buffers).
"""
if platform == 'win32' or self._is_buffer:
return
try:
with open(self.__cache_path, 'rb') as f:
self._shifts = load(f)
except FileNotFoundError: # cache not found
self.reset_index()
except IsADirectoryError as e:
raise IsADirectoryError(f'Please delete {self.__cache_path} directory') from e
except (UnpicklingError, EOFError) as e: # invalid file. ask user to check it.
raise UnpicklingError(f'Invalid cache file {self.__cache_path}. Please delete it') from e
def _load_cache(self):
"""
the method is implemented for the purpose of optimization, byte positions will not be re-read from a file
that has already been used, if the content of the file has changed, and the name has been left the same,
the old version of byte offsets will be loaded
:return: list of byte offsets from existing file
"""
try:
with open(self.__cache_path, 'rb') as f:
return load(f)
except FileNotFoundError:
return
except IsADirectoryError as e:
raise IsADirectoryError(f'Please delete {self.__cache_path} directory') from e
except (UnpicklingError, EOFError) as e:
raise UnpicklingError(f'Invalid cache file {self.__cache_path}. Please delete it') from e
def loads(bytes_obj, encoding="utf-8", errors="strict"):
""" Read a json object representation from the bytes representation.
Args:
bytes_obj (bytes): bytes object representation
encoding (str): encoding to use to decode bytes
errors (str): same as decode 'errors' argument.
Returns:
object representation as a dict
"""
str_obj = bytes_obj.decode(encoding=encoding, errors=errors)
try:
return json.loads(str_obj, object_hook=decode)
except ValueError as e:
raise UnpicklingError(str(e))
def persistent_load(self, pid):
if isinstance(pid, tuple):
remote_type = pid[0]
if remote_type == PICKLE_PID_TYPE_REMOTE_OBJ:
remote_id = pid[1]
proxy_def = self._rop.get_remote_proxy_def(remote_id)
return self._rop.get_remote_proxy(proxy_def)
if remote_type == PICKLE_PID_TYPE_REMOTE_BACKREF:
remote_id = pid[1]
return self._rop.get_remote_obj_by_id(remote_id)
if remote_type == PICKLE_PID_TYPE_REMOTE_EXC_TB:
exc_payload = pid[1]
return rebuild_exception(*exc_payload)
raise UnpicklingError("unsupported persistent id encountered: %r" % pid)
def load(self):
"""Read a pickled object representation from the open file.
Return the reconstituted object hierarchy specified in the file.
"""
self.mark = object() # any new unique object
self.stack = _Stack()
self.append = self.stack.append
try:
key = ord(self.read(1))
while key != STOP:
try:
meth = self.dispatch[key]
except KeyError:
raise UnpicklingError("invalid load key, %r." % chr(key))
meth(self)
key = ord(self.read(1))
except TypeError:
if self.read(1) == '':
raise EOFError
raise
return self.stack.pop()
