def initialize(self, max_pass): # List of (object, generator) tuples that initialize objects. generators = [] # Execute object's initialize to setup the generators. for obj in self.objects: if hasattr(obj, '__initialize__') and \ callable(obj.__initialize__): ret = obj.__initialize__() if isinstance(ret, GeneratorType): generators.append((obj, ret)) elif ret is not None: raise UnpicklingError('Unexpected return value from ' '__initialize__. %s returned %s' % (obj, ret)) # Ensure a maximum number of passes if max_pass < 0: max_pass = len(generators) # Now run the generators. count = 0 while len(generators) > 0: count += 1 if count > max_pass: not_done = [x[0] for x in generators] msg = """Reached maximum pass count %s. You may have a deadlock! The following objects are uninitialized: %s""" % (max_pass, not_done) raise UnpicklingError(msg) for o, g in generators[:]: try: g.next() except StopIteration: generators.remove((o, g))
async def load_long4(self): n, = unpack('<i', await self.read(4)) if n < 0: # Corrupt or hostile pickle -- we never write one like this raise UnpicklingError("LONG pickle has negative byte count") data = await self.read(n) self.append(decode_long(data))
def _create_lock(locked, *args): from threading import Lock lock = Lock() if locked: if not lock.acquire(False): raise UnpicklingError("Cannot acquire lock") return lock
def _get_target_class(self, module, name): """ Returns the class info that the class, within the specified module and with the specified name, should be instantiated as according to our associated updater. This is done in a manner that allows for chaining of class mappings but is tolerant of the fact that a mapping away from an intermediate class may not be registered until an attempt is made to load that class. """ # Keep a record of the original class asked for. original_module, original_name = module, name # Iterate through any mappings in a manner that allows us to detect any # infinite loops. visited = [] while self.updater.has_class_mapping(module, name): if (module, name) in visited: from enthought.sweet_pickle import UnpicklingError raise UnpicklingError('Detected infinite loop in class ' + \ 'mapping from [%s.%s] to [%s.%s] within Updater [%s]' % \ (original_module, original_name, module, name, self.updater)) visited.append( (module, name) ) # Get the mapping for the current class and try loading the class # to ensure any mappings away from it are registered. module, name = self.updater.class_map[(module, name)] try: super(VersionedUnpickler, self).find_class(module, name) except: logger.exception("_get_target_class can't find: %s" % (module, name)) pass return module, name
def loads(self, s): up = Unpickler(BytesIO(s)) up.persistent_load = self._get_object try: return up.load() except KeyError as e: raise UnpicklingError("Could not find Node class for %s" % e)
def pyobjectDecode(coder, setValue): tp = coder.decodeIntForKey_(kKIND) f = decode_dispatch.get(tp) if f is None: raise UnpicklingError("Unknown object kind: %s" % (tp, )) return f(coder, setValue)
def _create_filehandle(name, mode, position, closed, open=open): # buffering=0 # only pickles the handle, not the file contents... good? or StringIO(data)? # (for file contents see: http://effbot.org/librarybook/copy-reg.htm) # NOTE: handle special cases first (are there more special cases?) names = { '<stdin>': sys.__stdin__, '<stdout>': sys.__stdout__, '<stderr>': sys.__stderr__ } #XXX: better fileno=(0,1,2) ? if name in list(names.keys()): f = names[name] #XXX: safer "f=sys.stdin" elif name == '<tmpfile>': import os f = os.tmpfile() elif name == '<fdopen>': import tempfile f = tempfile.TemporaryFile(mode) else: try: # try to open the file by name # NOTE: has different fileno f = open(name, mode) #FIXME: missing: *buffering*, encoding,softspace except IOError: err = sys.exc_info()[1] try: # failing, then use /dev/null #XXX: better to just fail here? import os f = open(os.devnull, mode) except IOError: raise UnpicklingError(err) #XXX: python default is closed '<uninitialized file>' file/mode if closed: f.close() elif position >= 0: f.seek(position) return f
async def load(self): """Read a pickled object representation from the open file. Return the reconstituted object hierarchy specified in the file. """ # Check whether Unpickler was initialized correctly. This is # only needed to mimic the behavior of _pickle.Unpickler.dump(). if not hasattr(self, "_file_read"): raise UnpicklingError("Unpickler.__init__() was not called by " "%s.__init__()" % (self.__class__.__name__, )) self._unframer = _Unframer(self._file_read, self._file_readline) self.read = self._unframer.read self.readline = self._unframer.readline self.metastack = [] self.stack = [] self.append = self.stack.append self.proto = 0 read = self.read dispatch = self.dispatch try: while True: key = await read(1) if not key: raise EOFError assert isinstance(key, bytes_types) await dispatch[key[0]](self) except _Stop as stopinst: return stopinst.value
def persistent_load(self, pid: Tuple[str, str]) -> object: type_tag, key_id = pid if type_tag == "Namespace": assert self.namespace_name == key_id return self.namespace else: raise UnpicklingError("unsupported persistent object")
async def load_binstring(self): # Deprecated BINSTRING uses signed 32-bit length len, = unpack('<i', await self.read(4)) if len < 0: raise UnpicklingError("BINSTRING pickle has negative byte count") data = await self.read(len) self.append(self._decode_string(data))
def _Load(self): global BOOKMARKS global BOOKMARKS_MODE global UID Log("LOADING BOOKMARKS") try: savefile = open(self.SAVE_PATH, "rb") saveVersion = load(savefile) if saveVersion != VERSION: raise UnpicklingError("version difference in files") BOOKMARKS_MODE = load(savefile) UID = load(savefile) BOOKMARKS = load(savefile) except (OSError, IOError, UnpicklingError, EOFError, BaseException) as e: print ("\nEXCEPTION:------- ") print (e) print("\nUNABLE TO LOAD BOOKMARKS. NUKING LOAD FILE") #clear the load file :] open(self.SAVE_PATH, "wb").close() #if you can't load, try and save a "default" state self._Save()
async def load_persid(self): try: pid = (await self.readline())[:-1].decode("ascii") except UnicodeDecodeError: raise UnpicklingError( "persistent IDs in protocol 0 must be ASCII strings") self.append(self.persistent_load(pid))
def restricted_find_global(mod_name, fn_name): __import__(mod_name) mod = sys.modules[mod_name] fn = getattr(mod, fn_name) if not (fn in JOB_REGISTRY or fn in _UNPICKLE_WHITELIST): raise UnpicklingError('{}.{} is not allowed in jobs'.format( mod_name, fn_name)) return fn
async def load_string(self): data = (await self.readline())[:-1] # Strip outermost quotes if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'': data = data[1:-1] else: raise UnpicklingError("the STRING opcode argument must be quoted") self.append(self._decode_string(codecs.escape_decode(data)[0]))
def pickle_load(file, lookup_func=None): """Allows unpickling with manual control over class lookup on both Python 2 and Python 3. Will unpickle from the current position to the final stop marker. lookup_func gets passed a function for global lookup, the mod name to import and the attribute name to return from the module The lookup function passed to the callback can raise ImportError or AttributeError. Args: file (fileobj) lookup_func (callable or None) Returns: The unpickled objects Raises: pickle.UnpicklingError """ if PY2: inst = cPickle.Unpickler(file) if lookup_func is not None: # this is just a dummy unpickler we use for fallback class lookup unpickler = pickle.Unpickler(cBytesIO()) def find_global(mod, name): return lookup_func(unpickler.find_class, mod, name) inst.find_global = find_global else: if lookup_func is not None: class CustomUnpickler(pickle.Unpickler): def find_class(self, module, name): func = super(CustomUnpickler, self).find_class return lookup_func(func, module, name) unpickler_type = CustomUnpickler else: unpickler_type = pickle.Unpickler # helps a lot, but only on py3 if isinstance(file, BytesIO): file = BufferedReader(file) inst = unpickler_type(file, encoding="bytes") try: return inst.load() except UnpicklingError: raise except Exception as e: # unpickle can fail in many ways raise UnpicklingError(e)
def findGlobal(self, module, klass): """Find class name.""" if (module, klass) not in self.allowedGlobals(): raise UnpicklingError( "For security reasons, you can\'t unpickle" f' objects from module {module} with type {klass}.') g = {} exec(f'from {module} import {klass} as theClass', g) return g['theClass']
def __check_allowed(module): # check if we are allowed to unpickle from these modules. i = module.find('.') if i > 0: package = module[:i] else: package = module if package not in HDF5PersistentUnpickler.__allowed_packages: raise UnpicklingError( '{mod} not allowed to unpickle'.format(mod=module))
def pyobjectDecode(coder, setValue): if coder.allowsKeyedCoding(): tp = coder.decodeIntForKey_(kKIND) else: tp = coder.decodeValueOfObjCType_at_(objc._C_INT, None) f = decode_dispatch.get(tp) if f is None: raise UnpicklingError("Unknown object kind: %s" % (tp,)) return f(coder, setValue)
def persistent_load(self, pid): # This method is invoked whenever a persistent ID is encountered. # Here, pid is the type and the dataset id. type_tag, key_id = pid if type_tag == "np_array": return self.group[str(key_id)][:] else: # Always raises an error if you cannot return the correct object. # Otherwise, the unpickler will think None is the object referenced # by the persistent ID. raise UnpicklingError("unsupported persistent object")
def drop(cls, path): """Remove db file.""" if os.path.exists(path): try: # verify filetype before deleting _ = cls.read(path) os.remove(path) except UnpicklingError("Aborted: not a database file."): return except FileNotFoundError: logger.exception(f"nothing to delete at {path}")
def _recv(self): """ wrap recv method to provide additional features: unpickle the message and make sure receive one and only one message :rtype : BaseClientMessage """ def _receive_len_header(sock): """ return then length of the message return 0 if connection broken :rtype : int """ buf = b'' while not buf.endswith(b'\n'): temp_buf = sock.recv(1) if len(temp_buf) == 0: # client disconnected return 0 buf += temp_buf length = int(buf) return length def _recv_real_message(sock, length): """ receive data until size of length reached :rtype : BaseClientMessage :type socket.socket :type length: int """ buf = b'' while length > 0: temp_buf = sock.recv(length) if len(temp_buf) == 0: # client disconnected return b'' length -= len(temp_buf) buf += temp_buf return buf try: message_len = _receive_len_header(self.socket) if not message_len: raise Exception("connection broken") new_pmsg = _recv_real_message(self.socket, message_len) # pickled if not new_pmsg: raise Exception("connection broken") except Exception: # connection broken raise Exception("connection broken") try: new_msg = pickle.loads(new_pmsg) except Exception: raise UnpicklingError(new_pmsg) else: return new_msg
def find_class(self, module, name): """ Returns the class definition for the named class within the specified module. Overridden here to: - Allow updaters to redirect to a different class, possibly within a different module. - Ensure that any setstate hooks for the class are called when the instance of this class is unpickled. """ # Remove any extraneous characters that an Unpickler might handle # but a user wouldn't have included in their mapping definitions. module = module.strip() name = name.strip() # Attempt to find the class, this may cause a new mapping for that # very class to be introduced. That's why we ignore the result. try: klass = super(VersionedUnpickler, self).find_class(module, name) except: pass # Determine the target class that the requested class should be # mapped to according to our updater. The target class is the one # at the end of any chain of mappings. original_module, original_name = module, name if self.updater is not None and \ self.updater.has_class_mapping(module, name): module, name = self._get_target_class(module, name) if module != original_module or name != original_name: logger.debug('Unpickling [%s.%s] as [%s.%s]', original_module, original_name, module, name) # Retrieve the target class definition try: klass = super(VersionedUnpickler, self).find_class(module, name) except Exception as e: from apptools.sweet_pickle import UnpicklingError logger.debug('Traceback when finding class [%s.%s]:' \ % (module, name), exc_info=True) raise UnpicklingError( 'Unable to load class [%s.%s]. ' 'Original exception was, "%s". map:%s' % (module, name, str(e), self.updater.class_map)) # Make sure we run the updater's state functions if any are declared # for the target class. if self.updater is not None \ and self._has_state_function(original_module, original_name): self._add_unpickler(klass, original_module, original_name) return klass
def load_package(path): try: inputFile = open(path, 'rb') unpickledObj = pickle.load(inputFile) inputFile.close() except UnpicklingError: raise UnpicklingError( "Pickle has failed to load a Package object from disk.") else: return unpickledObj return unpickledObj
def load(file): """ Read a json object representation from the opened file object file. Args: file: file object Returns: object representation as a dict """ try: return json.load(file, object_hook=decode) except (TypeError, ValueError) as e: raise UnpicklingError(str(e))
def get_extension(self, code): nil = [] obj = _extension_cache.get(code, nil) if obj is not nil: self.append(obj) return key = _inverted_registry.get(code) if not key: if code <= 0: # note that 0 is forbidden # Corrupt or hostile pickle. raise UnpicklingError("EXT specifies code <= 0") raise ValueError("unregistered extension code %d" % code) obj = self.find_class(*key) _extension_cache[code] = obj self.append(obj)
def _load_cache(self): """ Load existing cache or create new. Working only for UNIX-like systems and local files (not buffers). """ if platform == 'win32' or self._is_buffer: return try: with open(self.__cache_path, 'rb') as f: self._shifts = load(f) except FileNotFoundError: # cache not found self.reset_index() except IsADirectoryError as e: raise IsADirectoryError(f'Please delete {self.__cache_path} directory') from e except (UnpicklingError, EOFError) as e: # invalid file. ask user to check it. raise UnpicklingError(f'Invalid cache file {self.__cache_path}. Please delete it') from e
def _load_cache(self): """ the method is implemented for the purpose of optimization, byte positions will not be re-read from a file that has already been used, if the content of the file has changed, and the name has been left the same, the old version of byte offsets will be loaded :return: list of byte offsets from existing file """ try: with open(self.__cache_path, 'rb') as f: return load(f) except FileNotFoundError: return except IsADirectoryError as e: raise IsADirectoryError(f'Please delete {self.__cache_path} directory') from e except (UnpicklingError, EOFError) as e: raise UnpicklingError(f'Invalid cache file {self.__cache_path}. Please delete it') from e
def loads(bytes_obj, encoding="utf-8", errors="strict"): """ Read a json object representation from the bytes representation. Args: bytes_obj (bytes): bytes object representation encoding (str): encoding to use to decode bytes errors (str): same as decode 'errors' argument. Returns: object representation as a dict """ str_obj = bytes_obj.decode(encoding=encoding, errors=errors) try: return json.loads(str_obj, object_hook=decode) except ValueError as e: raise UnpicklingError(str(e))
def persistent_load(self, pid): if isinstance(pid, tuple): remote_type = pid[0] if remote_type == PICKLE_PID_TYPE_REMOTE_OBJ: remote_id = pid[1] proxy_def = self._rop.get_remote_proxy_def(remote_id) return self._rop.get_remote_proxy(proxy_def) if remote_type == PICKLE_PID_TYPE_REMOTE_BACKREF: remote_id = pid[1] return self._rop.get_remote_obj_by_id(remote_id) if remote_type == PICKLE_PID_TYPE_REMOTE_EXC_TB: exc_payload = pid[1] return rebuild_exception(*exc_payload) raise UnpicklingError("unsupported persistent id encountered: %r" % pid)
def load(self): """Read a pickled object representation from the open file. Return the reconstituted object hierarchy specified in the file. """ self.mark = object() # any new unique object self.stack = _Stack() self.append = self.stack.append try: key = ord(self.read(1)) while key != STOP: try: meth = self.dispatch[key] except KeyError: raise UnpicklingError("invalid load key, %r." % chr(key)) meth(self) key = ord(self.read(1)) except TypeError: if self.read(1) == '': raise EOFError raise return self.stack.pop()