async def load_long4(self):
n, = unpack('<i', await self.read(4))
if n < 0:
# Corrupt or hostile pickle -- we never write one like this
raise UnpicklingError("LONG pickle has negative byte count")
data = await self.read(n)
self.append(decode_long(data))
async def load_persid(self):
try:
pid = (await self.readline())[:-1].decode("ascii")
except UnicodeDecodeError:
raise UnpicklingError(
"persistent IDs in protocol 0 must be ASCII strings")
self.append(self.persistent_load(pid))
async def load_string(self):
data = (await self.readline())[:-1]
# Strip outermost quotes
if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
data = data[1:-1]
else:
raise UnpicklingError("the STRING opcode argument must be quoted")
self.append(self._decode_string(codecs.escape_decode(data)[0]))
def restricted_find_global(mod_name, fn_name):
__import__(mod_name)
mod = sys.modules[mod_name]
fn = getattr(mod, fn_name)
if not (fn in JOB_REGISTRY or fn in _UNPICKLE_WHITELIST):
raise UnpicklingError('{}.{} is not allowed in jobs'.format(
mod_name, fn_name))
return fn
def pickle_load(file, lookup_func=None):
"""Allows unpickling with manual control over class lookup on both Python
2 and Python 3.
Will unpickle from the current position to the final stop marker.
lookup_func gets passed a function for global lookup, the mod name
to import and the attribute name to return from the module
The lookup function passed to the callback can raise ImportError
or AttributeError.
Args:
file (fileobj)
lookup_func (callable or None)
Returns:
The unpickled objects
Raises:
pickle.UnpicklingError
"""
if PY2:
inst = cPickle.Unpickler(file)
if lookup_func is not None:
# this is just a dummy unpickler we use for fallback class lookup
unpickler = pickle.Unpickler(cBytesIO())
def find_global(mod, name):
return lookup_func(unpickler.find_class, mod, name)
inst.find_global = find_global
else:
if lookup_func is not None:
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
func = super(CustomUnpickler, self).find_class
return lookup_func(func, module, name)
unpickler_type = CustomUnpickler
else:
unpickler_type = pickle.Unpickler
# helps a lot, but only on py3
if isinstance(file, BytesIO):
file = BufferedReader(file)
inst = unpickler_type(file, encoding="bytes")
try:
return inst.load()
except UnpicklingError:
raise
except Exception as e:
# unpickle can fail in many ways
raise UnpicklingError(e)
def findGlobal(self, module, klass):
"""Find class name."""
if (module, klass) not in self.allowedGlobals():
raise UnpicklingError(
"For security reasons, you can\'t unpickle"
f' objects from module {module} with type {klass}.')
g = {}
exec(f'from {module} import {klass} as theClass', g)
return g['theClass']
def __check_allowed(module):
# check if we are allowed to unpickle from these modules.
i = module.find('.')
if i > 0:
package = module[:i]
else:
package = module
if package not in HDF5PersistentUnpickler.__allowed_packages:
raise UnpicklingError(
'{mod} not allowed to unpickle'.format(mod=module))
def pyobjectDecode(coder, setValue):
if coder.allowsKeyedCoding():
tp = coder.decodeIntForKey_(kKIND)
else:
tp = coder.decodeValueOfObjCType_at_(objc._C_INT, None)
f = decode_dispatch.get(tp)
if f is None:
raise UnpicklingError("Unknown object kind: %s" % (tp,))
return f(coder, setValue)
def drop(cls, path):
"""Remove db file."""
if os.path.exists(path):
try:
# verify filetype before deleting
_ = cls.read(path)
os.remove(path)
except UnpicklingError("Aborted: not a database file."):
return
except FileNotFoundError:
logger.exception(f"nothing to delete at {path}")
def persistent_load(self, pid):
# This method is invoked whenever a persistent ID is encountered.
# Here, pid is the type and the dataset id.
type_tag, key_id = pid
if type_tag == "np_array":
return self.group[str(key_id)][:]
else:
# Always raises an error if you cannot return the correct object.
# Otherwise, the unpickler will think None is the object referenced
# by the persistent ID.
raise UnpicklingError("unsupported persistent object")
def find_class(self, module, name):
""" Returns the class definition for the named class within the
specified module.
Overridden here to:
- Allow updaters to redirect to a different class, possibly
within a different module.
- Ensure that any setstate hooks for the class are called
when the instance of this class is unpickled.
"""
# Remove any extraneous characters that an Unpickler might handle
# but a user wouldn't have included in their mapping definitions.
module = module.strip()
name = name.strip()
# Attempt to find the class, this may cause a new mapping for that
# very class to be introduced. That's why we ignore the result.
try:
klass = super(VersionedUnpickler, self).find_class(module, name)
except:
pass
# Determine the target class that the requested class should be
# mapped to according to our updater. The target class is the one
# at the end of any chain of mappings.
original_module, original_name = module, name
if self.updater is not None and \
self.updater.has_class_mapping(module, name):
module, name = self._get_target_class(module, name)
if module != original_module or name != original_name:
logger.debug('Unpickling [%s.%s] as [%s.%s]', original_module,
original_name, module, name)
# Retrieve the target class definition
try:
klass = super(VersionedUnpickler, self).find_class(module, name)
except Exception as e:
from apptools.sweet_pickle import UnpicklingError
logger.debug('Traceback when finding class [%s.%s]:' \
% (module, name), exc_info=True)
raise UnpicklingError(
'Unable to load class [%s.%s]. '
'Original exception was, "%s". map:%s' %
(module, name, str(e), self.updater.class_map))
# Make sure we run the updater's state functions if any are declared
# for the target class.
if self.updater is not None \
and self._has_state_function(original_module, original_name):
self._add_unpickler(klass, original_module, original_name)
return klass
def _recv(self):
"""
wrap recv method to provide additional features:
unpickle the message
and make sure receive one and only one message
:rtype : BaseClientMessage
"""
def _receive_len_header(sock):
"""
return then length of the message
return 0 if connection broken
:rtype : int
"""
buf = b''
while not buf.endswith(b'\n'):
temp_buf = sock.recv(1)
if len(temp_buf) == 0: # client disconnected
return 0
buf += temp_buf
length = int(buf)
return length
def _recv_real_message(sock, length):
"""
receive data until size of length reached
:rtype : BaseClientMessage
:type socket.socket
:type length: int
"""
buf = b''
while length > 0:
temp_buf = sock.recv(length)
if len(temp_buf) == 0: # client disconnected
return b''
length -= len(temp_buf)
buf += temp_buf
return buf
try:
message_len = _receive_len_header(self.socket)
if not message_len:
raise Exception("connection broken")
new_pmsg = _recv_real_message(self.socket, message_len) # pickled
if not new_pmsg:
raise Exception("connection broken")
except Exception: # connection broken
raise Exception("connection broken")
try:
new_msg = pickle.loads(new_pmsg)
except Exception:
raise UnpicklingError(new_pmsg)
else:
return new_msg
def load(file):
""" Read a json object representation from the opened file object file.
Args:
file: file object
Returns:
object representation as a dict
"""
try:
return json.load(file, object_hook=decode)
except (TypeError, ValueError) as e:
raise UnpicklingError(str(e))
def load_package(path):
try:
inputFile = open(path, 'rb')
unpickledObj = pickle.load(inputFile)
inputFile.close()
except UnpicklingError:
raise UnpicklingError(
"Pickle has failed to load a Package object from disk.")
else:
return unpickledObj
return unpickledObj
def initialize(self, max_pass):
# List of (object, generator) tuples that initialize objects.
generators = []
# Execute object's initialize to setup the generators.
for obj in self.objects:
if hasattr(obj, "__initialize__") and callable(obj.__initialize__):
ret = obj.__initialize__()
if isinstance(ret, GeneratorType):
generators.append((obj, ret))
elif ret is not None:
raise UnpicklingError("Unexpected return value from "
"__initialize__. %s returned %s" %
(obj, ret))
# Ensure a maximum number of passes
if max_pass < 0:
max_pass = len(generators)
# Now run the generators.
count = 0
while len(generators) > 0:
count += 1
if count > max_pass:
not_done = [x[0] for x in generators]
msg = """Reached maximum pass count %s. You may have
a deadlock! The following objects are
uninitialized: %s""" % (
max_pass,
not_done,
)
raise UnpicklingError(msg)
for o, g in generators[:]:
try:
next(g)
except StopIteration:
generators.remove((o, g))
def _load_cache(self):
"""
Load existing cache or create new. Working only for UNIX-like systems and local files (not buffers).
"""
if platform == 'win32' or self._is_buffer:
return
try:
with open(self.__cache_path, 'rb') as f:
self._shifts = load(f)
except FileNotFoundError: # cache not found
self.reset_index()
except IsADirectoryError as e:
raise IsADirectoryError(f'Please delete {self.__cache_path} directory') from e
except (UnpicklingError, EOFError) as e: # invalid file. ask user to check it.
raise UnpicklingError(f'Invalid cache file {self.__cache_path}. Please delete it') from e
def get_extension(self, code):
nil = []
obj = _extension_cache.get(code, nil)
if obj is not nil:
self.append(obj)
return
key = _inverted_registry.get(code)
if not key:
if code <= 0: # note that 0 is forbidden
# Corrupt or hostile pickle.
raise UnpicklingError("EXT specifies code <= 0")
raise ValueError("unregistered extension code %d" % code)
obj = self.find_class(*key)
_extension_cache[code] = obj
self.append(obj)
def loads(bytes_obj, encoding="utf-8", errors="strict"):
""" Read a json object representation from the bytes representation.
Args:
bytes_obj (bytes): bytes object representation
encoding (str): encoding to use to decode bytes
errors (str): same as decode 'errors' argument.
Returns:
object representation as a dict
"""
str_obj = bytes_obj.decode(encoding=encoding, errors=errors)
try:
return json.loads(str_obj, object_hook=decode)
except ValueError as e:
raise UnpicklingError(str(e))
def _load_cache(self):
"""
the method is implemented for the purpose of optimization, byte positions will not be re-read from a file
that has already been used, if the content of the file has changed, and the name has been left the same,
the old version of byte offsets will be loaded
:return: list of byte offsets from existing file
"""
try:
with open(self.__cache_path, 'rb') as f:
return load(f)
except FileNotFoundError:
return
except IsADirectoryError as e:
raise IsADirectoryError(f'Please delete {self.__cache_path} directory') from e
except (UnpicklingError, EOFError) as e:
raise UnpicklingError(f'Invalid cache file {self.__cache_path}. Please delete it') from e
def persistent_load(self, pid):
if isinstance(pid, tuple):
remote_type = pid[0]
if remote_type == PICKLE_PID_TYPE_REMOTE_OBJ:
remote_id = pid[1]
proxy_def = self._rop.get_remote_proxy_def(remote_id)
return self._rop.get_remote_proxy(proxy_def)
if remote_type == PICKLE_PID_TYPE_REMOTE_BACKREF:
remote_id = pid[1]
return self._rop.get_remote_obj_by_id(remote_id)
if remote_type == PICKLE_PID_TYPE_REMOTE_EXC_TB:
exc_payload = pid[1]
return rebuild_exception(*exc_payload)
raise UnpicklingError("unsupported persistent id encountered: %r" % pid)
def _create_filehandle(name, mode, position, closed, open=open): # buffering=0
# only pickles the handle, not the file contents... good? or StringIO(data)?
# (for file contents see: http://effbot.org/librarybook/copy-reg.htm)
# NOTE: handle special cases first (are there more special cases?)
names = {'<stdin>':sys.__stdin__, '<stdout>':sys.__stdout__,
'<stderr>':sys.__stderr__} #XXX: better fileno=(0,1,2) ?
if name in list(names.keys()): f = names[name] #XXX: safer "f=sys.stdin"
elif name == '<tmpfile>': import os; f = os.tmpfile()
elif name == '<fdopen>': import tempfile; f = tempfile.TemporaryFile(mode)
else:
try: # try to open the file by name # NOTE: has different fileno
f = open(name, mode)#FIXME: missing: *buffering*, encoding,softspace
except IOError:
err = sys.exc_info()[1]
try: # failing, then use /dev/null #XXX: better to just fail here?
import os; f = open(os.devnull, mode)
except IOError:
raise UnpicklingError(err)
#XXX: python default is closed '<uninitialized file>' file/mode
if closed: f.close()
elif position >= 0: f.seek(position)
return f
def load(self):
"""Read a pickled object representation from the open file.
Return the reconstituted object hierarchy specified in the file.
"""
self.mark = object() # any new unique object
self.stack = _Stack()
self.append = self.stack.append
try:
key = ord(self.read(1))
while key != STOP:
try:
meth = self.dispatch[key]
except KeyError:
raise UnpicklingError("invalid load key, %r." % chr(key))
meth(self)
key = ord(self.read(1))
except TypeError:
if self.read(1) == '':
raise EOFError
raise
return self.stack.pop()
def recv_from_server(self, my_sock):
# ----START HELPER FUNCTION----
def receive_len_header(sock):
buf = b''
while not buf.endswith(b'\n'):
temp_buf = sock.recv(1)
if len(temp_buf) == 0: # client disconnected
return 0
buf += temp_buf
length = int(buf)
#logger.debug("message length should be {}".format(length))
return length
def recv_real_message(sock, length):
buf = b''
while length > 0:
temp_buf = sock.recv(length)
if len(temp_buf) == 0: # client disconnected
return b''
length -= len(temp_buf)
buf += temp_buf
return buf
# ----END------------
try:
message_len = receive_len_header(my_sock)
if not message_len:
raise Exception("connection broken")
new_pmsg = recv_real_message(my_sock, message_len) # pickled
if not new_pmsg:
raise Exception("connection broken")
except Exception: # connection broken
raise Exception("connection broken")
try:
msg = pickle.loads(new_pmsg)
except Exception:
raise UnpicklingError(new_pmsg)
return msg
def restore(self, dpath=None, filename=None):
r""" Restore an EnergySystem instance.
"""
logging.info(
"Restoring attributes will overwrite existing attributes.")
if dpath is None:
dpath = os.path.join(os.path.expanduser("~"), '.oemof', 'dumps')
if filename is None:
filename = 'es_dump.oemof'
try:
self.__dict__ = pickle.load(
open(os.path.join(dpath, filename), "rb"))
except UnpicklingError as e:
if str(e) == "state is not a dictionary":
raise UnpicklingError(
"\n "
"Seems like you're trying to load an energy system "
"dumped with an older\n "
"oemof version. Unfortunetaly we made changes which "
"broke this from\n "
"v0.2.2 (more specifically commit `bec669b`) to its "
"successor.\n "
"If you really need this functionality, please file "
"a bug entitled\n\n "
'"Pickle customization removal breaks '
'`EnergySystem.restore`"\n\n '
"at\n\n "
"https://github.com/oemof/oemof/issues\n\n "
"or comment on it if it already exists.")
raise e
msg = ('Attributes restored from: {0}'.format(
os.path.join(dpath, filename)))
logging.debug(msg)
return msg
def _get_target_class(self, module, name):
""" Returns the class info that the class, within the specified module
and with the specified name, should be instantiated as according to
our associated updater.
This is done in a manner that allows for chaining of class mappings
but is tolerant of the fact that a mapping away from an
intermediate class may not be registered until an attempt is made
to load that class.
"""
# Keep a record of the original class asked for.
original_module, original_name = module, name
# Iterate through any mappings in a manner that allows us to detect any
# infinite loops.
visited = []
while self.updater.has_class_mapping(module, name):
if (module, name) in visited:
from apptools.sweet_pickle import UnpicklingError
raise UnpicklingError('Detected infinite loop in class ' + \
'mapping from [%s.%s] to [%s.%s] within Updater [%s]' % \
(original_module, original_name, module, name,
self.updater))
visited.append((module, name))
# Get the mapping for the current class and try loading the class
# to ensure any mappings away from it are registered.
module, name = self.updater.class_map[(module, name)]
try:
super(VersionedUnpickler, self).find_class(module, name)
except:
logger.exception("_get_target_class can't find: %s" %
(module, name))
pass
return module, name
def _create_filehandle(name, mode, position, closed, open, strictio, fmode,
fdata): # buffering=0
# only pickles the handle, not the file contents... good? or StringIO(data)?
# (for file contents see: http://effbot.org/librarybook/copy-reg.htm)
# NOTE: handle special cases first (are there more special cases?)
names = {
'<stdin>': sys.__stdin__,
'<stdout>': sys.__stdout__,
'<stderr>': sys.__stderr__
} #XXX: better fileno=(0,1,2) ?
if name in list(names.keys()):
f = names[name] #XXX: safer "f=sys.stdin"
elif name == '<tmpfile>':
f = os.tmpfile()
elif name == '<fdopen>':
import tempfile
f = tempfile.TemporaryFile(mode)
else:
# treat x mode as w mode
if "x" in mode and sys.hexversion < 0x03030000:
raise ValueError("invalid mode: '%s'" % mode)
try:
exists = os.path.exists(name)
except:
exists = False
if not exists:
if strictio:
raise FileNotFoundError(
"[Errno 2] No such file or directory: '%s'" % name)
elif "r" in mode and fmode != FILE_FMODE:
name = '<fdopen>' # or os.devnull?
current_size = 0 # or maintain position?
else:
current_size = os.path.getsize(name)
if position > current_size:
if strictio:
raise ValueError("invalid buffer size")
elif fmode == CONTENTS_FMODE:
position = current_size
# try to open the file by name
# NOTE: has different fileno
try:
#FIXME: missing: *buffering*, encoding, softspace
if fmode == FILE_FMODE:
f = open(name, mode if "w" in mode else "w")
f.write(fdata)
if "w" not in mode:
f.close()
f = open(name, mode)
elif name == '<fdopen>': # file did not exist
import tempfile
f = tempfile.TemporaryFile(mode)
elif fmode == CONTENTS_FMODE \
and ("w" in mode or "x" in mode):
# stop truncation when opening
flags = os.O_CREAT
if "+" in mode:
flags |= os.O_RDWR
else:
flags |= os.O_WRONLY
f = os.fdopen(os.open(name, flags), mode)
# set name to the correct value
if PY3:
r = getattr(f, "buffer", f)
r = getattr(r, "raw", r)
r.name = name
else:
if not HAS_CTYPES:
raise ImportError("No module named 'ctypes'")
class FILE(ctypes.Structure):
_fields_ = [("refcount", ctypes.c_long),
("type_obj", ctypes.py_object),
("file_pointer", ctypes.c_voidp),
("name", ctypes.py_object)]
class PyObject(ctypes.Structure):
_fields_ = [("ob_refcnt", ctypes.c_int),
("ob_type", ctypes.py_object)]
ctypes.cast(id(f),
ctypes.POINTER(FILE)).contents.name = name
ctypes.cast(
id(name),
ctypes.POINTER(PyObject)).contents.ob_refcnt += 1
assert f.name == name
else:
f = open(name, mode)
except (IOError, FileNotFoundError):
err = sys.exc_info()[1]
raise UnpicklingError(err)
if closed:
f.close()
elif position >= 0 and fmode != HANDLE_FMODE:
f.seek(position)
return f
def modify_state(self, obj, state, module, name):
""" Called to update the specified state dictionary, which represents
the class of the specified name within the specified module, to
complete the unpickling of the specified object.
"""
# Remove our setstate hook and associated data to ensure that
# instances unpickled through some other framework don't call us.
# IMPORTANT: Do this first to minimize the time this hook is in place!
self._remove_unpickler(obj.__class__)
# Determine what class and version we're starting from and going to.
# If there is no version information, then assume version 0. (0 is
# like an unversioned version.)
source_key = self.updater.get_version_attribute(module, name)
source_version = state.get(source_key, 0)
target_key = self.updater.get_version_attribute(
obj.__class__.__module__, obj.__class__.__name__)
target_version = getattr(obj, target_key, 0)
# Iterate through all the updates to the state by going one version
# at a time. Note that we assume there is exactly one path from our
# starting class and version to our ending class and version. As a
# result, we assume we update a given class to its latest version
# before looking for any class mappings. Note that the version in the
# updater is the version to convert *TO*.
version = source_version
next_version = version + 1
while True:
# Iterate through all version updates for the current class.
key = self.updater.get_version_attribute(module, name)
while (module, name, next_version) in self.updater.state_functions:
functions = self.updater.state_functions[(module, name,
next_version)]
for f in functions:
logger.debug('Modifying state from [%s.%s (v.%s)] to ' + \
'[%s.%s (v.%s)] using function %s', module, name,
version, module, name, next_version, f)
state = f(state)
# Avoid infinite loops due to versions not changing.
new_version = state.get(key, version)
if new_version == version:
new_version = version + 1
version = new_version
next_version = version + 1
# If there is one, move to the next class in the chain. (We
# explicitly keep the version number the same.)
if self.updater.has_class_mapping(module, name):
original_module, original_name = module, name
module, name = self.updater.class_map[(module, name)]
logger.debug('Modifying state from [%s.%s (v.%s)] to ' + \
'[%s.%s (v.%s)]', original_module, original_name, version,
module, name, version)
else:
break
# If one exists, call the final class's setstate method. According to
# standard pickling protocol, this method will apply the state to the
# instance so our state becomes None so that we don't try to apply our
# unfinished state to the object.
fn = getattr(obj, _SETSTATE_NAME, None)
if fn is not None:
fn(state)
result = None
version = getattr(obj, target_key)
else:
result = state
# Something is wrong if we aren't at our target class and version!
if module != obj.__class__.__module__ \
or name != obj.__class__.__name__ \
or version != target_version:
from apptools.sweet_pickle import UnpicklingError
raise UnpicklingError('Unexpected state! Got ' + \
'[%s.%s (v.%s)] expected [%s.%s (v.%s)]' % (module, name,
version, obj.__class__.__module__, obj.__class__.__name__,
target_version))
return result
def pop(self, index=-1):
try:
return list.pop(self, index)
except IndexError:
raise UnpicklingError("unpickling stack underflow")
def find_class(self, module, name):
if self.find_global is None:
raise UnpicklingError(
"Global and instance pickles are not supported.")
return self.find_global(module, name)
def persistent_load(self, pid):
raise UnpicklingError("unsupported persistent id encountered")