def getNewState(self, file):
# Would like to do load(file) here... but it doesn't work with universal line endings, see Python bug 1724366
from cStringIO import StringIO
unpickler = Unpickler(StringIO(file.read()))
# Magic to keep us backward compatible in the face of packages changing...
unpickler.find_global = self.findGlobal
return unpickler.load()
def getNewState(cls, file):
# Would like to do load(file) here... but it doesn't work with universal line endings, see Python bug 1724366
from cStringIO import StringIO
unpickler = Unpickler(StringIO(file.read()))
# Magic to keep us backward compatible in the face of packages changing...
unpickler.find_global = cls.findGlobal
return unpickler.load()
def unpickle(file_path=None):
"""Load pickled Python objects from a file.
Almost like ``cPickle.load(open(file_path))``, but also loads object saved
with older versions of Eelbrain, and allows using a system file dialog to
select a file.
Parameters
----------
file_path : None | str
Path to a pickled file. If None, a system file dialog will be used. If
the user cancels the file dialog, a RuntimeError is raised.
"""
if file_path is None:
filetypes = [("Pickles (*.pickled)", '*.pickled'), ("All files", '*')]
file_path = ui.ask_file("Select File to Unpickle", "Select a pickled "
"file to unpickle", filetypes)
if file_path is False:
raise RuntimeError("User canceled")
if not os.path.exists(file_path):
new_path = os.extsep.join((file_path, 'pickled'))
if os.path.exists(new_path):
file_path = new_path
with open(file_path, 'r') as fid:
unpickler = Unpickler(fid)
unpickler.find_global = map_paths
obj = unpickler.load()
return obj
def load_state(self, state):
"""Load an image_set_list's state from the string returned from save_state"""
self.__image_sets = []
self.__image_sets_by_key = {}
# Make a safe unpickler
p = Unpickler(StringIO(state))
def find_global(module_name, class_name):
logger.debug("Pickler wants %s:%s", module_name, class_name)
if module_name not in ("numpy", "numpy.core.multiarray"):
logger.critical(
"WARNING WARNING WARNING - your batch file has asked to load %s.%s."
" If this looks in any way suspicious please contact us at www.cellprofiler.org",
module_name,
class_name,
)
raise ValueError("Illegal attempt to unpickle class %s.%s", (module_name, class_name))
__import__(module_name)
mod = sys.modules[module_name]
return getattr(mod, class_name)
p.find_global = find_global
count = p.load()
all_keys = [p.load() for i in range(count)]
self.__legacy_fields = p.load()
#
# Have to do in this order in order for the image set's
# legacy_fields property to hook to the right legacy_fields
#
for i in range(count):
self.get_image_set(all_keys[i])
def unpickle(file_path=None):
"""Load pickled Python objects from a file.
Almost like ``cPickle.load(open(file_path))``, but also loads object saved
with older versions of Eelbrain, and allows using a system file dialog to
select a file.
Parameters
----------
file_path : None | str
Path to a pickled file. If None (default), a system file dialog will be
shown. If the user cancels the file dialog, a RuntimeError is raised.
"""
if file_path is None:
filetypes = [("Pickles (*.pickled)", '*.pickled'), ("All files", '*')]
file_path = ui.ask_file("Select File to Unpickle", "Select a pickled "
"file to unpickle", filetypes)
if file_path is False:
raise RuntimeError("User canceled")
else:
print repr(file_path)
else:
file_path = os.path.expanduser(file_path)
if not os.path.exists(file_path):
new_path = os.extsep.join((file_path, 'pickled'))
if os.path.exists(new_path):
file_path = new_path
with open(file_path, 'r') as fid:
unpickler = Unpickler(fid)
unpickler.find_global = map_paths
obj = unpickler.load()
return obj
def _unpickle(pickled):
""" Unpickles a string and catch all types of errors it can throw,
to raise only NotReadableJobError in case of error.
OpenERP stores the text fields as 'utf-8', so we specify the encoding.
`loads()` may raises many types of exceptions (AttributeError,
IndexError, TypeError, KeyError, ...). They are all catched and
raised as `NotReadableJobError`).
Pickle could be exploited by an attacker who would write a value in a job
that would run arbitrary code when unpickled. This is why we set a custom
``find_global`` method on the ``Unpickler``, only jobs and a whitelist of
classes/functions are allowed to be unpickled (plus the builtins types).
"""
def restricted_find_global(mod_name, fn_name):
__import__(mod_name)
mod = sys.modules[mod_name]
fn = getattr(mod, fn_name)
if not (fn in JOB_REGISTRY or fn in _UNPICKLE_WHITELIST):
raise UnpicklingError(
'{}.{} is not allowed in jobs'.format(mod_name, fn_name)
)
return fn
unpickler = Unpickler(StringIO(pickled))
unpickler.find_global = restricted_find_global
try:
unpickled = unpickler.load()
except (StandardError, UnpicklingError):
raise NotReadableJobError('Could not unpickle.', pickled)
return unpickled
def got_data(self, port_agent_packet):
"""
Called by the instrument connection when data is available.
Also add data to the chunker and when received call got_chunk
to publish results.
"""
data_length = port_agent_packet.get_data_length()
data = port_agent_packet.get_data()
timestamp = port_agent_packet.get_timestamp()
log.debug("Got Data: %s" % data)
log.debug("Add Port Agent Timestamp: %s" % timestamp)
unpickler = Unpickler(StringIO(data))
# Disable class unpickling, for security; record should be all
# built-in types. Note this only works with cPickle.
unpickler.find_global = None
# pkt is an antelope.Pkt.Packet object converted to a dict. Refer to
# the documentation for the Antelope Python bindings for compelete
# details.
pkt = unpickler.load()
for particle in self._particle_factory(pkt, timestamp):
self._publish_particle(particle)
def load_state(self, state):
'''Load an image_set_list's state from the string returned from save_state'''
self.__image_sets = []
self.__image_sets_by_key = {}
# Make a safe unpickler
p = Unpickler(StringIO(state))
def find_global(module_name, class_name):
logger.debug("Pickler wants %s:%s",module_name, class_name)
if (module_name not in ("numpy", "numpy.core.multiarray")):
logger.critical(
"WARNING WARNING WARNING - your batch file has asked to load %s.%s."
" If this looks in any way suspicious please contact us at www.cellprofiler.org",
module_name, class_name)
raise ValueError("Illegal attempt to unpickle class %s.%s",
(module_name, class_name))
__import__(module_name)
mod = sys.modules[module_name]
return getattr(mod, class_name)
p.find_global = find_global
count = p.load()
all_keys = [p.load() for i in range(count)]
self.__legacy_fields = p.load()
#
# Have to do in this order in order for the image set's
# legacy_fields property to hook to the right legacy_fields
#
for i in range(count):
self.get_image_set(all_keys[i])
def tryToResolveConflict(self,
oid,
committedSerial,
oldSerial,
newpickle,
committedData=''):
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
try:
prfactory = PersistentReferenceFactory()
newpickle = self._crs_untransform_record_data(newpickle)
file = StringIO(newpickle)
unpickler = Unpickler(file)
unpickler.find_global = find_global
unpickler.persistent_load = prfactory.persistent_load
meta = unpickler.load()
if isinstance(meta, tuple):
klass = meta[0]
newargs = meta[1] or ()
if isinstance(klass, tuple):
klass = find_global(*klass)
else:
klass = meta
newargs = ()
if klass in _unresolvable:
raise ConflictError
newstate = unpickler.load()
inst = klass.__new__(klass, *newargs)
try:
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
raise ConflictError
old = state(self, oid, oldSerial, prfactory)
committed = state(self, oid, committedSerial, prfactory, committedData)
resolved = resolve(old, committed, newstate)
file = StringIO()
pickler = Pickler(file, 1)
pickler.inst_persistent_id = persistent_id
pickler.dump(meta)
pickler.dump(resolved)
return self._crs_transform_record_data(file.getvalue(1))
except (ConflictError, BadClassName):
pass
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
# the original ConflictError. A client can recover from a
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.error("Unexpected error", exc_info=True)
raise ConflictError(oid=oid,
serials=(committedSerial, oldSerial),
data=newpickle)
def got_data(self, port_agent_packet):
"""
Called by the instrument connection when data is available.
Also add data to the chunker and when received call got_chunk
to publish results.
"""
data_length = port_agent_packet.get_data_length()
data = port_agent_packet.get_data()
timestamp = port_agent_packet.get_timestamp()
log.debug("Got Data: %s" % data)
log.debug("Add Port Agent Timestamp: %s" % timestamp)
unpickler = Unpickler(StringIO(data))
# Disable class unpickling, for security; record should be all
# built-in types. Note this only works with cPickle.
unpickler.find_global = None
# pkt is an antelope.Pkt.Packet object converted to a dict. Refer to
# the documentation for the Antelope Python bindings for compelete
# details.
pkt = unpickler.load()
for particle in self._particle_factory(pkt, timestamp):
self._publish_particle(particle)
def state(self, oid, serial, prfactory, p=''):
p = p or self.loadSerial(oid, serial)
file = StringIO(p)
unpickler = Unpickler(file)
unpickler.find_global = find_global
unpickler.persistent_load = prfactory.persistent_load
unpickler.load() # skip the class tuple
return unpickler.load()
def state(self, oid, serial, prfactory, p=''):
p = p or self.loadSerial(oid, serial)
file = StringIO(p)
unpickler = Unpickler(file)
unpickler.find_global = find_global
unpickler.persistent_load = prfactory.persistent_load
unpickler.load() # skip the class tuple
return unpickler.load()
def state(self, oid, serial, prfactory, p=""):
p = p or self.loadSerial(oid, serial)
p = self._crs_untransform_record_data(p)
file = StringIO(p)
unpickler = Unpickler(file)
unpickler.find_global = find_global
unpickler.persistent_load = prfactory.persistent_load
unpickler.load() # skip the class tuple
return unpickler.load()
def server_decode(msg):
"""Decodes msg and returns its parts"""
unpickler = Unpickler(StringIO(msg))
unpickler.find_global = server_find_global
try:
return unpickler.load() # msgid, flags, name, args
except:
log("can't decode message: %s" % short_repr(msg), level=logging.ERROR)
raise
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
committedData=''):
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
try:
prfactory = PersistentReferenceFactory()
newpickle = self._crs_untransform_record_data(newpickle)
file = StringIO(newpickle)
unpickler = Unpickler(file)
unpickler.find_global = find_global
unpickler.persistent_load = prfactory.persistent_load
meta = unpickler.load()
if isinstance(meta, tuple):
klass = meta[0]
newargs = meta[1] or ()
if isinstance(klass, tuple):
klass = find_global(*klass)
else:
klass = meta
newargs = ()
if klass in _unresolvable:
raise ConflictError
newstate = unpickler.load()
inst = klass.__new__(klass, *newargs)
try:
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
raise ConflictError
old = state(self, oid, oldSerial, prfactory)
committed = state(self, oid, committedSerial, prfactory, committedData)
resolved = resolve(old, committed, newstate)
file = StringIO()
pickler = Pickler(file,1)
pickler.inst_persistent_id = persistent_id
pickler.dump(meta)
pickler.dump(resolved)
return self._crs_transform_record_data(file.getvalue(1))
except (ConflictError, BadClassName):
pass
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
# the original ConflictError. A client can recover from a
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.error("Unexpected error", exc_info=True)
raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
data=newpickle)
def _cpickle_loads(pkl_str):
"""A replacement for cPickle.loads that overrides the find_global
attribute (Or find_class in Python3)
"""
pk = Unpickler(BytesIO(pkl_str))
if hasattr(pk, 'find_class'):
# This is the method name in Python3
pk.find_class = _find_global
else:
# hasattr doe not really work on the cPickle objects in Python2, so we
# just assume we're in python2 if hasattr returned false
pk.find_global = _find_global
return pk.load()
def unpickle_old_pyfits(fn):
"""
This function unpickles everything in the specified filename, and
correctly adapts pyfits files pre 2.3 to the new module structure.
"""
try:
import cPickle as pickle
from cPickle import Unpickler
except:
import pickle
from pickle import Unpickler
import imp, sys
def fg(modname, classname):
if 'NP_pyfits' in modname:
modname = modname.replace('NP_pyfits', 'core')
if '.' in modname:
mod = __import__(modname, fromlist=[0])
else:
mod = __import__(modname)
return getattr(mod, classname)
objs = []
if isinstance(fn, str):
f = open(fn)
else:
f = fn
try:
u = Unpickler(f)
u.find_global = fg
while True:
objs.append(u.load())
except EOFError:
pass
finally:
if isinstance(fn, str):
f.close()
if len(objs) == 1:
return objs[0]
else:
return objs
def load(self, file):
"""Unpickle a file."""
safeUnpickler = Unpickler(file)
safeUnpickler.find_global = self.findGlobal
return safeUnpickler.load()
def load(self, file):
"""Unpickle a file."""
safeUnpickler = Unpickler(file)
safeUnpickler.find_global = self.findGlobal
return safeUnpickler.load()
def import_during_commit(self, transaction, f, return_oid_list):
"""Import data during two-phase commit.
Invoked by the transaction manager mid commit.
Appends one item, the OID of the first object created,
to return_oid_list.
"""
oids = {}
# IMPORTANT: This code should be consistent with the code in
# serialize.py. It is currently out of date and doesn't handle
# weak references.
def persistent_load(ooid):
"""Remap a persistent id to a new ID and create a ghost for it."""
klass = None
if isinstance(ooid, tuple):
ooid, klass = ooid
if ooid in oids:
oid = oids[ooid]
else:
if klass is None:
oid = self._storage.new_oid()
else:
oid = self._storage.new_oid(), klass
oids[ooid] = oid
return Ghost(oid)
while 1:
header = f.read(16)
if header == export_end_marker:
break
if len(header) != 16:
raise ExportError("Truncated export file")
# Extract header information
ooid = header[:8]
length = u64(header[8:16])
data = f.read(length)
if len(data) != length:
raise ExportError("Truncated export file")
if oids:
oid = oids[ooid]
if isinstance(oid, tuple):
oid = oid[0]
else:
oids[ooid] = oid = self._storage.new_oid()
return_oid_list.append(oid)
# Blob support
blob_begin = f.read(len(blob_begin_marker))
if blob_begin == blob_begin_marker:
# Copy the blob data to a temporary file
# and remember the name
blob_len = u64(f.read(8))
blob_filename = mktemp()
blob_file = open(blob_filename, "wb")
cp(f, blob_file, blob_len)
blob_file.close()
else:
f.seek(-len(blob_begin_marker),1)
blob_filename = None
pfile = StringIO(data)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
unpickler.find_global = find_broken_global
newp = StringIO()
pickler = Pickler(newp, 1)
pickler.inst_persistent_id = persistent_id
pickler.dump(unpickler.load())
pickler.dump(unpickler.load())
data = newp.getvalue()
if blob_filename is not None:
self._storage.storeBlob(oid, None, data, blob_filename,
'', transaction)
else:
self._storage.store(oid, None, data, '', transaction)
