def checkConflictErrorDoesntImport(self):
from ZODB.serialize import ObjectWriter
from ZODB.POSException import ConflictError
from ZODB.tests.MinPO import MinPO
import cPickle as pickle
obj = MinPO()
data = ObjectWriter().serialize(obj)
# The pickle contains a GLOBAL ('c') opcode resolving to MinPO's
# module and class.
self.assert_('cZODB.tests.MinPO\nMinPO\n' in data)
# Fiddle the pickle so it points to something "impossible" instead.
data = data.replace('cZODB.tests.MinPO\nMinPO\n',
'cpath.that.does.not.exist\nlikewise.the.class\n')
# Pickle can't resolve that GLOBAL opcode -- gets ImportError.
self.assertRaises(ImportError, pickle.loads, data)
# Verify that building ConflictError doesn't get ImportError.
try:
raise ConflictError(object=obj, data=data)
except ConflictError, detail:
# And verify that the msg names the impossible path.
self.assert_('path.that.does.not.exist.likewise.the.class' in
str(detail))
def test_ConflictErrorDoesntImport(self):
from ZODB.serialize import ObjectWriter
from ZODB.POSException import ConflictError
from ZODB.tests.MinPO import MinPO
obj = MinPO()
data = ObjectWriter().serialize(obj)
# The pickle contains a GLOBAL ('c') opcode resolving to MinPO's
# module and class.
self.assertTrue(b'cZODB.tests.MinPO\nMinPO\n' in data)
# Fiddle the pickle so it points to something "impossible" instead.
data = data.replace(
b'cZODB.tests.MinPO\nMinPO\n',
b'cpath.that.does.not.exist\nlikewise.the.class\n')
# Pickle can't resolve that GLOBAL opcode -- gets ImportError.
self.assertRaises(ImportError, loads, data)
# Verify that building ConflictError doesn't get ImportError.
try:
raise ConflictError(object=obj, data=data)
except ConflictError as detail:
# And verify that the msg names the impossible path.
self.assertTrue(
'path.that.does.not.exist.likewise.the.class' in str(detail))
else:
self.fail("expected ConflictError, but no exception raised")
def checkPersistentIdHandlesDescriptor(self):
from ZODB.serialize import ObjectWriter
class P(Persistent):
pass
writer = ObjectWriter(None)
self.assertEqual(writer.persistent_id(P), None)
def test_PersistentIdHandlesDescriptor(self):
from ZODB.serialize import ObjectWriter
class P(Persistent):
pass
writer = ObjectWriter(None)
self.assertEqual(writer.persistent_id(P), None)
def canSerialize(obj):
result = False
container_type_tuple = (list, tuple, dict, set, frozenset)
# if object is a container, we need to check its elements for presence of
# objects that cannot be put inside the zodb
if isinstance(obj, container_type_tuple):
if isinstance(obj, dict):
result_list = []
for key, value in obj.iteritems():
result_list.append(canSerialize(key))
result_list.append(canSerialize(value))
else:
result_list = [canSerialize(element) for element in obj]
return all(result_list)
# if obj is an object and implements __getstate__, ZODB.serialize can check
# if we can store it
elif isinstance(obj, object) and hasattr(obj, '__getstate__'):
# Need to unwrap the variable, otherwise we get a TypeError, because
# objects cannot be pickled while inside an acquisition wrapper.
unwrapped_obj = Acquisition.aq_base(obj)
writer = ObjectWriter(unwrapped_obj)
for obj in writer:
try:
writer.serialize(obj)
# Because writer.serialize(obj) relies on the implementation of __getstate__
# of obj, all errors can happen, so the "except all" is necessary here.
except:
return False
return True
else:
# If cannot serialize object with ZODB.serialize, try with cPickle
# Only a dump of the object is not enough. Dumping and trying to
# load it will properly raise errors in all possible situations,
# for example: if the user defines a dict with an object of a class
# that he created the dump will stil work, but the load will fail.
try:
cPickle.loads(cPickle.dumps(obj))
# By unknowing reasons, trying to catch cPickle.PicklingError in the "normal"
# way isn't working. This issue might be related to some weirdness in
# pickle/cPickle that is reported in this issue: http://bugs.python.org/issue1457119.
#
# So, as a temporary fix, we're investigating the exception's class name as
# string to be able to identify them.
#
# Even though the issue seems complicated, this quickfix should be
# properly rewritten in a better way as soon as possible.
except Exception as e:
if type(e).__name__ in ('PicklingError', 'TypeError', 'NameError', 'AttributeError'):
return False
else:
raise e
else:
return True
def canSerialize(obj):
result = False
container_type_tuple = (list, tuple, dict, set, frozenset)
# if object is a container, we need to check its elements for presence of
# objects that cannot be put inside the zodb
if isinstance(obj, container_type_tuple):
if isinstance(obj, dict):
result_list = []
for key, value in obj.iteritems():
result_list.append(canSerialize(key))
result_list.append(canSerialize(value))
else:
result_list = [canSerialize(element) for element in obj]
return all(result_list)
# if obj is an object and implements __getstate__, ZODB.serialize can check
# if we can store it
elif isinstance(obj, object) and hasattr(obj, '__getstate__') and hasattr(
obj, '_p_jar'):
# Need to unwrap the variable, otherwise we get a TypeError, because
# objects cannot be pickled while inside an acquisition wrapper.
unwrapped_obj = Acquisition.aq_base(obj)
writer = ObjectWriter(unwrapped_obj)
for obj in writer:
try:
writer.serialize(obj)
# Because writer.serialize(obj) relies on the implementation of __getstate__
# of obj, all errors can happen, so the "except all" is necessary here.
except:
return False
return True
else:
# If cannot serialize object with ZODB.serialize, try with cPickle
# Only a dump of the object is not enough. Dumping and trying to
# load it will properly raise errors in all possible situations,
# for example: if the user defines a dict with an object of a class
# that he created the dump will stil work, but the load will fail.
try:
cPickle.loads(cPickle.dumps(obj))
# By unknowing reasons, trying to catch cPickle.PicklingError in the "normal"
# way isn't working. This issue might be related to some weirdness in
# pickle/cPickle that is reported in this issue: http://bugs.python.org/issue1457119.
#
# So, as a temporary fix, we're investigating the exception's class name as
# string to be able to identify them.
#
# Even though the issue seems complicated, this quickfix should be
# properly rewritten in a better way as soon as possible.
except (cPickle.PicklingError, TypeError, NameError,
AttributeError) as e:
return False
else:
return True
def __new__(cls, obj, recursive=False):
self = object.__new__(cls)
obj = aq_base(obj)
connection = obj._p_jar
ObjectReader.__init__(self, connection, connection._cache,
connection._db.classFactory)
ObjectWriter.__init__(self, obj)
migrated_oid_set = set()
oid_set = {obj._p_oid}
while oid_set:
oid = oid_set.pop()
obj = ObjectReader.load_oid(self, oid)
obj._p_activate()
klass = obj.__class__
self.lazy = None
if not recursive:
_setOb = getattr(klass, '_setOb', None)
if _setOb:
if isinstance(_setOb, WorkflowMethod):
_setOb = _setOb._m
import six
if six.get_unbound_function(
_setOb) is six.get_unbound_function(
OFS_Folder._setOb):
self.lazy = Ghost
elif klass.__module__[:
7] == 'BTrees.' and klass.__name__ != 'Length':
self.lazy = LazyBTree()
self.oid_dict = {}
self.oid_set = set()
p, serial = self._conn._storage.load(oid, '')
unpickler = self._get_unpickler(p)
def find_global(*args):
self.do_migrate = args != (klass.__module__, klass.__name__) and \
not isOldBTree('%s.%s' % args)
unpickler.find_global = self._get_class
return self._get_class(*args)
unpickler.find_global = find_global
unpickler.load() # class
state = unpickler.load()
if isinstance(self.lazy, LazyPersistent):
self.oid_set.update(self.lazy.getOidList(state))
migrated_oid_set.add(oid)
oid_set |= self.oid_set - migrated_oid_set
self.oid_set = None
if self.do_migrate:
log.debug('PickleUpdater: migrate %r (%r)', obj, klass)
self.setGhostState(obj, self.serialize(obj))
obj._p_changed = 1
def _commit(self, transaction):
"""Commit changes to an object"""
if self._import:
# We are importing an export file. We alsways do this
# while making a savepoint so we can copy export data
# directly to our storage, typically a TmpStore.
self._importDuringCommit(transaction, *self._import)
self._import = None
# Just in case an object is added as a side-effect of storing
# a modified object. If, for example, a __getstate__() method
# calls add(), the newly added objects will show up in
# _added_during_commit. This sounds insane, but has actually
# happened.
self._added_during_commit = []
if self._invalidatedCache:
raise ConflictError()
for obj in self._registered_objects:
oid = obj._p_oid
assert oid
if oid in self._conflicts:
raise ReadConflictError(object=obj)
if obj._p_jar is not self:
raise InvalidObjectReference(obj, obj._p_jar)
elif oid in self._added:
assert obj._p_serial == z64
elif obj._p_changed:
if oid in self._invalidated:
resolve = getattr(obj, "_p_resolveConflict", None)
if resolve is None:
raise ConflictError(object=obj)
self._modified.append(oid)
else:
# Nothing to do. It's been said that it's legal, e.g., for
# an object to set _p_changed to false after it's been
# changed and registered.
continue
self._store_objects(ObjectWriter(obj), transaction)
for obj in self._added_during_commit:
self._store_objects(ObjectWriter(obj), transaction)
self._added_during_commit = None
def __new__(cls, obj, recursive=False):
assert cls.get, "Persistent migration of pickle requires ZODB >= 3.5"
self = object.__new__(cls)
obj = aq_base(obj)
connection = obj._p_jar
ObjectReader.__init__(self, connection, connection._cache, connection._db.classFactory)
ObjectWriter.__init__(self, obj)
migrated_oid_set = set()
oid_set = set((obj._p_oid,))
while oid_set:
oid = oid_set.pop()
obj = self.get(oid)
obj._p_activate()
klass = obj.__class__
self.lazy = None
if not recursive:
_setOb = getattr(klass, "_setOb", None)
if _setOb:
if isinstance(_setOb, WorkflowMethod):
_setOb = _setOb._m
if _setOb.im_func is OFS_Folder._setOb.im_func:
self.lazy = Ghost
elif klass.__module__[:7] == "BTrees." and klass.__name__ != "Length":
self.lazy = LazyBTree()
self.oid_dict = {}
self.oid_set = set()
p, serial = self._conn._storage.load(oid, "")
unpickler = self._get_unpickler(p)
def find_global(*args):
self.do_migrate = args != (klass.__module__, klass.__name__) and not isOldBTree("%s.%s" % args)
unpickler.find_global = self._get_class
return self._get_class(*args)
unpickler.find_global = find_global
unpickler.load() # class
state = unpickler.load()
if isinstance(self.lazy, LazyPersistent):
self.oid_set.update(self.lazy.getOidList(state))
migrated_oid_set.add(oid)
oid_set |= self.oid_set - migrated_oid_set
self.oid_set = None
if self.do_migrate:
log.debug("PickleUpdater: migrate %r (%r)", obj, klass)
self.setGhostState(obj, self.serialize(obj))
obj._p_changed = 1
def persistent_id(self, obj):
assert type(obj) is not Ghost
oid = self.getOid(obj)
if type(oid) is str:
try:
return self.oid_dict[oid]
except KeyError:
obj._p_activate()
return ObjectWriter.persistent_id(self, obj)
def persistent_id(self, obj):
assert type(obj) is not Ghost
oid = self.getOid(obj)
if type(oid) is str:
try:
return self.oid_dict[oid]
except KeyError:
obj._p_activate()
return ObjectWriter.persistent_id(self, obj)
def load_persistent(self, oid, klass):
obj = ObjectReader.load_persistent(self, oid, klass)
if self.oid_set is not None:
if not self.lazy:
self.oid_set.add(oid)
obj._p_activate()
self.oid_dict[oid] = oid_klass = ObjectWriter.persistent_id(self, obj)
if oid_klass != (oid, klass):
self.do_migrate = True
return obj
def register(self, obj):
"""
Serialize early to inspect PicklingErrors
Raise any PicklingErrors when the object is added to the
transaction as opposed to when the transaction is committed.
Under pdb, for example, this allows inspecting the code that made
the change resulting in the PicklingError.
Requires either zope.testrunner or zope.testing which can be included using
the 'zodb' or 'zodb-testing' extras respectively.
"""
orig_register(self, obj)
writer = ObjectWriter(obj)
# Replace the pickler so that it doesn't set oids
import cPickle as pickle
writer._p = pickle.Pickler(writer._file, 1)
# Try to serialize to raise piclkling errors early
writer.serialize(obj)
