def get_refs(a_pickle):
"""Return oid and class information for references in a pickle
The result of a list of oid and class information tuples.
If the reference doesn't contain class information, then the
klass information is None.
"""
refs = []
u = PersistentUnpickler(None, refs.append, BytesIO(a_pickle))
u.noload()
u.noload()
# Now we have a list of references. Need to convert to list of
# oids and class info:
result = []
for reference in refs:
if isinstance(reference, tuple):
oid, klass = reference
elif isinstance(reference, (bytes, str)):
data, klass = reference, None
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode('ascii')
result.append((oid, klass))
return result
def get_refs(a_pickle):
"""Return oid and class information for references in a pickle
The result of a list of oid and class information tuples.
If the reference doesn't contain class information, then the
klass information is None.
"""
refs = []
u = PersistentUnpickler(None, refs.append, BytesIO(a_pickle))
u.noload()
u.noload()
# Now we have a list of references. Need to convert to list of
# oids and class info:
result = []
for reference in refs:
if isinstance(reference, tuple):
oid, klass = reference
elif isinstance(reference, (bytes, str)):
data, klass = reference, None
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode('ascii')
result.append((oid, klass))
return result
def state(self, oid, serial, prfactory, p=''):
p = p or self.loadSerial(oid, serial)
p = self._crs_untransform_record_data(p)
file = BytesIO(p)
unpickler = PersistentUnpickler(find_global, prfactory.persistent_load, file)
unpickler.load() # skip the class tuple
return unpickler.load()
def state(self, oid, serial, prfactory, p=''):
p = p or self.loadSerial(oid, serial)
p = self._crs_untransform_record_data(p)
file = BytesIO(p)
unpickler = PersistentUnpickler(find_global, prfactory.persistent_load,
file)
unpickler.load() # skip the class tuple
return unpickler.load()
def is_blob_record(record):
"""Check whether a database record is a blob record.
This is primarily intended to be used when copying data from one
storage to another.
"""
if record and (b'ZODB.blob' in record):
unpickler = PersistentUnpickler(find_global_Blob, None, BytesIO(record))
try:
return unpickler.load() is Blob
except (MemoryError, KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
return False
def is_blob_record(record):
"""Check whether a database record is a blob record.
This is primarily intended to be used when copying data from one
storage to another.
"""
if record and (b'ZODB.blob' in record):
unpickler = PersistentUnpickler(find_global_Blob, None, BytesIO(record))
try:
return unpickler.load() is Blob
except (MemoryError, KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
return False
def test_persistent_id_noload(self):
# make sure we can noload weak references and other list-based
# references like we expect. Protect explicitly against the
# breakage in CPython 2.7 and zodbpickle < 0.6.0
o = PersistentObject()
o._p_oid = b'abcd'
top = PersistentObject()
top._p_oid = b'efgh'
top.ref = WeakRef(o)
pickle = serialize.ObjectWriter().serialize(top)
refs = []
u = PersistentUnpickler(None, refs.append, BytesIO(pickle))
u.noload()
u.noload()
self.assertEqual(refs, [['w', (b'abcd', )]])
def test_persistent_id_noload(self):
# make sure we can noload weak references and other list-based
# references like we expect. Protect explicitly against the
# breakage in CPython 2.7 and zodbpickle < 0.6.0
o = PersistentObject()
o._p_oid = b'abcd'
top = PersistentObject()
top._p_oid = b'efgh'
top.ref = WeakRef(o)
pickle = serialize.ObjectWriter().serialize(top)
refs = []
u = PersistentUnpickler(None, refs.append, BytesIO(pickle))
u.noload()
u.noload()
self.assertEqual(refs, [['w', (b'abcd',)]])
def _get_unpickler(self, pickle):
file = BytesIO(pickle)
factory = self._factory
conn = self._conn
def find_global(modulename, name):
return factory(conn, modulename, name)
unpickler = PersistentUnpickler(find_global, self._persistent_load, file)
return unpickler
def referencesf(p, oids=None):
"""Return a list of object ids found in a pickle
A list may be passed in, in which case, information is
appended to it.
Only ordinary internal references are included.
Weak and multi-database references are not included.
"""
refs = []
u = PersistentUnpickler(None, refs.append, BytesIO(p))
u.noload()
u.noload()
# Now we have a list of referencs. Need to convert to list of
# oids:
if oids is None:
oids = []
for reference in refs:
if isinstance(reference, tuple):
oid = reference[0]
elif isinstance(reference, (bytes, str)):
oid = reference
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode('ascii')
oids.append(oid)
return oids
def referencesf(p, oids=None):
"""Return a list of object ids found in a pickle
A list may be passed in, in which case, information is
appended to it.
Only ordinary internal references are included.
Weak and multi-database references are not included.
"""
refs = []
u = PersistentUnpickler(None, refs.append, BytesIO(p))
u.noload()
u.noload()
# Now we have a list of referencs. Need to convert to list of
# oids:
if oids is None:
oids = []
for reference in refs:
if isinstance(reference, tuple):
oid = reference[0]
elif isinstance(reference, (bytes, str)):
oid = reference
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode('ascii')
oids.append(oid)
return oids
def tryToResolveConflict(self,
oid,
committedSerial,
oldSerial,
newpickle,
committedData=b''):
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
klass = 'n/a'
try:
prfactory = PersistentReferenceFactory()
newpickle = self._crs_untransform_record_data(newpickle)
file = BytesIO(newpickle)
unpickler = PersistentUnpickler(find_global, prfactory.persistent_load,
file)
meta = unpickler.load()
if isinstance(meta, tuple):
klass = meta[0]
newargs = meta[1] or ()
if isinstance(klass, tuple):
klass = find_global(*klass)
else:
klass = meta
newargs = ()
if klass in _unresolvable:
raise ConflictError
inst = klass.__new__(klass, *newargs)
try:
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
raise ConflictError
oldData = self.loadSerial(oid, oldSerial)
if not committedData:
committedData = self.loadSerial(oid, committedSerial)
newstate = unpickler.load()
old = state(self, oid, oldSerial, prfactory, oldData)
committed = state(self, oid, committedSerial, prfactory, committedData)
resolved = resolve(old, committed, newstate)
file = BytesIO()
pickler = PersistentPickler(persistent_id, file, _protocol)
pickler.dump(meta)
pickler.dump(resolved)
return self._crs_transform_record_data(file.getvalue())
except (ConflictError, BadClassName) as e:
logger.debug("Conflict resolution on %s failed with %s: %s", klass,
e.__class__.__name__, str(e))
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
# the original ConflictError. A client can recover from a
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.exception(
"Unexpected error while trying to resolve conflict on %s", klass)
raise ConflictError(oid=oid,
serials=(committedSerial, oldSerial),
data=newpickle)
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
committedData=b''):
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
klass = 'n/a'
try:
prfactory = PersistentReferenceFactory()
newpickle = self._crs_untransform_record_data(newpickle)
file = BytesIO(newpickle)
unpickler = PersistentUnpickler(
find_global, prfactory.persistent_load, file)
meta = unpickler.load()
if isinstance(meta, tuple):
klass = meta[0]
newargs = meta[1] or ()
if isinstance(klass, tuple):
klass = find_global(*klass)
else:
klass = meta
newargs = ()
if klass in _unresolvable:
raise ConflictError
inst = klass.__new__(klass, *newargs)
try:
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
raise ConflictError
oldData = self.loadSerial(oid, oldSerial)
if not committedData:
committedData = self.loadSerial(oid, committedSerial)
newstate = unpickler.load()
old = state(self, oid, oldSerial, prfactory, oldData)
committed = state(self, oid, committedSerial, prfactory, committedData)
resolved = resolve(old, committed, newstate)
file = BytesIO()
pickler = PersistentPickler(persistent_id, file, _protocol)
pickler.dump(meta)
pickler.dump(resolved)
return self._crs_transform_record_data(file.getvalue())
except (ConflictError, BadClassName) as e:
logger.debug(
"Conflict resolution on %s failed with %s: %s",
klass, e.__class__.__name__, str(e))
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
# the original ConflictError. A client can recover from a
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.exception(
"Unexpected error while trying to resolve conflict on %s", klass)
raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
data=newpickle)
def FakeUnpickler(f):
unpickler = PersistentUnpickler(fake_find_class, None, f)
return unpickler
