def testDeepCopyCanInvalidate(self):
"""
Tests regression for invalidation problems related to missing
readers and writers values in cloned objects (see
http://mail.zope.org/pipermail/zodb-dev/2008-August/012054.html)
"""
import ZODB.MappingStorage
database = DB(ZODB.blob.BlobStorage(
'blobs', ZODB.MappingStorage.MappingStorage()))
connection = database.open()
root = connection.root()
transaction.begin()
root['blob'] = Blob()
transaction.commit()
stream = BytesIO()
p = Pickler(stream, _protocol)
p.dump(root['blob'])
u = Unpickler(stream)
stream.seek(0)
clone = u.load()
clone._p_invalidate()
# it should also be possible to open the cloned blob
# (even though it won't contain the original data)
clone.open().close()
# tearDown
database.close()
def testDeepCopyCanInvalidate(self):
"""
Tests regression for invalidation problems related to missing
readers and writers values in cloned objects (see
http://mail.zope.org/pipermail/zodb-dev/2008-August/012054.html)
"""
import ZODB.MappingStorage
database = DB(ZODB.blob.BlobStorage(
'blobs', ZODB.MappingStorage.MappingStorage()))
connection = database.open()
root = connection.root()
transaction.begin()
root['blob'] = Blob()
transaction.commit()
stream = BytesIO()
p = Pickler(stream, _protocol)
p.dump(root['blob'])
u = Unpickler(stream)
stream.seek(0)
clone = u.load()
clone._p_invalidate()
# it should also be possible to open the cloned blob
# (even though it won't contain the original data)
clone.open().close()
# tearDown
database.close()
def cloneByPickle(obj, ignore_list=()):
"""Makes a copy of a ZODB object, loading ghosts as needed.
Ignores specified objects along the way, replacing them with None
in the copy.
"""
ignore_dict = {}
for o in ignore_list:
ignore_dict[id(o)] = o
def persistent_id(ob, ignore_dict=ignore_dict):
if id(ob) in ignore_dict:
return 'ignored'
if getattr(ob, '_p_changed', 0) is None:
ob._p_changed = 0
return None
def persistent_load(ref):
assert ref == 'ignored'
# Return a placeholder object that will be replaced by
# removeNonVersionedData().
placeholder = SimpleItem()
placeholder.id = "ignored_subobject"
return placeholder
stream = BytesIO()
p = Pickler(stream, 1)
p.persistent_id = persistent_id
p.dump(obj)
stream.seek(0)
u = Unpickler(stream)
u.persistent_load = persistent_load
return u.load()
def zodb_unpickle(data):
"""Unpickle an object stored using the format expected by ZODB."""
f = BytesIO(data)
u = Unpickler(f)
u.persistent_load = persistent_load
klass_info = u.load()
if isinstance(klass_info, tuple):
if isinstance(klass_info[0], type):
# Unclear: what is the second part of klass_info?
klass, xxx = klass_info
assert not xxx
else:
if isinstance(klass_info[0], tuple):
modname, klassname = klass_info[0]
else:
modname, klassname = klass_info
if modname == "__main__":
ns = globals()
else:
mod = import_helper(modname)
ns = mod.__dict__
try:
klass = ns[klassname]
except KeyError:
print("can't find %s in %r" % (klassname, ns), file=sys.stderr)
inst = klass()
else:
raise ValueError("expected class info: %s" % repr(klass_info))
state = u.load()
inst.__setstate__(state)
return inst
def cloneByPickle(obj, ignore_list=()):
"""Makes a copy of a ZODB object, loading ghosts as needed.
Ignores specified objects along the way, replacing them with None
in the copy.
"""
ignore_dict = {}
for o in ignore_list:
ignore_dict[id(o)] = o
def persistent_id(ob, ignore_dict=ignore_dict):
if id(ob) in ignore_dict:
return 'ignored'
if getattr(ob, '_p_changed', 0) is None:
ob._p_changed = 0
return None
def persistent_load(ref):
assert ref == 'ignored'
# Return a placeholder object that will be replaced by
# removeNonVersionedData().
placeholder = SimpleItem()
placeholder.id = "ignored_subobject"
return placeholder
stream = BytesIO()
p = Pickler(stream, 1)
p.persistent_id = persistent_id
p.dump(obj)
stream.seek(0)
u = Unpickler(stream)
u.persistent_load = persistent_load
return u.load()
def _get_unpickler(self, pickle):
file = BytesIO(pickle)
unpickler = Unpickler(file)
unpickler.persistent_load = self._persistent_load
factory = self._factory
conn = self._conn
def find_global(modulename, name):
return factory(conn, modulename, name)
unpickler.find_global = find_global
return unpickler
def _get_unpickler(self, pickle):
file = BytesIO(pickle)
unpickler = Unpickler(file)
unpickler.persistent_load = self._persistent_load
factory = self._factory
conn = self._conn
def find_global(modulename, name):
return factory(conn, modulename, name)
unpickler.find_global = find_global
return unpickler
def get_pickle_metadata(data):
# Returns a 2-tuple of strings.
# ZODB's data records contain two pickles. The first is the class
# of the object, the second is the object. We're only trying to
# pick apart the first here, to extract the module and class names.
if data[0] in (
0x80, # Py3k indexes bytes -> int
b'\x80' # Python2 indexes bytes -> bytes
): # protocol marker, protocol > 1
data = data[2:]
if data.startswith(b'(c'): # pickle MARK GLOBAL opcode sequence
global_prefix = 2
elif data.startswith(b'c'): # pickle GLOBAL opcode
global_prefix = 1
else:
global_prefix = 0
if global_prefix:
# Formats 1 and 2.
# Don't actually unpickle a class, because it will attempt to
# load the class. Just break open the pickle and get the
# module and class from it. The module and class names are given by
# newline-terminated strings following the GLOBAL opcode.
modname, classname, rest = data.split(b'\n', 2)
modname = modname[global_prefix:] # strip GLOBAL opcode
return modname.decode(), classname.decode()
# Else there are a bunch of other possible formats.
f = BytesIO(data)
u = Unpickler(f)
try:
class_info = u.load()
except Exception as err:
return '', ''
if isinstance(class_info, tuple):
if isinstance(class_info[0], tuple):
# Formats 3 and 4.
modname, classname = class_info[0]
else:
# Formats 5 and 6 (probably) end up here.
modname, classname = class_info
else:
# This isn't a known format.
modname = repr(class_info)
classname = ''
return modname, classname
def get_pickle_metadata(data):
# Returns a 2-tuple of strings.
# ZODB's data records contain two pickles. The first is the class
# of the object, the second is the object. We're only trying to
# pick apart the first here, to extract the module and class names.
if data[0] in (0x80, # Py3k indexes bytes -> int
b'\x80' # Python2 indexes bytes -> bytes
): # protocol marker, protocol > 1
data = data[2:]
if data.startswith(b'(c'): # pickle MARK GLOBAL opcode sequence
global_prefix = 2
elif data.startswith(b'c'): # pickle GLOBAL opcode
global_prefix = 1
else:
global_prefix = 0
if global_prefix:
# Formats 1 and 2.
# Don't actually unpickle a class, because it will attempt to
# load the class. Just break open the pickle and get the
# module and class from it. The module and class names are given by
# newline-terminated strings following the GLOBAL opcode.
modname, classname, rest = data.split(b'\n', 2)
modname = modname[global_prefix:] # strip GLOBAL opcode
return modname.decode(), classname.decode()
# Else there are a bunch of other possible formats.
f = BytesIO(data)
u = Unpickler(f)
try:
class_info = u.load()
except Exception as err:
return '', ''
if isinstance(class_info, tuple):
if isinstance(class_info[0], tuple):
# Formats 3 and 4.
modname, classname = class_info[0]
else:
# Formats 5 and 6 (probably) end up here.
modname, classname = class_info
else:
# This isn't a known format.
modname = repr(class_info)
classname = ''
return modname, classname
def reorderPickle(jar, p):
try:
from ZODB._compat import Unpickler, Pickler
except ImportError: # BBB: ZODB 3.10
from ZODB.ExportImport import Unpickler, Pickler
from ZODB.ExportImport import Ghost, persistent_id
oids = {}
storage = jar._storage
new_oid = storage.new_oid
store = storage.store
def persistent_load(ooid,
Ghost=Ghost,
oids=oids,
wrote_oid=oids.has_key,
new_oid=storage.new_oid):
"Remap a persistent id to an existing ID and create a ghost for it."
if type(ooid) is TupleType: ooid, klass = ooid
else: klass = None
try:
Ghost = Ghost()
Ghost.oid = ooid
except TypeError:
Ghost = Ghost(ooid)
return Ghost
# Reorder pickle by doing I/O
pfile = StringIO(p)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
newp = StringIO()
pickler = OrderedPickler(newp, 1)
pickler.persistent_id = persistent_id
classdef = unpickler.load()
obj = unpickler.load()
pickler.dump(classdef)
pickler.dump(obj)
p = newp.getvalue()
return obj, p
def reorderPickle(jar, p):
try:
from ZODB._compat import Unpickler, Pickler
except ImportError: # BBB: ZODB 3.10
from ZODB.ExportImport import Unpickler, Pickler
from ZODB.ExportImport import Ghost, persistent_id
oids = {}
storage = jar._storage
new_oid = storage.new_oid
store = storage.store
def persistent_load(ooid,
Ghost=Ghost,
oids=oids, wrote_oid=oids.has_key,
new_oid=storage.new_oid):
"Remap a persistent id to an existing ID and create a ghost for it."
if type(ooid) is TupleType: ooid, klass = ooid
else: klass=None
try:
Ghost=Ghost()
Ghost.oid=ooid
except TypeError:
Ghost=Ghost(ooid)
return Ghost
# Reorder pickle by doing I/O
pfile = StringIO(p)
unpickler=Unpickler(pfile)
unpickler.persistent_load=persistent_load
newp=StringIO()
pickler=OrderedPickler(newp,1)
pickler.persistent_id=persistent_id
classdef = unpickler.load()
obj = unpickler.load()
pickler.dump(classdef)
pickler.dump(obj)
p=newp.getvalue()
return obj, p
def is_blob_record(record):
"""Check whether a database record is a blob record.
This is primarily intended to be used when copying data from one
storage to another.
"""
if record and (b'ZODB.blob' in record):
unpickler = Unpickler(BytesIO(record))
unpickler.find_global = find_global_Blob
try:
return unpickler.load() is Blob
except (MemoryError, KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
return False
def is_blob_record(record):
"""Check whether a database record is a blob record.
This is primarily intended to be used when copying data from one
storage to another.
"""
if record and (b'ZODB.blob' in record):
unpickler = Unpickler(BytesIO(record))
unpickler.find_global = find_global_Blob
try:
return unpickler.load() is Blob
except (MemoryError, KeyboardInterrupt, SystemExit):
raise
except Exception:
pass
return False
def load(class_, fname):
with open(fname, 'rb') as f:
unpickler = Unpickler(f)
pos = unpickler.load()
if not isinstance(pos, INT_TYPES):
# NB: this might contain OIDs that got unpickled
# into Unicode strings on Python 3; hope the caller
# will pipe the result to fsIndex().update() to normalize
# the keys
return pos # Old format
index = class_()
data = index._data
while 1:
v = unpickler.load()
if not v:
break
k, v = v
data[ensure_bytes(k)] = fsBucket().fromString(ensure_bytes(v))
return dict(pos=pos, index=index)
def state(self, oid, serial, prfactory, p=''):
p = p or self.loadSerial(oid, serial)
p = self._crs_untransform_record_data(p)
file = BytesIO(p)
unpickler = Unpickler(file)
unpickler.find_global = find_global
unpickler.persistent_load = prfactory.persistent_load
unpickler.load() # skip the class tuple
return unpickler.load()
def TODO_checkNewPicklesAreSafe(self):
s = MappingStorage()
db = ZODB.DB(s)
r = db.open().root()
r[1] = 1
r[2] = 2
r[3] = r
transaction.commit()
# MappingStorage stores serialno + pickle in its _index.
root_pickle = s._index['\000' * 8][8:]
# XXX not BytesIO really?
f = cStringIO.StringIO(root_pickle)
u = Unpickler(f)
klass_info = u.load()
klass = find_global(*klass_info[0])
inst = klass.__new__(klass)
state = u.load()
inst.__setstate__(state)
self.assertTrue(hasattr(inst, '_container'))
self.assertTrue(not hasattr(inst, 'data'))
def TODO_checkNewPicklesAreSafe(self):
s = MappingStorage()
db = ZODB.DB(s)
r = db.open().root()
r[1] = 1
r[2] = 2
r[3] = r
transaction.commit()
# MappingStorage stores serialno + pickle in its _index.
root_pickle = s._index['\000' * 8][8:]
# XXX not BytesIO really?
f = cStringIO.StringIO(root_pickle)
u = Unpickler(f)
klass_info = u.load()
klass = find_global(*klass_info[0])
inst = klass.__new__(klass)
state = u.load()
inst.__setstate__(state)
self.assertTrue(hasattr(inst, '_container'))
self.assertTrue(not hasattr(inst, 'data'))
def referencesf(p, oids=None):
"""Return a list of object ids found in a pickle
A list may be passed in, in which case, information is
appended to it.
Only ordinary internal references are included.
Weak and multi-database references are not included.
"""
refs = []
u = Unpickler(BytesIO(p))
u.persistent_load = refs.append
u.noload()
u.noload()
# Now we have a list of referencs. Need to convert to list of
# oids:
if oids is None:
oids = []
for reference in refs:
if isinstance(reference, tuple):
oid = reference[0]
elif isinstance(reference, (bytes, str)):
oid = reference
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode('ascii')
oids.append(oid)
return oids
def referencesf(p, oids=None):
"""Return a list of object ids found in a pickle
A list may be passed in, in which case, information is
appended to it.
Only ordinary internal references are included.
Weak and multi-database references are not included.
"""
refs = []
u = Unpickler(BytesIO(p))
u.persistent_load = refs.append
u.noload()
u.noload()
# Now we have a list of referencs. Need to convert to list of
# oids:
if oids is None:
oids = []
for reference in refs:
if isinstance(reference, tuple):
oid = reference[0]
elif isinstance(reference, (bytes, str)):
oid = reference
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode("ascii")
oids.append(oid)
return oids
def get_refs(a_pickle):
"""Return oid and class information for references in a pickle
The result of a list of oid and class information tuples.
If the reference doesn't contain class information, then the
klass information is None.
"""
refs = []
u = Unpickler(BytesIO(a_pickle))
u.persistent_load = refs.append
u.noload()
u.noload()
# Now we have a list of references. Need to convert to list of
# oids and class info:
result = []
for reference in refs:
if isinstance(reference, tuple):
oid, klass = reference
elif isinstance(reference, (bytes, str)):
data, klass = reference, None
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode("ascii")
result.append((oid, klass))
return result
def get_refs(a_pickle):
"""Return oid and class information for references in a pickle
The result of a list of oid and class information tuples.
If the reference doesn't contain class information, then the
klass information is None.
"""
refs = []
u = Unpickler(BytesIO(a_pickle))
u.persistent_load = refs.append
u.noload()
u.noload()
# Now we have a list of references. Need to convert to list of
# oids and class info:
result = []
for reference in refs:
if isinstance(reference, tuple):
oid, klass = reference
elif isinstance(reference, (bytes, str)):
data, klass = reference, None
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode('ascii')
result.append((oid, klass))
return result
def _importDuringCommit(self, transaction, f, return_oid_list):
"""Import data during two-phase commit.
Invoked by the transaction manager mid commit.
Appends one item, the OID of the first object created,
to return_oid_list.
"""
oids = {}
# IMPORTANT: This code should be consistent with the code in
# serialize.py. It is currently out of date and doesn't handle
# weak references.
def persistent_load(ooid):
"""Remap a persistent id to a new ID and create a ghost for it."""
klass = None
if isinstance(ooid, tuple):
ooid, klass = ooid
if not isinstance(ooid, bytes):
assert isinstance(ooid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
ooid = ooid.encode('ascii')
if ooid in oids:
oid = oids[ooid]
else:
if klass is None:
oid = self._storage.new_oid()
else:
oid = self._storage.new_oid(), klass
oids[ooid] = oid
return Ghost(oid)
while 1:
header = f.read(16)
if header == export_end_marker:
break
if len(header) != 16:
raise ExportError("Truncated export file")
# Extract header information
ooid = header[:8]
length = u64(header[8:16])
data = f.read(length)
if len(data) != length:
raise ExportError("Truncated export file")
if oids:
oid = oids[ooid]
if isinstance(oid, tuple):
oid = oid[0]
else:
oids[ooid] = oid = self._storage.new_oid()
return_oid_list.append(oid)
# Blob support
blob_begin = f.read(len(blob_begin_marker))
if blob_begin == blob_begin_marker:
# Copy the blob data to a temporary file
# and remember the name
blob_len = u64(f.read(8))
blob_filename = mktemp()
blob_file = open(blob_filename, "wb")
cp(f, blob_file, blob_len)
blob_file.close()
else:
f.seek(-len(blob_begin_marker), 1)
blob_filename = None
pfile = BytesIO(data)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
newp = BytesIO()
pickler = PersistentPickler(persistent_id, newp, _protocol)
pickler.dump(unpickler.load())
pickler.dump(unpickler.load())
data = newp.getvalue()
if blob_filename is not None:
self._storage.storeBlob(oid, None, data, blob_filename, '',
transaction)
else:
self._storage.store(oid, None, data, '', transaction)
def loads(str, persfunc=self._cache.get):
fp = BytesIO(str)
u = Unpickler(fp)
u.persistent_load = persfunc
return u.load()
def loads(str, persfunc=self._cache.get):
fp = BytesIO(str)
u = Unpickler(fp)
u.persistent_load = persfunc
return u.load()
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
committedData=b''):
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
try:
prfactory = PersistentReferenceFactory()
newpickle = self._crs_untransform_record_data(newpickle)
file = BytesIO(newpickle)
unpickler = Unpickler(file)
unpickler.find_global = find_global
unpickler.persistent_load = prfactory.persistent_load
meta = unpickler.load()
if isinstance(meta, tuple):
klass = meta[0]
newargs = meta[1] or ()
if isinstance(klass, tuple):
klass = find_global(*klass)
else:
klass = meta
newargs = ()
if klass in _unresolvable:
raise ConflictError
inst = klass.__new__(klass, *newargs)
try:
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
raise ConflictError
oldData = self.loadSerial(oid, oldSerial)
if not committedData:
committedData = self.loadSerial(oid, committedSerial)
if newpickle == oldData:
# old -> new diff is empty, so merge is trivial
return committedData
if committedData == oldData:
# old -> committed diff is empty, so merge is trivial
return newpickle
newstate = unpickler.load()
old = state(self, oid, oldSerial, prfactory, oldData)
committed = state(self, oid, committedSerial, prfactory, committedData)
resolved = resolve(old, committed, newstate)
file = BytesIO()
pickler = Pickler(file, _protocol)
if sys.version_info[0] < 3:
pickler.inst_persistent_id = persistent_id
else:
pickler.persistent_id = persistent_id
pickler.dump(meta)
pickler.dump(resolved)
return self._crs_transform_record_data(file.getvalue())
except (ConflictError, BadClassName):
pass
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
# the original ConflictError. A client can recover from a
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.error("Unexpected error", exc_info=True)
raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
data=newpickle)
def unpickle_state(data):
unpickler = Unpickler(StringIO(data))
unpickler.persistent_load = PersistentReferenceFactory().persistent_load
unpickler.load() # skip the class tuple
return unpickler.load()
def FakeUnpickler(f):
unpickler = Unpickler(f)
unpickler.find_global = fake_find_class
return unpickler
def _importDuringCommit(self, transaction, f, return_oid_list):
"""Import data during two-phase commit.
Invoked by the transaction manager mid commit.
Appends one item, the OID of the first object created,
to return_oid_list.
"""
oids = {}
# IMPORTANT: This code should be consistent with the code in
# serialize.py. It is currently out of date and doesn't handle
# weak references.
def persistent_load(ooid):
"""Remap a persistent id to a new ID and create a ghost for it."""
klass = None
if isinstance(ooid, tuple):
ooid, klass = ooid
if not isinstance(ooid, bytes):
assert isinstance(ooid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
ooid = ooid.encode('ascii')
if ooid in oids:
oid = oids[ooid]
else:
if klass is None:
oid = self._storage.new_oid()
else:
oid = self._storage.new_oid(), klass
oids[ooid] = oid
return Ghost(oid)
while 1:
header = f.read(16)
if header == export_end_marker:
break
if len(header) != 16:
raise ExportError("Truncated export file")
# Extract header information
ooid = header[:8]
length = u64(header[8:16])
data = f.read(length)
if len(data) != length:
raise ExportError("Truncated export file")
if oids:
oid = oids[ooid]
if isinstance(oid, tuple):
oid = oid[0]
else:
oids[ooid] = oid = self._storage.new_oid()
return_oid_list.append(oid)
# Blob support
blob_begin = f.read(len(blob_begin_marker))
if blob_begin == blob_begin_marker:
# Copy the blob data to a temporary file
# and remember the name
blob_len = u64(f.read(8))
blob_filename = mktemp()
blob_file = open(blob_filename, "wb")
cp(f, blob_file, blob_len)
blob_file.close()
else:
f.seek(-len(blob_begin_marker),1)
blob_filename = None
pfile = BytesIO(data)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
newp = BytesIO()
pickler = PersistentPickler(persistent_id, newp, _protocol)
pickler.dump(unpickler.load())
pickler.dump(unpickler.load())
data = newp.getvalue()
if blob_filename is not None:
self._storage.storeBlob(oid, None, data, blob_filename,
'', transaction)
else:
self._storage.store(oid, None, data, '', transaction)
def FakeUnpickler(f):
unpickler = Unpickler(f)
unpickler.find_global = fake_find_class
return unpickler