def zodb_unpickle(data):
"""Unpickle an object stored using the format expected by ZODB."""
f = BytesIO(data)
u = Unpickler(f)
u.persistent_load = persistent_load
klass_info = u.load()
if isinstance(klass_info, tuple):
if isinstance(klass_info[0], type):
# Unclear: what is the second part of klass_info?
klass, xxx = klass_info
assert not xxx
else:
if isinstance(klass_info[0], tuple):
modname, klassname = klass_info[0]
else:
modname, klassname = klass_info
if modname == "__main__":
ns = globals()
else:
mod = import_helper(modname)
ns = mod.__dict__
try:
klass = ns[klassname]
except KeyError:
print("can't find %s in %r" % (klassname, ns), file=sys.stderr)
inst = klass()
else:
raise ValueError("expected class info: %s" % repr(klass_info))
state = u.load()
inst.__setstate__(state)
return inst
def cloneByPickle(obj, ignore_list=()):
"""Makes a copy of a ZODB object, loading ghosts as needed.
Ignores specified objects along the way, replacing them with None
in the copy.
"""
ignore_dict = {}
for o in ignore_list:
ignore_dict[id(o)] = o
def persistent_id(ob, ignore_dict=ignore_dict):
if id(ob) in ignore_dict:
return 'ignored'
if getattr(ob, '_p_changed', 0) is None:
ob._p_changed = 0
return None
def persistent_load(ref):
assert ref == 'ignored'
# Return a placeholder object that will be replaced by
# removeNonVersionedData().
placeholder = SimpleItem()
placeholder.id = "ignored_subobject"
return placeholder
stream = BytesIO()
p = Pickler(stream, 1)
p.persistent_id = persistent_id
p.dump(obj)
stream.seek(0)
u = Unpickler(stream)
u.persistent_load = persistent_load
return u.load()
def cloneByPickle(obj, ignore_list=()):
"""Makes a copy of a ZODB object, loading ghosts as needed.
Ignores specified objects along the way, replacing them with None
in the copy.
"""
ignore_dict = {}
for o in ignore_list:
ignore_dict[id(o)] = o
def persistent_id(ob, ignore_dict=ignore_dict):
if id(ob) in ignore_dict:
return 'ignored'
if getattr(ob, '_p_changed', 0) is None:
ob._p_changed = 0
return None
def persistent_load(ref):
assert ref == 'ignored'
# Return a placeholder object that will be replaced by
# removeNonVersionedData().
placeholder = SimpleItem()
placeholder.id = "ignored_subobject"
return placeholder
stream = BytesIO()
p = Pickler(stream, 1)
p.persistent_id = persistent_id
p.dump(obj)
stream.seek(0)
u = Unpickler(stream)
u.persistent_load = persistent_load
return u.load()
def state(self, oid, serial, prfactory, p=''):
p = p or self.loadSerial(oid, serial)
p = self._crs_untransform_record_data(p)
file = BytesIO(p)
unpickler = Unpickler(file)
unpickler.find_global = find_global
unpickler.persistent_load = prfactory.persistent_load
unpickler.load() # skip the class tuple
return unpickler.load()
def _get_unpickler(self, pickle):
file = BytesIO(pickle)
unpickler = Unpickler(file)
unpickler.persistent_load = self._persistent_load
factory = self._factory
conn = self._conn
def find_global(modulename, name):
return factory(conn, modulename, name)
unpickler.find_global = find_global
return unpickler
def _get_unpickler(self, pickle):
file = BytesIO(pickle)
unpickler = Unpickler(file)
unpickler.persistent_load = self._persistent_load
factory = self._factory
conn = self._conn
def find_global(modulename, name):
return factory(conn, modulename, name)
unpickler.find_global = find_global
return unpickler
def reorderPickle(jar, p):
try:
from ZODB._compat import Unpickler, Pickler
except ImportError: # BBB: ZODB 3.10
from ZODB.ExportImport import Unpickler, Pickler
from ZODB.ExportImport import Ghost, persistent_id
oids = {}
storage = jar._storage
new_oid = storage.new_oid
store = storage.store
def persistent_load(ooid,
Ghost=Ghost,
oids=oids,
wrote_oid=oids.has_key,
new_oid=storage.new_oid):
"Remap a persistent id to an existing ID and create a ghost for it."
if type(ooid) is TupleType: ooid, klass = ooid
else: klass = None
try:
Ghost = Ghost()
Ghost.oid = ooid
except TypeError:
Ghost = Ghost(ooid)
return Ghost
# Reorder pickle by doing I/O
pfile = StringIO(p)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
newp = StringIO()
pickler = OrderedPickler(newp, 1)
pickler.persistent_id = persistent_id
classdef = unpickler.load()
obj = unpickler.load()
pickler.dump(classdef)
pickler.dump(obj)
p = newp.getvalue()
return obj, p
def reorderPickle(jar, p):
try:
from ZODB._compat import Unpickler, Pickler
except ImportError: # BBB: ZODB 3.10
from ZODB.ExportImport import Unpickler, Pickler
from ZODB.ExportImport import Ghost, persistent_id
oids = {}
storage = jar._storage
new_oid = storage.new_oid
store = storage.store
def persistent_load(ooid,
Ghost=Ghost,
oids=oids, wrote_oid=oids.has_key,
new_oid=storage.new_oid):
"Remap a persistent id to an existing ID and create a ghost for it."
if type(ooid) is TupleType: ooid, klass = ooid
else: klass=None
try:
Ghost=Ghost()
Ghost.oid=ooid
except TypeError:
Ghost=Ghost(ooid)
return Ghost
# Reorder pickle by doing I/O
pfile = StringIO(p)
unpickler=Unpickler(pfile)
unpickler.persistent_load=persistent_load
newp=StringIO()
pickler=OrderedPickler(newp,1)
pickler.persistent_id=persistent_id
classdef = unpickler.load()
obj = unpickler.load()
pickler.dump(classdef)
pickler.dump(obj)
p=newp.getvalue()
return obj, p
def referencesf(p, oids=None):
"""Return a list of object ids found in a pickle
A list may be passed in, in which case, information is
appended to it.
Only ordinary internal references are included.
Weak and multi-database references are not included.
"""
refs = []
u = Unpickler(BytesIO(p))
u.persistent_load = refs.append
u.noload()
u.noload()
# Now we have a list of referencs. Need to convert to list of
# oids:
if oids is None:
oids = []
for reference in refs:
if isinstance(reference, tuple):
oid = reference[0]
elif isinstance(reference, (bytes, str)):
oid = reference
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode("ascii")
oids.append(oid)
return oids
def referencesf(p, oids=None):
"""Return a list of object ids found in a pickle
A list may be passed in, in which case, information is
appended to it.
Only ordinary internal references are included.
Weak and multi-database references are not included.
"""
refs = []
u = Unpickler(BytesIO(p))
u.persistent_load = refs.append
u.noload()
u.noload()
# Now we have a list of referencs. Need to convert to list of
# oids:
if oids is None:
oids = []
for reference in refs:
if isinstance(reference, tuple):
oid = reference[0]
elif isinstance(reference, (bytes, str)):
oid = reference
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode('ascii')
oids.append(oid)
return oids
def get_refs(a_pickle):
"""Return oid and class information for references in a pickle
The result of a list of oid and class information tuples.
If the reference doesn't contain class information, then the
klass information is None.
"""
refs = []
u = Unpickler(BytesIO(a_pickle))
u.persistent_load = refs.append
u.noload()
u.noload()
# Now we have a list of references. Need to convert to list of
# oids and class info:
result = []
for reference in refs:
if isinstance(reference, tuple):
oid, klass = reference
elif isinstance(reference, (bytes, str)):
data, klass = reference, None
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode("ascii")
result.append((oid, klass))
return result
def get_refs(a_pickle):
"""Return oid and class information for references in a pickle
The result of a list of oid and class information tuples.
If the reference doesn't contain class information, then the
klass information is None.
"""
refs = []
u = Unpickler(BytesIO(a_pickle))
u.persistent_load = refs.append
u.noload()
u.noload()
# Now we have a list of references. Need to convert to list of
# oids and class info:
result = []
for reference in refs:
if isinstance(reference, tuple):
oid, klass = reference
elif isinstance(reference, (bytes, str)):
data, klass = reference, None
else:
assert isinstance(reference, list)
continue
if not isinstance(oid, bytes):
assert isinstance(oid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
oid = oid.encode('ascii')
result.append((oid, klass))
return result
def loads(str, persfunc=self._cache.get):
fp = BytesIO(str)
u = Unpickler(fp)
u.persistent_load = persfunc
return u.load()
def unpickle_state(data):
unpickler = Unpickler(StringIO(data))
unpickler.persistent_load = PersistentReferenceFactory().persistent_load
unpickler.load() # skip the class tuple
return unpickler.load()
def _importDuringCommit(self, transaction, f, return_oid_list):
"""Import data during two-phase commit.
Invoked by the transaction manager mid commit.
Appends one item, the OID of the first object created,
to return_oid_list.
"""
oids = {}
# IMPORTANT: This code should be consistent with the code in
# serialize.py. It is currently out of date and doesn't handle
# weak references.
def persistent_load(ooid):
"""Remap a persistent id to a new ID and create a ghost for it."""
klass = None
if isinstance(ooid, tuple):
ooid, klass = ooid
if not isinstance(ooid, bytes):
assert isinstance(ooid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
ooid = ooid.encode('ascii')
if ooid in oids:
oid = oids[ooid]
else:
if klass is None:
oid = self._storage.new_oid()
else:
oid = self._storage.new_oid(), klass
oids[ooid] = oid
return Ghost(oid)
while 1:
header = f.read(16)
if header == export_end_marker:
break
if len(header) != 16:
raise ExportError("Truncated export file")
# Extract header information
ooid = header[:8]
length = u64(header[8:16])
data = f.read(length)
if len(data) != length:
raise ExportError("Truncated export file")
if oids:
oid = oids[ooid]
if isinstance(oid, tuple):
oid = oid[0]
else:
oids[ooid] = oid = self._storage.new_oid()
return_oid_list.append(oid)
# Blob support
blob_begin = f.read(len(blob_begin_marker))
if blob_begin == blob_begin_marker:
# Copy the blob data to a temporary file
# and remember the name
blob_len = u64(f.read(8))
blob_filename = mktemp()
blob_file = open(blob_filename, "wb")
cp(f, blob_file, blob_len)
blob_file.close()
else:
f.seek(-len(blob_begin_marker),1)
blob_filename = None
pfile = BytesIO(data)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
newp = BytesIO()
pickler = PersistentPickler(persistent_id, newp, _protocol)
pickler.dump(unpickler.load())
pickler.dump(unpickler.load())
data = newp.getvalue()
if blob_filename is not None:
self._storage.storeBlob(oid, None, data, blob_filename,
'', transaction)
else:
self._storage.store(oid, None, data, '', transaction)
def loads(str, persfunc=self._cache.get):
fp = BytesIO(str)
u = Unpickler(fp)
u.persistent_load = persfunc
return u.load()
def _importDuringCommit(self, transaction, f, return_oid_list):
"""Import data during two-phase commit.
Invoked by the transaction manager mid commit.
Appends one item, the OID of the first object created,
to return_oid_list.
"""
oids = {}
# IMPORTANT: This code should be consistent with the code in
# serialize.py. It is currently out of date and doesn't handle
# weak references.
def persistent_load(ooid):
"""Remap a persistent id to a new ID and create a ghost for it."""
klass = None
if isinstance(ooid, tuple):
ooid, klass = ooid
if not isinstance(ooid, bytes):
assert isinstance(ooid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
ooid = ooid.encode('ascii')
if ooid in oids:
oid = oids[ooid]
else:
if klass is None:
oid = self._storage.new_oid()
else:
oid = self._storage.new_oid(), klass
oids[ooid] = oid
return Ghost(oid)
while 1:
header = f.read(16)
if header == export_end_marker:
break
if len(header) != 16:
raise ExportError("Truncated export file")
# Extract header information
ooid = header[:8]
length = u64(header[8:16])
data = f.read(length)
if len(data) != length:
raise ExportError("Truncated export file")
if oids:
oid = oids[ooid]
if isinstance(oid, tuple):
oid = oid[0]
else:
oids[ooid] = oid = self._storage.new_oid()
return_oid_list.append(oid)
# Blob support
blob_begin = f.read(len(blob_begin_marker))
if blob_begin == blob_begin_marker:
# Copy the blob data to a temporary file
# and remember the name
blob_len = u64(f.read(8))
blob_filename = mktemp()
blob_file = open(blob_filename, "wb")
cp(f, blob_file, blob_len)
blob_file.close()
else:
f.seek(-len(blob_begin_marker), 1)
blob_filename = None
pfile = BytesIO(data)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
newp = BytesIO()
pickler = PersistentPickler(persistent_id, newp, _protocol)
pickler.dump(unpickler.load())
pickler.dump(unpickler.load())
data = newp.getvalue()
if blob_filename is not None:
self._storage.storeBlob(oid, None, data, blob_filename, '',
transaction)
else:
self._storage.store(oid, None, data, '', transaction)
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
committedData=b''):
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
try:
prfactory = PersistentReferenceFactory()
newpickle = self._crs_untransform_record_data(newpickle)
file = BytesIO(newpickle)
unpickler = Unpickler(file)
unpickler.find_global = find_global
unpickler.persistent_load = prfactory.persistent_load
meta = unpickler.load()
if isinstance(meta, tuple):
klass = meta[0]
newargs = meta[1] or ()
if isinstance(klass, tuple):
klass = find_global(*klass)
else:
klass = meta
newargs = ()
if klass in _unresolvable:
raise ConflictError
inst = klass.__new__(klass, *newargs)
try:
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
raise ConflictError
oldData = self.loadSerial(oid, oldSerial)
if not committedData:
committedData = self.loadSerial(oid, committedSerial)
if newpickle == oldData:
# old -> new diff is empty, so merge is trivial
return committedData
if committedData == oldData:
# old -> committed diff is empty, so merge is trivial
return newpickle
newstate = unpickler.load()
old = state(self, oid, oldSerial, prfactory, oldData)
committed = state(self, oid, committedSerial, prfactory, committedData)
resolved = resolve(old, committed, newstate)
file = BytesIO()
pickler = Pickler(file, _protocol)
if sys.version_info[0] < 3:
pickler.inst_persistent_id = persistent_id
else:
pickler.persistent_id = persistent_id
pickler.dump(meta)
pickler.dump(resolved)
return self._crs_transform_record_data(file.getvalue())
except (ConflictError, BadClassName):
pass
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
# the original ConflictError. A client can recover from a
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.error("Unexpected error", exc_info=True)
raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
data=newpickle)