def __init__(self, obj=None):
self._file = BytesIO()
self._p = PersistentPickler(self.persistent_id, self._file, _protocol)
self._stack = []
if obj is not None:
self._stack.append(obj)
jar = obj._p_jar
assert myhasattr(jar, "new_oid")
self._jar = jar
def dumps(obj):
def getpersid(obj):
if hasattr(obj, 'getoid'):
return obj.getoid()
return None
s = BytesIO()
p = PersistentPickler(getpersid, s, _protocol)
p.dump(obj)
p.dump(None)
return s.getvalue()
def __init__(self, obj=None):
self._file = BytesIO()
self._p = PersistentPickler(self.persistent_id, self._file, _protocol)
self._stack = []
if obj is not None:
self._stack.append(obj)
jar = obj._p_jar
assert myhasattr(jar, "new_oid")
self._jar = jar
def dumps(obj):
def getpersid(obj):
if hasattr(obj, 'getoid'):
return obj.getoid()
return None
s = BytesIO()
p = PersistentPickler(getpersid, s, _protocol)
p.dump(obj)
p.dump(None)
return s.getvalue()
def zodb_pickle(obj):
"""Create a pickle in the format expected by ZODB."""
f = BytesIO()
p = PersistentPickler(_persistent_id, f, _protocol)
klass = obj.__class__
assert not hasattr(obj, '__getinitargs__'), "not ready for constructors"
args = None
mod = getattr(klass, '__module__', None)
if mod is not None:
klass = mod, klass.__name__
state = obj.__getstate__()
p.dump((klass, args))
p.dump(state)
return f.getvalue()
def _importDuringCommit(self, transaction, f, return_oid_list):
"""Import data during two-phase commit.
Invoked by the transaction manager mid commit.
Appends one item, the OID of the first object created,
to return_oid_list.
"""
oids = {}
# IMPORTANT: This code should be consistent with the code in
# serialize.py. It is currently out of date and doesn't handle
# weak references.
def persistent_load(ooid):
"""Remap a persistent id to a new ID and create a ghost for it."""
klass = None
if isinstance(ooid, tuple):
ooid, klass = ooid
if not isinstance(ooid, bytes):
assert isinstance(ooid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
ooid = ooid.encode('ascii')
if ooid in oids:
oid = oids[ooid]
else:
if klass is None:
oid = self._storage.new_oid()
else:
oid = self._storage.new_oid(), klass
oids[ooid] = oid
return Ghost(oid)
while 1:
header = f.read(16)
if header == export_end_marker:
break
if len(header) != 16:
raise ExportError("Truncated export file")
# Extract header information
ooid = header[:8]
length = u64(header[8:16])
data = f.read(length)
if len(data) != length:
raise ExportError("Truncated export file")
if oids:
oid = oids[ooid]
if isinstance(oid, tuple):
oid = oid[0]
else:
oids[ooid] = oid = self._storage.new_oid()
return_oid_list.append(oid)
# Blob support
blob_begin = f.read(len(blob_begin_marker))
if blob_begin == blob_begin_marker:
# Copy the blob data to a temporary file
# and remember the name
blob_len = u64(f.read(8))
blob_filename = mktemp()
blob_file = open(blob_filename, "wb")
cp(f, blob_file, blob_len)
blob_file.close()
else:
f.seek(-len(blob_begin_marker), 1)
blob_filename = None
pfile = BytesIO(data)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
newp = BytesIO()
pickler = PersistentPickler(persistent_id, newp, _protocol)
pickler.dump(unpickler.load())
pickler.dump(unpickler.load())
data = newp.getvalue()
if blob_filename is not None:
self._storage.storeBlob(oid, None, data, blob_filename, '',
transaction)
else:
self._storage.store(oid, None, data, '', transaction)
def _importDuringCommit(self, transaction, f, return_oid_list):
"""Import data during two-phase commit.
Invoked by the transaction manager mid commit.
Appends one item, the OID of the first object created,
to return_oid_list.
"""
oids = {}
# IMPORTANT: This code should be consistent with the code in
# serialize.py. It is currently out of date and doesn't handle
# weak references.
def persistent_load(ooid):
"""Remap a persistent id to a new ID and create a ghost for it."""
klass = None
if isinstance(ooid, tuple):
ooid, klass = ooid
if not isinstance(ooid, bytes):
assert isinstance(ooid, str)
# this happens on Python 3 when all bytes in the oid are < 0x80
ooid = ooid.encode('ascii')
if ooid in oids:
oid = oids[ooid]
else:
if klass is None:
oid = self._storage.new_oid()
else:
oid = self._storage.new_oid(), klass
oids[ooid] = oid
return Ghost(oid)
while 1:
header = f.read(16)
if header == export_end_marker:
break
if len(header) != 16:
raise ExportError("Truncated export file")
# Extract header information
ooid = header[:8]
length = u64(header[8:16])
data = f.read(length)
if len(data) != length:
raise ExportError("Truncated export file")
if oids:
oid = oids[ooid]
if isinstance(oid, tuple):
oid = oid[0]
else:
oids[ooid] = oid = self._storage.new_oid()
return_oid_list.append(oid)
# Blob support
blob_begin = f.read(len(blob_begin_marker))
if blob_begin == blob_begin_marker:
# Copy the blob data to a temporary file
# and remember the name
blob_len = u64(f.read(8))
blob_filename = mktemp()
blob_file = open(blob_filename, "wb")
cp(f, blob_file, blob_len)
blob_file.close()
else:
f.seek(-len(blob_begin_marker),1)
blob_filename = None
pfile = BytesIO(data)
unpickler = Unpickler(pfile)
unpickler.persistent_load = persistent_load
newp = BytesIO()
pickler = PersistentPickler(persistent_id, newp, _protocol)
pickler.dump(unpickler.load())
pickler.dump(unpickler.load())
data = newp.getvalue()
if blob_filename is not None:
self._storage.storeBlob(oid, None, data, blob_filename,
'', transaction)
else:
self._storage.store(oid, None, data, '', transaction)
class ObjectWriter:
"""Serializes objects for storage in the database.
The ObjectWriter creates object pickles in the ZODB format. It
also detects new persistent objects reachable from the current
object.
"""
_jar = None
def __init__(self, obj=None):
self._file = BytesIO()
self._p = PersistentPickler(self.persistent_id, self._file, _protocol)
self._stack = []
if obj is not None:
self._stack.append(obj)
jar = obj._p_jar
assert myhasattr(jar, "new_oid")
self._jar = jar
def persistent_id(self, obj):
"""Return the persistent id for obj.
>>> from ZODB.tests.util import P
>>> class DummyJar:
... xrefs = True
... def new_oid(self):
... return 42
... def db(self):
... return self
... databases = {}
>>> jar = DummyJar()
>>> class O:
... _p_jar = jar
>>> writer = ObjectWriter(O)
Normally, object references include the oid and a cached named
reference to the class. Having the class information
available allows fast creation of the ghost, avoiding
requiring an additional database lookup.
>>> bob = P('bob')
>>> oid, cls = writer.persistent_id(bob)
>>> oid
42
>>> cls is P
True
If a persistent object does not already have an oid and jar,
these will be assigned by persistent_id():
>>> bob._p_oid
42
>>> bob._p_jar is jar
True
If the object already has a persistent id, the id is not changed:
>>> bob._p_oid = 24
>>> oid, cls = writer.persistent_id(bob)
>>> oid
24
>>> cls is P
True
If the jar doesn't match that of the writer, an error is raised:
>>> bob._p_jar = DummyJar()
>>> writer.persistent_id(bob)
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
Traceback (most recent call last):
...
InvalidObjectReference:
('Attempt to store an object from a foreign database connection',
<ZODB.serialize.DummyJar ...>, P(bob))
Constructor arguments used by __new__(), as returned by
__getnewargs__(), can affect memory allocation, but may also
change over the life of the object. This makes it useless to
cache even the object's class.
>>> class PNewArgs(P):
... def __getnewargs__(self):
... return ()
>>> sam = PNewArgs('sam')
>>> writer.persistent_id(sam)
42
>>> sam._p_oid
42
>>> sam._p_jar is jar
True
Check that simple objects don't get accused of persistence:
>>> writer.persistent_id(42)
>>> writer.persistent_id(object())
Check that a classic class doesn't get identified improperly:
>>> class ClassicClara:
... pass
>>> clara = ClassicClara()
>>> writer.persistent_id(clara)
"""
# Most objects are not persistent. The following cheap test
# identifies most of them. For these, we return None,
# signalling that the object should be pickled normally.
if not isinstance(obj, (Persistent, type, WeakRef)):
# Not persistent, pickle normally
return None
# Any persistent object must have an oid:
try:
oid = obj._p_oid
except AttributeError:
# Not persistent, pickle normally
return None
if not (oid is None or isinstance(oid, bytes)):
# Deserves a closer look:
# Make sure it's not a descriptor
if hasattr(oid, '__get__'):
# The oid is a descriptor. That means obj is a non-persistent
# class whose instances are persistent, so ...
# Not persistent, pickle normally
return None
if oid is WeakRefMarker:
# we have a weakref, see weakref.py
oid = obj.oid
if oid is None:
target = obj() # get the referenced object
oid = target._p_oid
if oid is None:
# Here we are causing the object to be saved in
# the database. One could argue that we shouldn't
# do this, because a weakref should not cause an object
# to be added. We'll be optimistic, though, and
# assume that the object will be added eventually.
oid = self._jar.new_oid()
target._p_jar = self._jar
target._p_oid = oid
self._stack.append(target)
obj.oid = oid
obj.dm = target._p_jar
obj.database_name = obj.dm.db().database_name
if obj.dm is self._jar:
return ['w', (oid, )]
else:
return ['w', (oid, obj.database_name)]
# Since we have an oid, we have either a persistent instance
# (an instance of Persistent), or a persistent class.
# NOTE! Persistent classes don't (and can't) subclass persistent.
database_name = None
if oid is None:
oid = obj._p_oid = self._jar.new_oid()
obj._p_jar = self._jar
self._stack.append(obj)
elif obj._p_jar is not self._jar:
if not self._jar.db().xrefs:
raise InvalidObjectReference(
"Database %r doesn't allow implicit cross-database "
"references" % self._jar.db().database_name, self._jar,
obj)
try:
otherdb = obj._p_jar.db()
database_name = otherdb.database_name
except AttributeError:
otherdb = self
if self._jar.db().databases.get(database_name) is not otherdb:
raise InvalidObjectReference(
"Attempt to store an object from a foreign "
"database connection",
self._jar,
obj,
)
if self._jar.get_connection(database_name) is not obj._p_jar:
raise InvalidObjectReference(
"Attempt to store a reference to an object from "
"a separate connection to the same database or "
"multidatabase",
self._jar,
obj,
)
# OK, we have an object from another database.
# Lets make sure the object ws not *just* loaded.
if obj._p_jar._implicitlyAdding(oid):
raise InvalidObjectReference(
"A new object is reachable from multiple databases. "
"Won't try to guess which one was correct!",
self._jar,
obj,
)
klass = type(obj)
if hasattr(klass, '__getnewargs__'):
# We don't want to save newargs in object refs.
# It's possible that __getnewargs__ is degenerate and
# returns (), but we don't want to have to deghostify
# the object to find out.
# Note that this has the odd effect that, if the class has
# __getnewargs__ of its own, we'll lose the optimization
# of caching the class info.
if database_name is not None:
return ['n', (database_name, oid)]
return oid
# Note that we never get here for persistent classes.
# We'll use direct refs for normal classes.
if database_name is not None:
return ['m', (database_name, oid, klass)]
return oid, klass
def serialize(self, obj):
# We don't use __class__ here, because obj could be a persistent proxy.
# We don't want to be fooled by proxies.
klass = type(obj)
# We want to serialize persistent classes by name if they have
# a non-None non-empty module so as not to have a direct
# ref. This is important when copying. We probably want to
# revisit this in the future.
newargs = getattr(obj, "__getnewargs__", None)
if (isinstance(getattr(klass, '_p_oid', 0), _oidtypes)
and klass.__module__):
# This is a persistent class with a non-empty module. This
# uses pickle format #3 or #7.
klass = klass.__module__, klass.__name__
if newargs is None:
meta = klass, None
else:
meta = klass, newargs()
elif newargs is None:
# Pickle format #1.
meta = klass
else:
# Pickle format #2.
meta = klass, newargs()
return self._dump(meta, obj.__getstate__())
def _dump(self, classmeta, state):
# To reuse the existing BytesIO object, we must reset
# the file position to 0 and truncate the file after the
# new pickle is written.
self._file.seek(0)
self._p.clear_memo()
self._p.dump(classmeta)
self._p.dump(state)
self._file.truncate()
return self._file.getvalue()
def __iter__(self):
return NewObjectIterator(self._stack)
def tryToResolveConflict(self,
oid,
committedSerial,
oldSerial,
newpickle,
committedData=b''):
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
klass = 'n/a'
try:
prfactory = PersistentReferenceFactory()
newpickle = self._crs_untransform_record_data(newpickle)
file = BytesIO(newpickle)
unpickler = PersistentUnpickler(find_global, prfactory.persistent_load,
file)
meta = unpickler.load()
if isinstance(meta, tuple):
klass = meta[0]
newargs = meta[1] or ()
if isinstance(klass, tuple):
klass = find_global(*klass)
else:
klass = meta
newargs = ()
if klass in _unresolvable:
raise ConflictError
inst = klass.__new__(klass, *newargs)
try:
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
raise ConflictError
oldData = self.loadSerial(oid, oldSerial)
if not committedData:
committedData = self.loadSerial(oid, committedSerial)
newstate = unpickler.load()
old = state(self, oid, oldSerial, prfactory, oldData)
committed = state(self, oid, committedSerial, prfactory, committedData)
resolved = resolve(old, committed, newstate)
file = BytesIO()
pickler = PersistentPickler(persistent_id, file, _protocol)
pickler.dump(meta)
pickler.dump(resolved)
return self._crs_transform_record_data(file.getvalue())
except (ConflictError, BadClassName) as e:
logger.debug("Conflict resolution on %s failed with %s: %s", klass,
e.__class__.__name__, str(e))
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
# the original ConflictError. A client can recover from a
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.exception(
"Unexpected error while trying to resolve conflict on %s", klass)
raise ConflictError(oid=oid,
serials=(committedSerial, oldSerial),
data=newpickle)
class ObjectWriter:
"""Serializes objects for storage in the database.
The ObjectWriter creates object pickles in the ZODB format. It
also detects new persistent objects reachable from the current
object.
"""
_jar = None
def __init__(self, obj=None):
self._file = BytesIO()
self._p = PersistentPickler(self.persistent_id, self._file, _protocol)
self._stack = []
if obj is not None:
self._stack.append(obj)
jar = obj._p_jar
assert myhasattr(jar, "new_oid")
self._jar = jar
def persistent_id(self, obj):
"""Return the persistent id for obj.
>>> from ZODB.tests.util import P
>>> class DummyJar:
... xrefs = True
... def new_oid(self):
... return 42
... def db(self):
... return self
... databases = {}
>>> jar = DummyJar()
>>> class O:
... _p_jar = jar
>>> writer = ObjectWriter(O)
Normally, object references include the oid and a cached named
reference to the class. Having the class information
available allows fast creation of the ghost, avoiding
requiring an additional database lookup.
>>> bob = P('bob')
>>> oid, cls = writer.persistent_id(bob)
>>> oid
42
>>> cls is P
True
If a persistent object does not already have an oid and jar,
these will be assigned by persistent_id():
>>> bob._p_oid
42
>>> bob._p_jar is jar
True
If the object already has a persistent id, the id is not changed:
>>> bob._p_oid = 24
>>> oid, cls = writer.persistent_id(bob)
>>> oid
24
>>> cls is P
True
If the jar doesn't match that of the writer, an error is raised:
>>> bob._p_jar = DummyJar()
>>> writer.persistent_id(bob)
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
Traceback (most recent call last):
...
InvalidObjectReference:
('Attempt to store an object from a foreign database connection',
<ZODB.serialize.DummyJar ...>, P(bob))
Constructor arguments used by __new__(), as returned by
__getnewargs__(), can affect memory allocation, but may also
change over the life of the object. This makes it useless to
cache even the object's class.
>>> class PNewArgs(P):
... def __getnewargs__(self):
... return ()
>>> sam = PNewArgs('sam')
>>> writer.persistent_id(sam)
42
>>> sam._p_oid
42
>>> sam._p_jar is jar
True
Check that simple objects don't get accused of persistence:
>>> writer.persistent_id(42)
>>> writer.persistent_id(object())
Check that a classic class doesn't get identified improperly:
>>> class ClassicClara:
... pass
>>> clara = ClassicClara()
>>> writer.persistent_id(clara)
"""
# Most objects are not persistent. The following cheap test
# identifies most of them. For these, we return None,
# signalling that the object should be pickled normally.
if not isinstance(obj, (Persistent, type, WeakRef)):
# Not persistent, pickle normally
return None
# Any persistent object must have an oid:
try:
oid = obj._p_oid
except AttributeError:
# Not persistent, pickle normally
return None
if not (oid is None or isinstance(oid, bytes)):
# Deserves a closer look:
# Make sure it's not a descriptor
if hasattr(oid, '__get__'):
# The oid is a descriptor. That means obj is a non-persistent
# class whose instances are persistent, so ...
# Not persistent, pickle normally
return None
if oid is WeakRefMarker:
# we have a weakref, see weakref.py
oid = obj.oid
if oid is None:
target = obj() # get the referenced object
oid = target._p_oid
if oid is None:
# Here we are causing the object to be saved in
# the database. One could argue that we shouldn't
# do this, because a weakref should not cause an object
# to be added. We'll be optimistic, though, and
# assume that the object will be added eventually.
oid = self._jar.new_oid()
target._p_jar = self._jar
target._p_oid = oid
self._stack.append(target)
obj.oid = oid
obj.dm = target._p_jar
obj.database_name = obj.dm.db().database_name
if obj.dm is self._jar:
return ['w', (oid, )]
else:
return ['w', (oid, obj.database_name)]
# Since we have an oid, we have either a persistent instance
# (an instance of Persistent), or a persistent class.
# NOTE! Persistent classes don't (and can't) subclass persistent.
database_name = None
if oid is None:
oid = obj._p_oid = self._jar.new_oid()
obj._p_jar = self._jar
self._stack.append(obj)
elif obj._p_jar is not self._jar:
if not self._jar.db().xrefs:
raise InvalidObjectReference(
"Database %r doesn't allow implicit cross-database "
"references" % self._jar.db().database_name,
self._jar, obj)
try:
otherdb = obj._p_jar.db()
database_name = otherdb.database_name
except AttributeError:
otherdb = self
if self._jar.db().databases.get(database_name) is not otherdb:
raise InvalidObjectReference(
"Attempt to store an object from a foreign "
"database connection", self._jar, obj,
)
if self._jar.get_connection(database_name) is not obj._p_jar:
raise InvalidObjectReference(
"Attempt to store a reference to an object from "
"a separate connection to the same database or "
"multidatabase", self._jar, obj,
)
# OK, we have an object from another database.
# Lets make sure the object ws not *just* loaded.
if obj._p_jar._implicitlyAdding(oid):
raise InvalidObjectReference(
"A new object is reachable from multiple databases. "
"Won't try to guess which one was correct!",
self._jar, obj,
)
klass = type(obj)
if hasattr(klass, '__getnewargs__'):
# We don't want to save newargs in object refs.
# It's possible that __getnewargs__ is degenerate and
# returns (), but we don't want to have to deghostify
# the object to find out.
# Note that this has the odd effect that, if the class has
# __getnewargs__ of its own, we'll lose the optimization
# of caching the class info.
if database_name is not None:
return ['n', (database_name, oid)]
return oid
# Note that we never get here for persistent classes.
# We'll use direct refs for normal classes.
if database_name is not None:
return ['m', (database_name, oid, klass)]
return oid, klass
def serialize(self, obj):
# We don't use __class__ here, because obj could be a persistent proxy.
# We don't want to be fooled by proxies.
klass = type(obj)
# We want to serialize persistent classes by name if they have
# a non-None non-empty module so as not to have a direct
# ref. This is important when copying. We probably want to
# revisit this in the future.
newargs = getattr(obj, "__getnewargs__", None)
if (isinstance(getattr(klass, '_p_oid', 0), _oidtypes)
and klass.__module__):
# This is a persistent class with a non-empty module. This
# uses pickle format #3 or #7.
klass = klass.__module__, klass.__name__
if newargs is None:
meta = klass, None
else:
meta = klass, newargs()
elif newargs is None:
# Pickle format #1.
meta = klass
else:
# Pickle format #2.
meta = klass, newargs()
return self._dump(meta, obj.__getstate__())
def _dump(self, classmeta, state):
# To reuse the existing BytesIO object, we must reset
# the file position to 0 and truncate the file after the
# new pickle is written.
self._file.seek(0)
self._p.clear_memo()
self._p.dump(classmeta)
self._p.dump(state)
self._file.truncate()
return self._file.getvalue()
def __iter__(self):
return NewObjectIterator(self._stack)
def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
committedData=b''):
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0
klass = 'n/a'
try:
prfactory = PersistentReferenceFactory()
newpickle = self._crs_untransform_record_data(newpickle)
file = BytesIO(newpickle)
unpickler = PersistentUnpickler(
find_global, prfactory.persistent_load, file)
meta = unpickler.load()
if isinstance(meta, tuple):
klass = meta[0]
newargs = meta[1] or ()
if isinstance(klass, tuple):
klass = find_global(*klass)
else:
klass = meta
newargs = ()
if klass in _unresolvable:
raise ConflictError
inst = klass.__new__(klass, *newargs)
try:
resolve = inst._p_resolveConflict
except AttributeError:
_unresolvable[klass] = 1
raise ConflictError
oldData = self.loadSerial(oid, oldSerial)
if not committedData:
committedData = self.loadSerial(oid, committedSerial)
newstate = unpickler.load()
old = state(self, oid, oldSerial, prfactory, oldData)
committed = state(self, oid, committedSerial, prfactory, committedData)
resolved = resolve(old, committed, newstate)
file = BytesIO()
pickler = PersistentPickler(persistent_id, file, _protocol)
pickler.dump(meta)
pickler.dump(resolved)
return self._crs_transform_record_data(file.getvalue())
except (ConflictError, BadClassName) as e:
logger.debug(
"Conflict resolution on %s failed with %s: %s",
klass, e.__class__.__name__, str(e))
except:
# If anything else went wrong, catch it here and avoid passing an
# arbitrary exception back to the client. The error here will mask
# the original ConflictError. A client can recover from a
# ConflictError, but not necessarily from other errors. But log
# the error so that any problems can be fixed.
logger.exception(
"Unexpected error while trying to resolve conflict on %s", klass)
raise ConflictError(oid=oid, serials=(committedSerial, oldSerial),
data=newpickle)