def _setstate(self, obj):
# Helper for setstate(), which provides logging of failures.
# The control flow is complicated here to avoid loading an
# object revision that we are sure we aren't going to use. As
# a result, invalidation tests occur before and after the
# load. We can only be sure about invalidations after the
# load.
# If an object has been invalidated, there are several cases
# to consider:
# 1. Check _p_independent()
# 2. Try MVCC
# 3. Raise ConflictError.
# Does anything actually use _p_independent()? It would simplify
# the code if we could drop support for it.
# (BTrees.Length does.)
# There is a harmless data race with self._invalidated. A
# dict update could go on in another thread, but we don't care
# because we have to check again after the load anyway.
if self._invalidatedCache:
raise ReadConflictError()
if (obj._p_oid in self._invalidated
and not myhasattr(obj, "_p_independent")):
# If the object has _p_independent(), we will handle it below.
self._load_before_or_conflict(obj)
return
p, serial = self._storage.load(obj._p_oid, self._version)
self._load_count += 1
self._inv_lock.acquire()
try:
invalid = obj._p_oid in self._invalidated
finally:
self._inv_lock.release()
if invalid:
if myhasattr(obj, "_p_independent"):
# This call will raise a ReadConflictError if something
# goes wrong
self._handle_independent(obj)
else:
self._load_before_or_conflict(obj)
return
self._reader.setGhostState(obj, p)
obj._p_serial = serial
self._cache.update_object_size_estimation(obj._p_oid, len(p))
obj._p_estimated_size = len(p)
# Blob support
if isinstance(obj, Blob):
obj._p_blob_uncommitted = None
obj._p_blob_committed = self._storage.loadBlob(obj._p_oid, serial)
def _commit(self, transaction):
"""Commit changes to an object"""
if self._import:
# We are importing an export file. We alsways do this
# while making a savepoint so we can copy export data
# directly to our storage, typically a TmpStore.
self._importDuringCommit(transaction, *self._import)
self._import = None
# Just in case an object is added as a side-effect of storing
# a modified object. If, for example, a __getstate__() method
# calls add(), the newly added objects will show up in
# _added_during_commit. This sounds insane, but has actually
# happened.
self._added_during_commit = []
if self._invalidatedCache:
raise ConflictError()
for obj in self._registered_objects:
oid = obj._p_oid
assert oid
if oid in self._conflicts:
raise ReadConflictError(object=obj)
if obj._p_jar is not self:
raise InvalidObjectReference(obj, obj._p_jar)
elif oid in self._added:
assert obj._p_serial == z64
elif obj._p_changed:
if oid in self._invalidated:
resolve = getattr(obj, "_p_resolveConflict", None)
if resolve is None:
raise ConflictError(object=obj)
self._modified.append(oid)
else:
# Nothing to do. It's been said that it's legal, e.g., for
# an object to set _p_changed to false after it's been
# changed and registered.
continue
self._store_objects(ObjectWriter(obj), transaction)
for obj in self._added_during_commit:
self._store_objects(ObjectWriter(obj), transaction)
self._added_during_commit = None
def _handle_independent(self, obj):
# Helper method for setstate() handles possibly independent objects
# Call _p_independent(), if it returns True, setstate() wins.
# Otherwise, raise a ConflictError.
if obj._p_independent():
self._inv_lock.acquire()
try:
try:
self._invalidated.remove(obj._p_oid)
except KeyError:
pass
finally:
self._inv_lock.release()
else:
self._conflicts[obj._p_oid] = 1
self._register(obj)
raise ReadConflictError(object=obj)
def _check_tid_after_load(self,
oid_int,
actual_tid_int,
expect_tid_int=None):
"""Verify the tid of an object loaded from the database is sane."""
if actual_tid_int > self.current_tid:
# Strangely, the database just gave us data from a future
# transaction. We can't give the data to ZODB because that
# would be a consistency violation. However, the cause is
# hard to track down, so issue a ReadConflictError and
# hope that the application retries successfully.
msg = ("Got data for OID 0x%(oid_int)x from "
"future transaction %(actual_tid_int)d (%(got_ts)s). "
"Current transaction is %(current_tid)d (%(current_ts)s)." %
{
'oid_int': oid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'got_ts': str(TimeStamp(p64(actual_tid_int))),
'current_ts': str(TimeStamp(p64(self.current_tid))),
})
raise ReadConflictError(msg)
if expect_tid_int is not None and actual_tid_int != expect_tid_int:
# Uh-oh, the cache is inconsistent with the database.
# We didn't get a TID from the future, but it's not what we
# had in our delta_after0 map, which means...we missed a change
# somewhere.
#
# Possible causes:
#
# - The database MUST provide a snapshot view for each
# session; this error can occur if that requirement is
# violated. For example, MySQL's MyISAM engine is not
# sufficient for the object_state table because MyISAM
# can not provide a snapshot view. (InnoDB is
# sufficient.)
#
# - (Similar to the last one.) Using too low of a
# isolation level for the database connection and
# viewing unrelated data.
#
# - Something could be writing to the database out
# of order, such as a version of RelStorage that
# acquires a different commit lock.
#
# - A software bug. In the past, there was a subtle bug
# in after_poll() that caused it to ignore the
# transaction order, leading it to sometimes put the
# wrong tid in delta_after*.
#
# - Restarting a load connection at a future point we hadn't
# actually polled to, such that our current_tid is out of sync
# with the connection's *actual* viewable tid?
cp0, cp1 = self.checkpoints
msg = ("Detected an inconsistency "
"between the RelStorage cache and the database "
"while loading an object using the delta_after0 dict. "
"Please verify the database is configured for "
"ACID compliance and that all clients are using "
"the same commit lock. "
"(oid_int=%(oid_int)r, expect_tid_int=%(expect_tid_int)r, "
"actual_tid_int=%(actual_tid_int)r, "
"current_tid=%(current_tid)r, cp0=%(cp0)r, cp1=%(cp1)r, "
"len(delta_after0)=%(lda0)r, len(delta_after1)=%(lda1)r, "
"pid=%(pid)r, thread_ident=%(thread_ident)r)" % {
'oid_int': oid_int,
'expect_tid_int': expect_tid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'cp0': cp0,
'cp1': cp1,
'lda0': len(self.delta_after0),
'lda1': len(self.delta_after1),
'pid': os.getpid(),
'thread_ident': threading.current_thread(),
})
# We reset ourself as if we hadn't polled, and hope the transient
# error gets retried in a working, consistent view.
self._reset(msg)
def _check_tid_after_load(self,
oid_int,
actual_tid_int,
expect_tid_int=None):
"""Verify the tid of an object loaded from the database is sane."""
if actual_tid_int > self.current_tid:
# Strangely, the database just gave us data from a future
# transaction. We can't give the data to ZODB because that
# would be a consistency violation. However, the cause is hard
# to track down, so issue a ReadConflictError and hope that
# the application retries successfully.
raise ReadConflictError(
"Got data for OID 0x%(oid_int)x from "
"future transaction %(actual_tid_int)d (%(got_ts)s). "
"Current transaction is %(current_tid)d (%(current_ts)s)." % {
'oid_int': oid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'got_ts': str(TimeStamp(p64(actual_tid_int))),
'current_ts': str(TimeStamp(p64(self.current_tid))),
})
if expect_tid_int is not None and actual_tid_int != expect_tid_int:
# Uh-oh, the cache is inconsistent with the database.
# Possible causes:
#
# - The database MUST provide a snapshot view for each
# session; this error can occur if that requirement is
# violated. For example, MySQL's MyISAM engine is not
# sufficient for the object_state table because MyISAM
# can not provide a snapshot view. (InnoDB is
# sufficient.)
#
# - Something could be writing to the database out
# of order, such as a version of RelStorage that
# acquires a different commit lock.
#
# - A software bug. In the past, there was a subtle bug
# in after_poll() that caused it to ignore the
# transaction order, leading it to sometimes put the
# wrong tid in delta_after*.
cp0, cp1 = self.checkpoints
import os
import thread
raise AssertionError(
"Detected an inconsistency "
"between the RelStorage cache and the database "
"while loading an object using the delta_after0 dict. "
"Please verify the database is configured for "
"ACID compliance and that all clients are using "
"the same commit lock. "
"(oid_int=%(oid_int)r, expect_tid_int=%(expect_tid_int)r, "
"actual_tid_int=%(actual_tid_int)r, "
"current_tid=%(current_tid)r, cp0=%(cp0)r, cp1=%(cp1)r, "
"len(delta_after0)=%(lda0)r, len(delta_after1)=%(lda1)r, "
"pid=%(pid)r, thread_ident=%(thread_ident)r)" % {
'oid_int': oid_int,
'expect_tid_int': expect_tid_int,
'actual_tid_int': actual_tid_int,
'current_tid': self.current_tid,
'cp0': cp0,
'cp1': cp1,
'lda0': len(self.delta_after0),
'lda1': len(self.delta_after1),
'pid': os.getpid(),
'thread_ident': thread.get_ident(),
})
def poll_invalidations(self, conn, cursor, prev_polled_tid, ignore_tid):
"""Polls for new transactions.
conn and cursor must have been created previously by open_for_load().
prev_polled_tid is the tid returned at the last poll, or None
if this is the first poll. If ignore_tid is not None, changes
committed in that transaction will not be included in the list
of changed OIDs.
Returns (changes, new_polled_tid), where changes is either
a list of (oid, tid) that have changed, or None to indicate
that the changes are too complex to list. new_polled_tid can be
0 if there is no data in the database.
"""
# find out the tid of the most recent transaction.
cursor.execute(self.poll_query)
rows = list(cursor)
if not rows:
# No data.
return None, 0
new_polled_tid = rows[0][0]
if not new_polled_tid:
# No data.
return None, 0
if prev_polled_tid is None:
# This is the first time the connection has polled.
return None, new_polled_tid
if new_polled_tid == prev_polled_tid:
# No transactions have been committed since prev_polled_tid.
return (), new_polled_tid
elif new_polled_tid > prev_polled_tid:
# New transaction(s) have been added.
if self.keep_history:
# If the previously polled transaction no longer exists,
# the cache is too old and needs to be cleared.
# XXX Do we actually need to detect this condition? I think
# if we delete this block of code, all the unreachable
# objects will be garbage collected anyway. So, as a test,
# there is no equivalent of this block of code for
# history-free storage. If something goes wrong, then we'll
# know there's some other edge condition we have to account
# for.
stmt = "SELECT 1 FROM transaction WHERE tid = %(tid)s"
cursor.execute(
intern(stmt % self.runner.script_vars),
{'tid': prev_polled_tid})
rows = cursor.fetchall()
if not rows:
# Transaction not found; perhaps it has been packed.
# The connection cache should be cleared.
return None, new_polled_tid
# Get the list of changed OIDs and return it.
if self.keep_history:
stmt = """
SELECT zoid, tid
FROM current_object
WHERE tid > %(tid)s
"""
else:
stmt = """
SELECT zoid, tid
FROM object_state
WHERE tid > %(tid)s
"""
params = {'tid': prev_polled_tid}
if ignore_tid is not None:
stmt += " AND tid != %(self_tid)s"
params['self_tid'] = ignore_tid
stmt = intern(stmt % self.runner.script_vars)
cursor.execute(stmt, params)
changes = cursor.fetchall()
return changes, new_polled_tid
else:
# The database connection is stale. This can happen after
# reading an asynchronous slave that is not fully up to date.
# (It may also suggest that transaction IDs are not being created
# in order, which would be a serious bug leading to consistency
# violations.)
if self.revert_when_stale:
# This client prefers to revert to the old state.
log.warning(
"Reverting to stale transaction ID %d and clearing cache. "
"(prev_polled_tid=%d)",
new_polled_tid, prev_polled_tid)
# We have to invalidate the whole cPickleCache, otherwise
# the cache would be inconsistent with the reverted state.
return None, new_polled_tid
else:
# This client never wants to revert to stale data, so
# raise ReadConflictError to trigger a retry.
# We're probably just waiting for async replication
# to catch up, so retrying could do the trick.
raise ReadConflictError(
"The database connection is stale: new_polled_tid=%d, "
"prev_polled_tid=%d." % (new_polled_tid, prev_polled_tid))
def _handleConflicts(self, txn_context, tryToResolveConflict):
result = []
append = result.append
# Check for conflicts
data_dict = txn_context['data_dict']
object_base_serial_dict = txn_context['object_base_serial_dict']
object_serial_dict = txn_context['object_serial_dict']
conflict_serial_dict = txn_context['conflict_serial_dict'].copy()
txn_context['conflict_serial_dict'].clear()
resolved_conflict_serial_dict = txn_context[
'resolved_conflict_serial_dict']
for oid, conflict_serial_set in conflict_serial_dict.iteritems():
conflict_serial = max(conflict_serial_set)
serial = object_serial_dict[oid]
if ZERO_TID in conflict_serial_set:
if 1:
# XXX: disable deadlock avoidance code until it is fixed
logging.info('Deadlock avoidance on %r:%r', dump(oid),
dump(serial))
# 'data' parameter of ConflictError is only used to report the
# class of the object. It doesn't matter if 'data' is None
# because the transaction is too big.
try:
data = data_dict[oid]
except KeyError:
data = txn_context['cache_dict'][oid]
else:
# Storage refused us from taking object lock, to avoid a
# possible deadlock. TID is actually used for some kind of
# "locking priority": when a higher value has the lock,
# this means we stored objects "too late", and we would
# otherwise cause a deadlock.
# To recover, we must ask storages to release locks we
# hold (to let possibly-competing transactions acquire
# them), and requeue our already-sent store requests.
# XXX: currently, brute-force is implemented: we send
# object data again.
# WARNING: not maintained code
logging.info('Deadlock avoidance triggered on %r:%r',
dump(oid), dump(serial))
for store_oid, store_data in data_dict.iteritems():
store_serial = object_serial_dict[store_oid]
if store_data is CHECKED_SERIAL:
self._checkCurrentSerialInTransaction(
txn_context, store_oid, store_serial)
else:
if store_data is None:
# Some undo
logging.warning(
'Deadlock avoidance cannot reliably'
' work with undo, this must be implemented.'
)
conflict_serial = ZERO_TID
break
self._store(txn_context,
store_oid,
store_serial,
store_data,
unlock=True)
else:
continue
else:
data = data_dict.pop(oid)
if data is CHECKED_SERIAL:
raise ReadConflictError(oid=oid,
serials=(conflict_serial, serial))
# TODO: data can be None if a conflict happens during undo
if data:
txn_context['data_size'] -= len(data)
resolved_serial_set = resolved_conflict_serial_dict.setdefault(
oid, set())
if resolved_serial_set and conflict_serial <= max(
resolved_serial_set):
# A later serial has already been resolved, skip.
resolved_serial_set.update(conflict_serial_set)
continue
try:
new_data = tryToResolveConflict(oid, conflict_serial,
serial, data)
except ConflictError:
logging.info(
'Conflict resolution failed for '
'%r:%r with %r', dump(oid), dump(serial),
dump(conflict_serial))
else:
logging.info(
'Conflict resolution succeeded for '
'%r:%r with %r', dump(oid), dump(serial),
dump(conflict_serial))
# Mark this conflict as resolved
resolved_serial_set.update(conflict_serial_set)
# Base serial changes too, as we resolved a conflict
object_base_serial_dict[oid] = conflict_serial
# Try to store again
self._store(txn_context, oid, conflict_serial, new_data)
append(oid)
continue
# With recent ZODB, get_pickle_metadata (from ZODB.utils) does
# not support empty values, so do not pass 'data' in this case.
raise ConflictError(oid=oid,
serials=(conflict_serial, serial),
data=data or None)
return result
def setstate(self, obj):
oid = obj._p_oid
self.before_load()
try:
p, serial = self._storage.load(oid, self._version)
self._load_count = self._load_count + 1
# XXX this is quite conservative!
# We need, however, to avoid reading data from a transaction
# that committed after the current "session" started, as
# that might lead to mixing of cached data from earlier
# transactions and new inconsistent data.
#
# Note that we (carefully) wait until after we call the
# storage to make sure that we don't miss an invaildation
# notifications between the time we check and the time we
# read.
#invalid = self._invalidated.get
invalid = self._invalidated.__contains__
if invalid(oid) or invalid(None):
if not hasattr(obj.__class__, '_p_independent'):
transaction.get().register(self)
raise ReadConflictError(object=obj)
invalid = 1
else:
invalid = 0
file = StringIO(p)
unpickler = Unpickler(file)
# SDH: external references are reassembled elsewhere.
# unpickler.persistent_load=self._persistent_load
classification = unpickler.load()
state = unpickler.load()
# SDH: Let the object mapper do the state setting.
# if hasattr(object, '__setstate__'):
# object.__setstate__(state)
# else:
# d=object.__dict__
# for k,v in state.items(): d[k]=v
osio = self._get_osio()
event = osio.deserialize(oid, obj, classification, state)
if event.upos:
self._handle_unmanaged(obj, event.upos)
self._set_serial(obj, serial)
if invalid:
if obj._p_independent():
try:
self._invalidated.remove(oid)
except KeyError:
pass
else:
transaction.get().register(self)
raise ConflictError(object=obj)
except ConflictError:
raise
except:
LOG('ZODB',
ERROR,
"Couldn't load state for %s" % ` oid `,
error=sys.exc_info())
raise
def _handleConflicts(self, txn_context):
data_dict = txn_context.data_dict
pop_conflict = txn_context.conflict_dict.popitem
resolved_dict = txn_context.resolved_dict
tryToResolveConflict = txn_context.Storage.tryToResolveConflict
while 1:
# We iterate over conflict_dict, and clear it,
# because new items may be added by calls to _store.
# This is also done atomically, to avoid race conditions
# with PrimaryNotificationsHandler.notifyDeadlock
try:
oid, serial = pop_conflict()
except KeyError:
return
try:
data, old_serial, _ = data_dict.pop(oid)
except KeyError:
assert oid is None, (oid, serial)
# Storage refused us from taking object lock, to avoid a
# possible deadlock. TID is actually used for some kind of
# "locking priority": when a higher value has the lock,
# this means we stored objects "too late", and we would
# otherwise cause a deadlock.
# To recover, we must ask storages to release locks we
# hold (to let possibly-competing transactions acquire
# them), and requeue our already-sent store requests.
ttid = txn_context.ttid
logging.info('Deadlock avoidance triggered for TXN %s'
' with new locking TID %s', dump(ttid), dump(serial))
txn_context.locking_tid = serial
packet = Packets.AskRebaseTransaction(ttid, serial)
for uuid in txn_context.conn_dict:
self._askStorageForWrite(txn_context, uuid, packet)
else:
if data is CHECKED_SERIAL:
raise ReadConflictError(oid=oid,
serials=(serial, old_serial))
# TODO: data can be None if a conflict happens during undo
if data:
txn_context.data_size -= len(data)
if self.last_tid < serial:
self.sync() # possible late invalidation (very rare)
try:
data = tryToResolveConflict(oid, serial, old_serial, data)
except ConflictError:
logging.info(
'Conflict resolution failed for %s@%s with %s',
dump(oid), dump(old_serial), dump(serial))
# With recent ZODB, get_pickle_metadata (from ZODB.utils)
# does not support empty values, so do not pass 'data'
# in this case.
raise ConflictError(oid=oid, serials=(serial, old_serial),
data=data or None)
else:
logging.info(
'Conflict resolution succeeded for %s@%s with %s',
dump(oid), dump(old_serial), dump(serial))
# Mark this conflict as resolved
resolved_dict[oid] = serial
# Try to store again
self._store(txn_context, oid, serial, data)
def _load_before_or_conflict(self, obj):
"""Load non-current state for obj or raise ReadConflictError."""
if not ((not self._version) and self._setstate_noncurrent(obj)):
self._register(obj)
self._conflicts[obj._p_oid] = True
raise ReadConflictError(object=obj)
