def list_changes(self, cursor, after_tid, last_tid):
"""Return the (oid, tid) values changed in a range of transactions.
The returned iterable must include the latest changes in the range
after_tid < tid <= last_tid.
"""
if self.keep_history:
stmt = """
SELECT zoid, tid
FROM current_object
WHERE tid > %(min_tid)s
AND tid <= %(max_tid)s
"""
else:
stmt = """
SELECT zoid, tid
FROM object_state
WHERE tid > %(min_tid)s
AND tid <= %(max_tid)s
"""
params = {'min_tid': after_tid, 'max_tid': last_tid}
stmt = intern(stmt % self.runner.script_vars)
cursor.execute(stmt, params)
return cursor.fetchall()
def run_many(self, cursor, stmt, items):
"""Execute a statement repeatedly. Items should be a list of tuples.
stmt should use '%s' parameter format.
"""
# replace '%s' with ':n'
matches = []
def replace(match):
matches.append(None)
return ':%d' % len(matches)
stmt = intern(re.sub('%s', replace, stmt))
cursor.executemany(stmt, items)
def run_many(self, cursor, stmt, items):
"""Execute a statement repeatedly. Items should be a list of tuples.
stmt should use '%s' parameter format.
"""
# replace '%s' with ':n'
matches = []
def replace(match):
matches.append(None)
return ":%d" % len(matches)
stmt = intern(re.sub("%s", replace, stmt))
cursor.executemany(stmt, items)
def poll_invalidations(self, conn, cursor, prev_polled_tid, ignore_tid):
"""Polls for new transactions.
conn and cursor must have been created previously by open_for_load().
prev_polled_tid is the tid returned at the last poll, or None
if this is the first poll. If ignore_tid is not None, changes
committed in that transaction will not be included in the list
of changed OIDs.
Returns (changes, new_polled_tid), where changes is either
a list of (oid, tid) that have changed, or None to indicate
that the changes are too complex to list. new_polled_tid can be
0 if there is no data in the database.
"""
# find out the tid of the most recent transaction.
cursor.execute(self.poll_query)
rows = list(cursor)
if not rows:
# No data.
return None, 0
new_polled_tid = rows[0][0]
if not new_polled_tid:
# No data.
return None, 0
if prev_polled_tid is None:
# This is the first time the connection has polled.
return None, new_polled_tid
if new_polled_tid == prev_polled_tid:
# No transactions have been committed since prev_polled_tid.
return (), new_polled_tid
elif new_polled_tid > prev_polled_tid:
# New transaction(s) have been added.
if self.keep_history:
# If the previously polled transaction no longer exists,
# the cache is too old and needs to be cleared.
# XXX Do we actually need to detect this condition? I think
# if we delete this block of code, all the unreachable
# objects will be garbage collected anyway. So, as a test,
# there is no equivalent of this block of code for
# history-free storage. If something goes wrong, then we'll
# know there's some other edge condition we have to account
# for.
stmt = "SELECT 1 FROM transaction WHERE tid = %(tid)s"
cursor.execute(
intern(stmt % self.runner.script_vars),
{'tid': prev_polled_tid})
rows = cursor.fetchall()
if not rows:
# Transaction not found; perhaps it has been packed.
# The connection cache should be cleared.
return None, new_polled_tid
# Get the list of changed OIDs and return it.
if self.keep_history:
stmt = """
SELECT zoid, tid
FROM current_object
WHERE tid > %(tid)s
"""
else:
stmt = """
SELECT zoid, tid
FROM object_state
WHERE tid > %(tid)s
"""
params = {'tid': prev_polled_tid}
if ignore_tid is not None:
stmt += " AND tid != %(self_tid)s"
params['self_tid'] = ignore_tid
stmt = intern(stmt % self.runner.script_vars)
cursor.execute(stmt, params)
changes = cursor.fetchall()
return changes, new_polled_tid
else:
# The database connection is stale. This can happen after
# reading an asynchronous slave that is not fully up to date.
# (It may also suggest that transaction IDs are not being created
# in order, which would be a serious bug leading to consistency
# violations.)
if self.revert_when_stale:
# This client prefers to revert to the old state.
log.warning(
"Reverting to stale transaction ID %d and clearing cache. "
"(prev_polled_tid=%d)",
new_polled_tid, prev_polled_tid)
# We have to invalidate the whole cPickleCache, otherwise
# the cache would be inconsistent with the reverted state.
return None, new_polled_tid
else:
# This client never wants to revert to stale data, so
# raise ReadConflictError to trigger a retry.
# We're probably just waiting for async replication
# to catch up, so retrying could do the trick.
raise ReadConflictError(
"The database connection is stale: new_polled_tid=%d, "
"prev_polled_tid=%d." % (new_polled_tid, prev_polled_tid))
