def list_changes(self, cursor, after_tid, last_tid):
"""Return the (oid, tid) values changed in a range of transactions.
The returned iterable must include the latest changes in the range
after_tid < tid <= last_tid.
"""
if self.keep_history:
stmt = """
SELECT zoid, tid
FROM current_object
WHERE tid > %(min_tid)s
AND tid <= %(max_tid)s
"""
else:
stmt = """
SELECT zoid, tid
FROM object_state
WHERE tid > %(min_tid)s
AND tid <= %(max_tid)s
"""
params = {'min_tid': after_tid, 'max_tid': last_tid}
stmt = intern(stmt % self.runner.script_vars)
cursor.execute(stmt, params)
return cursor.fetchall()
def __dict_to_tuple(self, sql, args):
"""
Transform a dict-format expression into the equivalent
tuple version.
Caches statements. We know we only use a small number of small
hard coded strings.
"""
try:
# This is racy, but it's idempotent
tuple_sql, itemgetter = _dict_cache[sql]
except KeyError:
dict_exprs = param_match.findall(sql)
if not dict_exprs:
tuple_sql = sql
itemgetter = lambda d: ()
else:
itemgetter = operator.itemgetter(*[dict_expr[2:-2] for dict_expr in dict_exprs])
if len(dict_exprs) == 1:
_itemgetter = itemgetter
itemgetter = lambda d: (_itemgetter(d),)
tuple_sql = param_match.sub('%s', sql)
tuple_sql = intern(tuple_sql)
_dict_cache[sql] = (tuple_sql, itemgetter)
return tuple_sql, itemgetter(args)
def __dict_to_tuple(self, sql, args):
"""
Transform a dict-format expression into the equivalent
tuple version.
Caches statements. We know we only use a small number of small
hard coded strings.
"""
try:
# This is racy, but it's idempotent
tuple_sql, itemgetter = _dict_cache[sql]
except KeyError:
dict_exprs = param_match.findall(sql)
if not dict_exprs:
tuple_sql = sql
itemgetter = lambda d: ()
else:
itemgetter = operator.itemgetter(
*[dict_expr[2:-2] for dict_expr in dict_exprs])
if len(dict_exprs) == 1:
_itemgetter = itemgetter
itemgetter = lambda d: (_itemgetter(d), )
tuple_sql = param_match.sub('%s', sql)
tuple_sql = intern(tuple_sql)
_dict_cache[sql] = (tuple_sql, itemgetter)
return tuple_sql, itemgetter(args)
def __init__(self, poll_query, keep_history, runner, revert_when_stale):
self.poll_query = poll_query
self.keep_history = keep_history
self.runner = runner
self.revert_when_stale = revert_when_stale
list_changes_range_stmt = self._HP_LIST_CHANGES_RANGE if self.keep_history else self._HF_LIST_CHANGES_RANGE
self._list_changes_range_stmt = intern(list_changes_range_stmt % self.runner.script_vars)
poll_inv_stmt = self._HP_POLL_INV if self.keep_history else self._HF_POLL_INV
poll_inv_stmt_exc = poll_inv_stmt + " AND tid != %(self_tid)s"
self._poll_inv_stmt = intern(poll_inv_stmt % self.runner.script_vars)
self._poll_inv_stmt_exc = intern(poll_inv_stmt_exc % self.runner.script_vars)
self._tran_exists_stmt = intern(self._HP_TRAN_EXISTS % self.runner.script_vars)
def prop(inst):
queries = getattr(inst, base_name + '_queries')
query = queries[0] if inst.keep_history else queries[1]
if isinstance(query, Exception):
raise query
if extension:
query = query + extension
if formatted:
query = intern(query % inst.runner.script_vars)
return query
def run_many(self, cursor, stmt, items):
"""Execute a statement repeatedly. Items should be a list of tuples.
stmt should use '%s' parameter format.
"""
# replace '%s' with ':n'
matches = []
def replace(_match):
matches.append(None)
return ':%d' % len(matches)
stmt = intern(re.sub('%s', replace, stmt))
cursor.executemany(stmt, items)
def format_to_named(stmt):
"""
Convert '%s' pyformat strings to :n numbered
strings. Intended only for static strings.
"""
try:
return _stmt_cache[stmt]
except KeyError:
matches = []
def replace(_match):
matches.append(None)
return ':%d' % len(matches)
new_stmt = intern(re.sub('%s', replace, stmt))
_stmt_cache[stmt] = new_stmt
return new_stmt
def _format_to_named(stmt):
"""
Convert '%s' pyformat strings to :n numbered
strings. Intended only for static strings.
This is legacy. Replace strings that use this with SQL statements
constructed from the schema.
"""
import re
from relstorage._compat import intern
try:
return _stmt_cache[stmt]
except KeyError:
matches = []
def replace(_match):
matches.append(None)
return ':%d' % len(matches)
new_stmt = intern(re.sub('%s', replace, stmt))
_stmt_cache[stmt] = new_stmt
return new_stmt
def poll_invalidations(self, conn, cursor, prev_polled_tid, ignore_tid):
"""Polls for new transactions.
conn and cursor must have been created previously by open_for_load().
prev_polled_tid is the tid returned at the last poll, or None
if this is the first poll. If ignore_tid is not None, changes
committed in that transaction will not be included in the list
of changed OIDs.
Returns (changes, new_polled_tid), where changes is either
a list of (oid, tid) that have changed, or None to indicate
that the changes are too complex to list. new_polled_tid can be
0 if there is no data in the database.
"""
# find out the tid of the most recent transaction.
cursor.execute(self.poll_query)
rows = list(cursor)
if not rows:
# No data.
return None, 0
new_polled_tid = rows[0][0]
if not new_polled_tid:
# No data.
return None, 0
if prev_polled_tid is None:
# This is the first time the connection has polled.
return None, new_polled_tid
if new_polled_tid == prev_polled_tid:
# No transactions have been committed since prev_polled_tid.
return (), new_polled_tid
elif new_polled_tid > prev_polled_tid:
# New transaction(s) have been added.
if self.keep_history:
# If the previously polled transaction no longer exists,
# the cache is too old and needs to be cleared.
# XXX Do we actually need to detect this condition? I think
# if we delete this block of code, all the unreachable
# objects will be garbage collected anyway. So, as a test,
# there is no equivalent of this block of code for
# history-free storage. If something goes wrong, then we'll
# know there's some other edge condition we have to account
# for.
stmt = "SELECT 1 FROM transaction WHERE tid = %(tid)s"
cursor.execute(
intern(stmt % self.runner.script_vars),
{'tid': prev_polled_tid})
rows = cursor.fetchall()
if not rows:
# Transaction not found; perhaps it has been packed.
# The connection cache should be cleared.
return None, new_polled_tid
# Get the list of changed OIDs and return it.
if self.keep_history:
stmt = """
SELECT zoid, tid
FROM current_object
WHERE tid > %(tid)s
"""
else:
stmt = """
SELECT zoid, tid
FROM object_state
WHERE tid > %(tid)s
"""
params = {'tid': prev_polled_tid}
if ignore_tid is not None:
stmt += " AND tid != %(self_tid)s"
params['self_tid'] = ignore_tid
stmt = intern(stmt % self.runner.script_vars)
cursor.execute(stmt, params)
changes = cursor.fetchall()
return changes, new_polled_tid
else:
# The database connection is stale. This can happen after
# reading an asynchronous slave that is not fully up to date.
# (It may also suggest that transaction IDs are not being created
# in order, which would be a serious bug leading to consistency
# violations.)
if self.revert_when_stale:
# This client prefers to revert to the old state.
log.warning(
"Reverting to stale transaction ID %d and clearing cache. "
"(prev_polled_tid=%d)",
new_polled_tid, prev_polled_tid)
# We have to invalidate the whole cPickleCache, otherwise
# the cache would be inconsistent with the reverted state.
return None, new_polled_tid
else:
# This client never wants to revert to stale data, so
# raise ReadConflictError to trigger a retry.
# We're probably just waiting for async replication
# to catch up, so retrying could do the trick.
raise ReadConflictError(
"The database connection is stale: new_polled_tid=%d, "
"prev_polled_tid=%d." % (new_polled_tid, prev_polled_tid))
def finalize(self):
return intern(self.buf.getvalue().strip()), {v: k for k, v in self.placeholders.items()}
def prepare(self):
# This is correct for PostgreSQL. This needs moved to a dialect specific
# spot.
datatypes = self._find_datatypes_for_prepared_query()
query = self.buf.getvalue()
name = self._next_prepared_stmt_name(query)
if datatypes:
assert isinstance(datatypes, (list, tuple))
datatypes = ', '.join(datatypes)
datatypes = ' (%s)' % (datatypes,)
else:
datatypes = ''
q = query.strip()
# PREPARE needs the query string to use $1, $2, $3, etc,
# as placeholders.
# In MySQL, it's a plain question mark.
placeholder_to_number = {}
counter = 0
for placeholder_name in self.placeholders.values():
counter += 1
placeholder = self._placeholder(placeholder_name)
placeholder_to_number[placeholder_name] = counter
param = self._prepared_param(counter)
q = q.replace(placeholder, param, 1)
q = self._quote_query_for_prepare(q)
stmt = 'PREPARE {name}{datatypes} {conjunction} {query}'.format(
name=name, datatypes=datatypes,
query=q,
conjunction=self._PREPARED_CONJUNCTION,
)
if placeholder_to_number:
execute = 'EXECUTE {name}({params})'.format(
name=name,
params=','.join(['%s'] * len(self.placeholders)),
)
else:
# Neither MySQL nor PostgreSQL like a set of empty parens: ()
execute = 'EXECUTE {name}'.format(name=name)
if '%s' in placeholder_to_number:
# There was an ordered param. If there was one,
# they must all be ordered, so there's no need to convert anything.
assert len(placeholder_to_number) == 1
def convert(p):
return p
else:
def convert(d):
# TODO: This may not actually be needed, since we issue a regular
# cursor.execute(), it may be able to handle named?
params = [None] * len(placeholder_to_number)
for placeholder_name, ix in placeholder_to_number.items():
params[ix - 1] = d[placeholder_name]
return params
return intern(stmt), intern(execute), convert
