def list_changes(self, cursor, after_tid, last_tid): """Return the (oid, tid) values changed in a range of transactions. The returned iterable must include the latest changes in the range after_tid < tid <= last_tid. """ if self.keep_history: stmt = """ SELECT zoid, tid FROM current_object WHERE tid > %(min_tid)s AND tid <= %(max_tid)s """ else: stmt = """ SELECT zoid, tid FROM object_state WHERE tid > %(min_tid)s AND tid <= %(max_tid)s """ params = {'min_tid': after_tid, 'max_tid': last_tid} stmt = intern(stmt % self.runner.script_vars) cursor.execute(stmt, params) return cursor.fetchall()
def __dict_to_tuple(self, sql, args): """ Transform a dict-format expression into the equivalent tuple version. Caches statements. We know we only use a small number of small hard coded strings. """ try: # This is racy, but it's idempotent tuple_sql, itemgetter = _dict_cache[sql] except KeyError: dict_exprs = param_match.findall(sql) if not dict_exprs: tuple_sql = sql itemgetter = lambda d: () else: itemgetter = operator.itemgetter(*[dict_expr[2:-2] for dict_expr in dict_exprs]) if len(dict_exprs) == 1: _itemgetter = itemgetter itemgetter = lambda d: (_itemgetter(d),) tuple_sql = param_match.sub('%s', sql) tuple_sql = intern(tuple_sql) _dict_cache[sql] = (tuple_sql, itemgetter) return tuple_sql, itemgetter(args)
def __dict_to_tuple(self, sql, args): """ Transform a dict-format expression into the equivalent tuple version. Caches statements. We know we only use a small number of small hard coded strings. """ try: # This is racy, but it's idempotent tuple_sql, itemgetter = _dict_cache[sql] except KeyError: dict_exprs = param_match.findall(sql) if not dict_exprs: tuple_sql = sql itemgetter = lambda d: () else: itemgetter = operator.itemgetter( *[dict_expr[2:-2] for dict_expr in dict_exprs]) if len(dict_exprs) == 1: _itemgetter = itemgetter itemgetter = lambda d: (_itemgetter(d), ) tuple_sql = param_match.sub('%s', sql) tuple_sql = intern(tuple_sql) _dict_cache[sql] = (tuple_sql, itemgetter) return tuple_sql, itemgetter(args)
def __init__(self, poll_query, keep_history, runner, revert_when_stale): self.poll_query = poll_query self.keep_history = keep_history self.runner = runner self.revert_when_stale = revert_when_stale list_changes_range_stmt = self._HP_LIST_CHANGES_RANGE if self.keep_history else self._HF_LIST_CHANGES_RANGE self._list_changes_range_stmt = intern(list_changes_range_stmt % self.runner.script_vars) poll_inv_stmt = self._HP_POLL_INV if self.keep_history else self._HF_POLL_INV poll_inv_stmt_exc = poll_inv_stmt + " AND tid != %(self_tid)s" self._poll_inv_stmt = intern(poll_inv_stmt % self.runner.script_vars) self._poll_inv_stmt_exc = intern(poll_inv_stmt_exc % self.runner.script_vars) self._tran_exists_stmt = intern(self._HP_TRAN_EXISTS % self.runner.script_vars)
def prop(inst): queries = getattr(inst, base_name + '_queries') query = queries[0] if inst.keep_history else queries[1] if isinstance(query, Exception): raise query if extension: query = query + extension if formatted: query = intern(query % inst.runner.script_vars) return query
def run_many(self, cursor, stmt, items): """Execute a statement repeatedly. Items should be a list of tuples. stmt should use '%s' parameter format. """ # replace '%s' with ':n' matches = [] def replace(_match): matches.append(None) return ':%d' % len(matches) stmt = intern(re.sub('%s', replace, stmt)) cursor.executemany(stmt, items)
def format_to_named(stmt): """ Convert '%s' pyformat strings to :n numbered strings. Intended only for static strings. """ try: return _stmt_cache[stmt] except KeyError: matches = [] def replace(_match): matches.append(None) return ':%d' % len(matches) new_stmt = intern(re.sub('%s', replace, stmt)) _stmt_cache[stmt] = new_stmt return new_stmt
def _format_to_named(stmt): """ Convert '%s' pyformat strings to :n numbered strings. Intended only for static strings. This is legacy. Replace strings that use this with SQL statements constructed from the schema. """ import re from relstorage._compat import intern try: return _stmt_cache[stmt] except KeyError: matches = [] def replace(_match): matches.append(None) return ':%d' % len(matches) new_stmt = intern(re.sub('%s', replace, stmt)) _stmt_cache[stmt] = new_stmt return new_stmt
def poll_invalidations(self, conn, cursor, prev_polled_tid, ignore_tid): """Polls for new transactions. conn and cursor must have been created previously by open_for_load(). prev_polled_tid is the tid returned at the last poll, or None if this is the first poll. If ignore_tid is not None, changes committed in that transaction will not be included in the list of changed OIDs. Returns (changes, new_polled_tid), where changes is either a list of (oid, tid) that have changed, or None to indicate that the changes are too complex to list. new_polled_tid can be 0 if there is no data in the database. """ # find out the tid of the most recent transaction. cursor.execute(self.poll_query) rows = list(cursor) if not rows: # No data. return None, 0 new_polled_tid = rows[0][0] if not new_polled_tid: # No data. return None, 0 if prev_polled_tid is None: # This is the first time the connection has polled. return None, new_polled_tid if new_polled_tid == prev_polled_tid: # No transactions have been committed since prev_polled_tid. return (), new_polled_tid elif new_polled_tid > prev_polled_tid: # New transaction(s) have been added. if self.keep_history: # If the previously polled transaction no longer exists, # the cache is too old and needs to be cleared. # XXX Do we actually need to detect this condition? I think # if we delete this block of code, all the unreachable # objects will be garbage collected anyway. So, as a test, # there is no equivalent of this block of code for # history-free storage. If something goes wrong, then we'll # know there's some other edge condition we have to account # for. stmt = "SELECT 1 FROM transaction WHERE tid = %(tid)s" cursor.execute( intern(stmt % self.runner.script_vars), {'tid': prev_polled_tid}) rows = cursor.fetchall() if not rows: # Transaction not found; perhaps it has been packed. # The connection cache should be cleared. return None, new_polled_tid # Get the list of changed OIDs and return it. if self.keep_history: stmt = """ SELECT zoid, tid FROM current_object WHERE tid > %(tid)s """ else: stmt = """ SELECT zoid, tid FROM object_state WHERE tid > %(tid)s """ params = {'tid': prev_polled_tid} if ignore_tid is not None: stmt += " AND tid != %(self_tid)s" params['self_tid'] = ignore_tid stmt = intern(stmt % self.runner.script_vars) cursor.execute(stmt, params) changes = cursor.fetchall() return changes, new_polled_tid else: # The database connection is stale. This can happen after # reading an asynchronous slave that is not fully up to date. # (It may also suggest that transaction IDs are not being created # in order, which would be a serious bug leading to consistency # violations.) if self.revert_when_stale: # This client prefers to revert to the old state. log.warning( "Reverting to stale transaction ID %d and clearing cache. " "(prev_polled_tid=%d)", new_polled_tid, prev_polled_tid) # We have to invalidate the whole cPickleCache, otherwise # the cache would be inconsistent with the reverted state. return None, new_polled_tid else: # This client never wants to revert to stale data, so # raise ReadConflictError to trigger a retry. # We're probably just waiting for async replication # to catch up, so retrying could do the trick. raise ReadConflictError( "The database connection is stale: new_polled_tid=%d, " "prev_polled_tid=%d." % (new_polled_tid, prev_polled_tid))
def finalize(self): return intern(self.buf.getvalue().strip()), {v: k for k, v in self.placeholders.items()}
def prepare(self): # This is correct for PostgreSQL. This needs moved to a dialect specific # spot. datatypes = self._find_datatypes_for_prepared_query() query = self.buf.getvalue() name = self._next_prepared_stmt_name(query) if datatypes: assert isinstance(datatypes, (list, tuple)) datatypes = ', '.join(datatypes) datatypes = ' (%s)' % (datatypes,) else: datatypes = '' q = query.strip() # PREPARE needs the query string to use $1, $2, $3, etc, # as placeholders. # In MySQL, it's a plain question mark. placeholder_to_number = {} counter = 0 for placeholder_name in self.placeholders.values(): counter += 1 placeholder = self._placeholder(placeholder_name) placeholder_to_number[placeholder_name] = counter param = self._prepared_param(counter) q = q.replace(placeholder, param, 1) q = self._quote_query_for_prepare(q) stmt = 'PREPARE {name}{datatypes} {conjunction} {query}'.format( name=name, datatypes=datatypes, query=q, conjunction=self._PREPARED_CONJUNCTION, ) if placeholder_to_number: execute = 'EXECUTE {name}({params})'.format( name=name, params=','.join(['%s'] * len(self.placeholders)), ) else: # Neither MySQL nor PostgreSQL like a set of empty parens: () execute = 'EXECUTE {name}'.format(name=name) if '%s' in placeholder_to_number: # There was an ordered param. If there was one, # they must all be ordered, so there's no need to convert anything. assert len(placeholder_to_number) == 1 def convert(p): return p else: def convert(d): # TODO: This may not actually be needed, since we issue a regular # cursor.execute(), it may be able to handle named? params = [None] * len(placeholder_to_number) for placeholder_name, ix in placeholder_to_number.items(): params[ix - 1] = d[placeholder_name] return params return intern(stmt), intern(execute), convert