def _transaction_iterator(self, cursor):
"""Iterate over a list of transactions returned from the database.
Each row begins with (tid, username, description, extension)
and may have other columns.
"""
# Iterating the cursor itself in a generator is not safe if
# the cursor doesn't actually buffer all the rows *anyway*. If
# we break from the iterating loop before exhausting all the
# rows, a subsequent query or close operation can lead to
# things like MySQL Connector/Python raising
# InternalError(unread results)
rows = cursor.fetchall()
for row in rows:
tid, username, description, ext = row[:4]
# Although the transaction interface for username and description are
# defined as strings, this layer works with bytes. PY3.
if username is None:
username = b''
else:
username = db_binary_to_bytes(username)
if description is None:
description = b''
else:
description = db_binary_to_bytes(description)
if ext is None:
ext = b''
else:
ext = db_binary_to_bytes(ext)
yield (tid, username, description, ext) + tuple(row[4:])
def _transaction_iterator(self, cursor):
"""Iterate over a list of transactions returned from the database.
Each row begins with (tid, username, description, extension)
and may have other columns.
"""
for row in cursor:
tid, username, description, ext = row[:4]
# Although the transaction interface for username and description are
# defined as strings, this layer works with bytes. PY3.
if username is None:
username = b''
else:
username = db_binary_to_bytes(username)
if description is None:
description = b''
else:
description = db_binary_to_bytes(description)
if ext is None:
ext = b''
else:
ext = db_binary_to_bytes(ext)
yield (tid, username, description, ext) + tuple(row[4:])
def _transaction_iterator(self, cursor):
"""Iterate over a list of transactions returned from the database.
Each row begins with (tid, username, description, extension)
and may have other columns.
"""
for row in cursor:
tid, username, description, ext = row[:4]
# Although the transaction interface for username and description are
# defined as strings, this layer works with bytes. PY3.
if username is None:
username = b''
else:
username = db_binary_to_bytes(username)
if description is None:
description = b''
else:
description = db_binary_to_bytes(description)
if ext is None:
ext = b''
else:
ext = db_binary_to_bytes(ext)
yield (tid, username, description, ext) + tuple(row[4:])
def postgresql_load_current(self, cursor, oid):
"""Returns the current pickle and integer tid for an object.
oid is an integer. Returns (None, None) if object does not exist.
"""
if self.keep_history:
stmt = """
SELECT state, tid
FROM current_object
JOIN object_state USING(zoid, tid)
WHERE zoid = %s
"""
else:
stmt = """
SELECT state, tid
FROM object_state
WHERE zoid = %s
"""
cursor.execute(stmt, (oid,))
if cursor.rowcount:
assert cursor.rowcount == 1
state, tid = cursor.fetchone()
state = db_binary_to_bytes(state)
# If it's None, the object's creation has been
# undone.
return state, tid
else:
return None, None
def _add_refs_for_tid(self, cursor, tid, get_references):
"""Fill object_refs with all states for a transaction.
Returns the number of references added.
"""
log.debug("pre_pack: transaction %d: computing references ", tid)
from_count = 0
stmt = """
SELECT zoid, state
FROM object_state
WHERE tid = %(tid)s
"""
self.runner.run_script_stmt(cursor, stmt, {'tid': tid})
add_rows = [] # [(from_oid, tid, to_oid)]
for from_oid, state in fetchmany(cursor):
state = db_binary_to_bytes(state)
if hasattr(state, 'read'):
# Oracle
state = state.read()
if state:
assert isinstance(state, bytes), type(
state) # PY3: used to be str(state)
from_count += 1
try:
to_oids = get_references(state)
except:
log.error(
"pre_pack: can't unpickle "
"object %d in transaction %d; state length = %d" %
(from_oid, tid, len(state)))
raise
for to_oid in to_oids:
add_rows.append((from_oid, tid, to_oid))
# A previous pre-pack may have been interrupted. Delete rows
# from the interrupted attempt.
stmt = "DELETE FROM object_ref WHERE tid = %(tid)s"
self.runner.run_script_stmt(cursor, stmt, {'tid': tid})
# Add the new references.
stmt = """
INSERT INTO object_ref (zoid, tid, to_zoid)
VALUES (%s, %s, %s)
"""
self.runner.run_many(cursor, stmt, add_rows)
# The references have been computed for this transaction.
stmt = """
INSERT INTO object_refs_added (tid)
VALUES (%(tid)s)
"""
self.runner.run_script_stmt(cursor, stmt, {'tid': tid})
to_count = len(add_rows)
log.debug(
"pre_pack: transaction %d: has %d reference(s) "
"from %d object(s)", tid, to_count, from_count)
return to_count
def _add_refs_for_tid(self, cursor, tid, get_references):
"""Fill object_refs with all states for a transaction.
Returns the number of references added.
"""
log.debug("pre_pack: transaction %d: computing references ", tid)
from_count = 0
stmt = """
SELECT zoid, state
FROM object_state
WHERE tid = %(tid)s
"""
self.runner.run_script_stmt(cursor, stmt, {'tid': tid})
add_rows = [] # [(from_oid, tid, to_oid)]
for from_oid, state in fetchmany(cursor):
state = db_binary_to_bytes(state)
if hasattr(state, 'read'):
# Oracle
state = state.read()
if state:
assert isinstance(state, bytes), type(state) # PY3: used to be str(state)
from_count += 1
try:
to_oids = get_references(state)
except:
log.error(
"pre_pack: can't unpickle "
"object %d in transaction %d; state length = %d" % (
from_oid, tid, len(state)))
raise
for to_oid in to_oids:
add_rows.append((from_oid, tid, to_oid))
# A previous pre-pack may have been interrupted. Delete rows
# from the interrupted attempt.
stmt = "DELETE FROM object_ref WHERE tid = %(tid)s"
self.runner.run_script_stmt(cursor, stmt, {'tid': tid})
# Add the new references.
stmt = """
INSERT INTO object_ref (zoid, tid, to_zoid)
VALUES (%s, %s, %s)
"""
self.runner.run_many(cursor, stmt, add_rows)
# The references have been computed for this transaction.
stmt = """
INSERT INTO object_refs_added (tid)
VALUES (%(tid)s)
"""
self.runner.run_script_stmt(cursor, stmt, {'tid': tid})
to_count = len(add_rows)
log.debug("pre_pack: transaction %d: has %d reference(s) "
"from %d object(s)", tid, to_count, from_count)
return to_count
def _add_refs_for_oids(self, cursor, oids, get_references):
"""Fill object_refs with the states for some objects.
Returns the number of references added.
"""
# XXX PY3: This could be tricky
oid_list = ','.join(str(oid) for oid in oids)
stmt = """
SELECT zoid, tid, state
FROM object_state
WHERE zoid IN (%s)
""" % oid_list
self.runner.run_script_stmt(cursor, stmt)
add_objects = []
add_refs = []
for from_oid, tid, state in self._fetchmany(cursor):
state = db_binary_to_bytes(state)
if hasattr(state, 'read'):
# Oracle
state = state.read()
add_objects.append((from_oid, tid))
if state:
assert isinstance(state, bytes), type(state)
# XXX PY3 state = str(state)
try:
to_oids = get_references(state)
except:
log.error(
"pre_pack: can't unpickle "
"object %d in transaction %d; state length = %d",
from_oid, tid, len(state))
raise
for to_oid in to_oids:
add_refs.append((from_oid, tid, to_oid))
if not add_objects:
return 0
stmt = "DELETE FROM object_refs_added WHERE zoid IN (%s)" % oid_list
self.runner.run_script_stmt(cursor, stmt)
stmt = "DELETE FROM object_ref WHERE zoid IN (%s)" % oid_list
self.runner.run_script_stmt(cursor, stmt)
stmt = """
INSERT INTO object_ref (zoid, tid, to_zoid) VALUES (%s, %s, %s)
"""
self.runner.run_many(cursor, stmt, add_refs)
stmt = """
INSERT INTO object_refs_added (zoid, tid) VALUES (%s, %s)
"""
self.runner.run_many(cursor, stmt, add_objects)
return len(add_refs)
def _add_refs_for_oids(self, cursor, oids, get_references):
"""Fill object_refs with the states for some objects.
Returns the number of references added.
"""
# XXX PY3: This could be tricky
oid_list = ','.join(str(oid) for oid in oids)
stmt = """
SELECT zoid, tid, state
FROM object_state
WHERE zoid IN (%s)
""" % oid_list
self.runner.run_script_stmt(cursor, stmt)
add_objects = []
add_refs = []
for from_oid, tid, state in self._fetchmany(cursor):
state = db_binary_to_bytes(state)
if hasattr(state, 'read'):
# Oracle
state = state.read()
add_objects.append((from_oid, tid))
if state:
assert isinstance(state, bytes), type(state)
# XXX PY3 state = str(state)
try:
to_oids = get_references(state)
except:
log.error("pre_pack: can't unpickle "
"object %d in transaction %d; state length = %d",
from_oid, tid, len(state))
raise
for to_oid in to_oids:
add_refs.append((from_oid, tid, to_oid))
if not add_objects:
return 0
stmt = "DELETE FROM object_refs_added WHERE zoid IN (%s)" % oid_list
self.runner.run_script_stmt(cursor, stmt)
stmt = "DELETE FROM object_ref WHERE zoid IN (%s)" % oid_list
self.runner.run_script_stmt(cursor, stmt)
stmt = """
INSERT INTO object_ref (zoid, tid, to_zoid) VALUES (%s, %s, %s)
"""
self.runner.run_many(cursor, stmt, add_refs)
stmt = """
INSERT INTO object_refs_added (zoid, tid) VALUES (%s, %s)
"""
self.runner.run_many(cursor, stmt, add_objects)
return len(add_refs)
def _to_native_str(self, value):
# Almost all drivers return CHAR/VARCHAR as
# bytes or possibly str. mysql connector/python, though,
# will return them as unicode on Py2 if not properly configured.
# If properly configured, it will return them as bytearray.
# This doesn't seem configurable.
# sigh.
value = db_binary_to_bytes(value)
if not isinstance(value, str):
return value.decode('ascii')
return value
def _to_native_str(self, value):
# Almost all drivers return CHAR/VARCHAR as
# bytes or possibly str. mysql connector/python, though,
# will return them as unicode on Py2 if not properly configured.
# If properly configured, it will return them as bytearray.
# This doesn't seem configurable.
# sigh.
value = db_binary_to_bytes(value)
if not isinstance(value, str):
return value.decode('ascii')
return value
def load_revision(self, cursor, oid, tid):
"""Returns the pickle for an object on a particular transaction.
Returns None if no such state exists.
"""
stmt = self._load_revision_query
cursor.execute(stmt, (oid, tid))
row = cursor.fetchone()
if row:
(state,) = row
return db_binary_to_bytes(state)
return None
def postgresql_load_revision(self, cursor, oid, tid):
"""Returns the pickle for an object on a particular transaction.
Returns None if no such state exists.
"""
stmt = """
SELECT state
FROM object_state
WHERE zoid = %s
AND tid = %s
"""
cursor.execute(stmt, (oid, tid))
if cursor.rowcount:
assert cursor.rowcount == 1
(state,) = cursor.fetchone()
return db_binary_to_bytes(state)
return None
def load_current(self, cursor, oid):
"""Returns the current pickle and integer tid for an object.
oid is an integer. Returns (None, None) if object does not exist.
"""
stmt = self._load_current_query
cursor.execute(stmt, (oid,))
if cursor.rowcount:
assert cursor.rowcount == 1
state, tid = cursor.fetchone()
state = db_binary_to_bytes(state)
# If it's None, the object's creation has been
# undone.
return state, tid
else:
return None, None
def iter_objects(self, cursor, tid):
"""Iterate over object states in a transaction.
Yields (oid, prev_tid, state) for each object state.
"""
stmt = """
SELECT zoid, state
FROM object_state
WHERE tid = %(tid)s
ORDER BY zoid
"""
self.runner.run_script_stmt(cursor, stmt, {'tid': tid})
for oid, state in cursor:
if hasattr(state, 'read'):
# Oracle
state = state.read()
state = db_binary_to_bytes(state)
yield oid, state
def iter_objects(self, cursor, tid):
"""Iterate over object states in a transaction.
Yields (oid, prev_tid, state) for each object state.
"""
stmt = """
SELECT zoid, state
FROM object_state
WHERE tid = %(tid)s
ORDER BY zoid
"""
self.runner.run_script_stmt(cursor, stmt, {'tid': tid})
for oid, state in cursor:
if hasattr(state, 'read'):
# Oracle
state = state.read()
state = db_binary_to_bytes(state)
yield oid, state
def load_current(self, cursor, oid):
"""Returns the current pickle and integer tid for an object.
oid is an integer. Returns (None, None) if object does not exist.
"""
stmt = self._load_current_query
cursor.execute(stmt, (oid,))
# Note that we cannot rely on cursor.rowcount being
# a valid indicator. The DB-API doesn't require it, and
# some implementations, like MySQL Connector/Python are
# unbuffered by default and can't provide it.
row = cursor.fetchone()
if row:
state, tid = row
state = db_binary_to_bytes(state)
# If it's None, the object's creation has been
# undone.
return state, tid
return None, None
def load_before(self, cursor, oid, tid):
"""Returns the pickle and tid of an object before transaction tid.
Returns (None, None) if no earlier state exists.
"""
stmt = """
SELECT state, tid
FROM object_state
WHERE zoid = %s
AND tid < %s
ORDER BY tid DESC
LIMIT 1
"""
cursor.execute(stmt, (oid, tid))
row = cursor.fetchone()
if row:
state, tid = row
state = db_binary_to_bytes(state)
# None in state means The object's creation has been undone
return state, tid
return None, None