def set_up(self, u, engine):
"""Special setup for mysql engines"""
# add the reconnecting PoolListener that will detect a
# disconnected connection and automatically start a new
# one. This provides a measure of additional safety over
# the pool_recycle parameter, and is useful when e.g., the
# mysql server goes away
def checkout_listener(dbapi_con, con_record, con_proxy):
try:
cursor = dbapi_con.cursor()
cursor.execute("SELECT 1")
except dbapi_con.OperationalError as ex:
if self.is_disconnect(ex.args):
# sqlalchemy will re-create the connection
log.msg('connection will be removed')
raise sa.exc.DisconnectionError()
log.msg('exception happened {}'.format(ex))
raise
# older versions of sqlalchemy require the listener to be specified
# in the kwargs, in a class instance
if sautils.sa_version() < (0, 7, 0):
class ReconnectingListener:
pass
rcl = ReconnectingListener()
rcl.checkout = checkout_listener
engine.pool.add_listener(rcl)
else:
sa.event.listen(engine.pool, 'checkout', checkout_listener)
def patch():
# fix for http://www.sqlalchemy.org/trac/ticket/2189, backported to 0.6.0
if sautils.sa_version()[:2] == (0, 6):
get_columns_fixed = get_columns_06x_fixed
else:
get_columns_fixed = get_columns_07x_fixed
SQLiteDialect.get_columns = get_columns_fixed
def set_up_sqlite_engine(self, u, engine):
"""Special setup for sqlite engines"""
# try to enable WAL logging
if u.database:
def connect_listener(connection, record):
connection.execute("pragma checkpoint_fullfsync = off")
if sautils.sa_version() < (0, 7, 0):
class CheckpointFullfsyncDisabler(object):
pass
disabler = CheckpointFullfsyncDisabler()
disabler.connect = connect_listener
engine.pool.add_listener(disabler)
else:
sa.event.listen(engine.pool, 'connect', connect_listener)
log.msg("setting database journal mode to 'wal'")
try:
engine.execute("pragma journal_mode = wal")
except:
log.msg("failed to set journal mode - database may fail")
def __thd_clean_database(self, conn):
# In general it's nearly impossible to do "bullet proof" database
# cleanup with SQLAlchemy that will work on a range of databases
# and they configurations.
#
# Following approaches were considered.
#
# 1. Drop Buildbot Model schema:
#
# model.Model.metadata.drop_all(bind=conn, checkfirst=True)
#
# Dropping schema from model is correct and working operation only
# if database schema is exactly corresponds to the model schema.
#
# If it is not (e.g. migration script failed or migration results in
# old version of model), then some tables outside model schema may be
# present, which may reference tables in the model schema.
# In this case either dropping model schema will fail (if database
# enforces referential integrity, e.g. PostgreSQL), or
# dropping left tables in the code below will fail (if database allows
# removing of tables on which other tables have references,
# e.g. SQLite).
#
# 2. Introspect database contents and drop found tables.
#
# meta = MetaData(bind=conn)
# meta.reflect()
# meta.drop_all()
#
# May fail if schema contains reference cycles (and Buildbot schema
# has them). Reflection looses metadata about how reference cycles
# can be teared up (e.g. use_alter=True).
# Introspection may fail if schema has invalid references
# (e.g. possible in SQLite).
#
# 3. What is actually needed here is accurate code for each engine
# and each engine configuration that will drop all tables,
# indexes, constraints, etc in proper order or in a proper way
# (using tables alternation, or DROP TABLE ... CASCADE, etc).
#
# Conclusion: use approach 2 with manually teared apart known
# reference cycles.
try:
meta = MetaData(bind=conn)
# Reflect database contents. May fail, e.g. if table references
# non-existent table in SQLite.
meta.reflect()
# Table.foreign_key_constraints introduced in SQLAlchemy 1.0.
if sa_version()[:2] >= (1, 0):
# Restore `use_alter` settings to break known reference cycles.
# Main goal of this part is to remove SQLAlchemy warning
# about reference cycle.
# Looks like it's OK to do it only with SQLAlchemy >= 1.0.0,
# since it's not issued in SQLAlchemy == 0.8.0
# List of reference links (table_name, ref_table_name) that
# should be broken by adding use_alter=True.
table_referenced_table_links = [
('buildsets', 'builds'), ('builds', 'buildrequests')]
for table_name, ref_table_name in table_referenced_table_links:
if table_name in meta.tables:
table = meta.tables[table_name]
for fkc in table.foreign_key_constraints:
if fkc.referred_table.name == ref_table_name:
fkc.use_alter = True
# Drop all reflected tables and indices. May fail, e.g. if
# SQLAlchemy wouldn't be able to break circular references.
# Sqlalchemy fk support with sqlite is not yet perfect, so we must deactivate fk during that
# operation, even though we made our possible to use use_alter
with withoutSqliteForeignKeys(conn.engine, conn):
meta.drop_all()
except Exception:
# sometimes this goes badly wrong; being able to see the schema
# can be a big help
if conn.engine.dialect.name == 'sqlite':
r = conn.execute("select sql from sqlite_master "
"where type='table'")
log.msg("Current schema:")
for row in r.fetchall():
log.msg(row.sql)
raise
def patch_sqlalchemy2189():
# fix for SQLAlchemy bug 2189
if sautils.sa_version() <= (0, 7, 1):
from buildbot.monkeypatches import sqlalchemy2189
sqlalchemy2189.patch()
def patch_sqlalchemy2364():
# fix for SQLAlchemy bug 2364
if sautils.sa_version() < (0, 7, 5):
from buildbot.monkeypatches import sqlalchemy2364
sqlalchemy2364.patch()
def __thd_clean_database(self, conn):
# In general it's nearly impossible to do "bullet proof" database
# cleanup with SQLAlchemy that will work on a range of databases
# and they configurations.
#
# Following approaches were considered.
#
# 1. Drop Buildbot Model schema:
#
# model.Model.metadata.drop_all(bind=conn, checkfirst=True)
#
# Dropping schema from model is correct and working operation only
# if database schema is exactly corresponds to the model schema.
#
# If it is not (e.g. migration script failed or migration results in
# old version of model), then some tables outside model schema may be
# present, which may reference tables in the model schema.
# In this case either dropping model schema will fail (if database
# enforces referential integrity, e.g. PostgreSQL), or
# dropping left tables in the code below will fail (if database allows
# removing of tables on which other tables have references,
# e.g. SQLite).
#
# 2. Introspect database contents and drop found tables.
#
# meta = MetaData(bind=conn)
# meta.reflect()
# meta.drop_all()
#
# May fail if schema contains reference cycles (and Buildbot schema
# has them). Reflection looses metadata about how reference cycles
# can be teared up (e.g. use_alter=True).
# Introspection may fail if schema has invalid references
# (e.g. possible in SQLite).
#
# 3. What is actually needed here is accurate code for each engine
# and each engine configuration that will drop all tables,
# indexes, constraints, etc in proper order or in a proper way
# (using tables alternation, or DROP TABLE ... CASCADE, etc).
#
# Conclusion: use approach 2 with manually teared apart known
# reference cycles.
try:
meta = MetaData(bind=conn)
# Reflect database contents. May fail, e.g. if table references
# non-existent table in SQLite.
meta.reflect()
# Table.foreign_key_constraints introduced in SQLAlchemy 1.0.
if sa_version()[:2] >= (1, 0):
# Restore `use_alter` settings to break known reference cycles.
# Main goal of this part is to remove SQLAlchemy warning
# about reference cycle.
# Looks like it's OK to do it only with SQLAlchemy >= 1.0.0,
# since it's not issued in SQLAlchemy == 0.8.0
# List of reference links (table_name, ref_table_name) that
# should be broken by adding use_alter=True.
table_referenced_table_links = [
('buildsets', 'builds'), ('builds', 'buildrequests')]
for table_name, ref_table_name in table_referenced_table_links:
if table_name in meta.tables:
table = meta.tables[table_name]
for fkc in table.foreign_key_constraints:
if fkc.referred_table.name == ref_table_name:
fkc.use_alter = True
# Drop all reflected tables and indices. May fail, e.g. if
# SQLAlchemy wouldn't be able to break circular references.
# Sqlalchemy fk support with sqlite is not yet perfect, so we must deactivate fk during that
# operation, even though we made our possible to use use_alter
with withoutSqliteForeignKeys(conn.engine, conn):
meta.drop_all()
except Exception:
# sometimes this goes badly wrong; being able to see the schema
# can be a big help
if conn.engine.dialect.name == 'sqlite':
r = conn.execute("select sql from sqlite_master "
"where type='table'")
log.msg("Current schema:")
for row in r.fetchall():
log.msg(row.sql)
raise
def patch_sqlalchemy2189():
# fix for SQLAlchemy bug 2189
if sautils.sa_version() <= (0,7,1):
from buildbot.monkeypatches import sqlalchemy2189
sqlalchemy2189.patch()
def test_sa_version(self):
self.assertTrue(sautils.sa_version() > (0, 5, 0))
def patch_sqlalchemy2364():
# fix for SQLAlchemy bug 2364
if sautils.sa_version() < (0,7,5):
from buildbot.monkeypatches import sqlalchemy2364
sqlalchemy2364.patch()
class BuildbotEngineStrategy(strategies.ThreadLocalEngineStrategy):
# A subclass of the ThreadLocalEngineStrategy that can effectively interact
# with Buildbot.
#
# This adjusts the passed-in parameters to ensure that we get the behaviors
# Buildbot wants from particular drivers, and wraps the outgoing Engine
# object so that its methods run in threads and return deferreds.
name = 'buildbot'
def special_case_sqlite(self, u, kwargs):
"""For sqlite, percent-substitute %(basedir)s and use a full
path to the basedir. If using a memory database, force the
pool size to be 1."""
max_conns = None
# when given a database path, stick the basedir in there
if u.database:
# Use NullPool instead of the sqlalchemy-0.6.8-default
# SingletonThreadpool for sqlite to suppress the error in
# http://groups.google.com/group/sqlalchemy/msg/f8482e4721a89589,
# which also explains that NullPool is the new default in
# sqlalchemy 0.7 for non-memory SQLite databases.
kwargs.setdefault('poolclass', NullPool)
u.database = u.database % dict(basedir=kwargs['basedir'])
if not os.path.isabs(u.database[0]):
u.database = os.path.join(kwargs['basedir'], u.database)
# in-memory databases need exactly one connection
if not u.database:
kwargs['pool_size'] = 1
max_conns = 1
# allow serializing access to the db
if 'serialize_access' in u.query:
u.query.pop('serialize_access')
max_conns = 1
return u, kwargs, max_conns
def set_up_sqlite_engine(self, u, engine):
"""Special setup for sqlite engines"""
# try to enable WAL logging
if u.database:
def connect_listener(connection, record):
connection.execute("pragma checkpoint_fullfsync = off")
if sautils.sa_version() < (0, 7, 0):
class CheckpointFullfsyncDisabler(object):
pass
disabler = CheckpointFullfsyncDisabler()
disabler.connect = connect_listener
engine.pool.add_listener(disabler)
else:
sa.event.listen(engine.pool, 'connect', connect_listener)
log.msg("setting database journal mode to 'wal'")
try:
engine.execute("pragma journal_mode = wal")
except:
log.msg("failed to set journal mode - database may fail")
def special_case_mysql(self, u, kwargs):
"""For mysql, take max_idle out of the query arguments, and
use its value for pool_recycle. Also, force use_unicode and
charset to be True and 'utf8', failing if they were set to
anything else."""
kwargs['pool_recycle'] = int(u.query.pop('max_idle', 3600))
# default to the InnoDB storage engine
storage_engine = u.query.pop('storage_engine', 'MyISAM')
kwargs['connect_args'] = {
'init_command': 'SET storage_engine=%s' % storage_engine,
}
if 'use_unicode' in u.query:
if u.query['use_unicode'] != "True":
raise TypeError("Buildbot requires use_unicode=True " +
"(and adds it automatically)")
else:
u.query['use_unicode'] = True
if 'charset' in u.query:
if u.query['charset'] != "utf8":
raise TypeError("Buildbot requires charset=utf8 " +
"(and adds it automatically)")
else:
u.query['charset'] = 'utf8'
return u, kwargs, None
def set_up_mysql_engine(self, u, engine):
"""Special setup for mysql engines"""
# add the reconnecting PoolListener that will detect a
# disconnected connection and automatically start a new
# one. This provides a measure of additional safety over
# the pool_recycle parameter, and is useful when e.g., the
# mysql server goes away
def checkout_listener(dbapi_con, con_record, con_proxy):
try:
cursor = dbapi_con.cursor()
cursor.execute("SELECT 1")
except dbapi_con.OperationalError, ex:
if ex.args[0] in (2006, 2013, 2014, 2045, 2055):
# sqlalchemy will re-create the connection
raise sa.exc.DisconnectionError()
raise
# older versions of sqlalchemy require the listener to be specified
# in the kwargs, in a class instance
if sautils.sa_version() < (0, 7, 0):
class ReconnectingListener(object):
pass
rcl = ReconnectingListener()
rcl.checkout = checkout_listener
engine.pool.add_listener(rcl)
else:
sa.event.listen(engine.pool, 'checkout', checkout_listener)
def test_sa_version(self):
self.failUnless(sautils.sa_version() > (0, 5, 0))
def test_sa_version(self):
self.failUnless(sautils.sa_version() > (0,5,0))
def test_sa_version(self):
self.assertTrue(sautils.sa_version() > (0, 5, 0))
