def set_up(self, u, engine): """Special setup for mysql engines""" # add the reconnecting PoolListener that will detect a # disconnected connection and automatically start a new # one. This provides a measure of additional safety over # the pool_recycle parameter, and is useful when e.g., the # mysql server goes away def checkout_listener(dbapi_con, con_record, con_proxy): try: cursor = dbapi_con.cursor() cursor.execute("SELECT 1") except dbapi_con.OperationalError as ex: if self.is_disconnect(ex.args): # sqlalchemy will re-create the connection log.msg('connection will be removed') raise sa.exc.DisconnectionError() log.msg('exception happened {}'.format(ex)) raise # older versions of sqlalchemy require the listener to be specified # in the kwargs, in a class instance if sautils.sa_version() < (0, 7, 0): class ReconnectingListener: pass rcl = ReconnectingListener() rcl.checkout = checkout_listener engine.pool.add_listener(rcl) else: sa.event.listen(engine.pool, 'checkout', checkout_listener)
def patch(): # fix for http://www.sqlalchemy.org/trac/ticket/2189, backported to 0.6.0 if sautils.sa_version()[:2] == (0, 6): get_columns_fixed = get_columns_06x_fixed else: get_columns_fixed = get_columns_07x_fixed SQLiteDialect.get_columns = get_columns_fixed
def set_up_sqlite_engine(self, u, engine): """Special setup for sqlite engines""" # try to enable WAL logging if u.database: def connect_listener(connection, record): connection.execute("pragma checkpoint_fullfsync = off") if sautils.sa_version() < (0, 7, 0): class CheckpointFullfsyncDisabler(object): pass disabler = CheckpointFullfsyncDisabler() disabler.connect = connect_listener engine.pool.add_listener(disabler) else: sa.event.listen(engine.pool, 'connect', connect_listener) log.msg("setting database journal mode to 'wal'") try: engine.execute("pragma journal_mode = wal") except: log.msg("failed to set journal mode - database may fail")
def __thd_clean_database(self, conn): # In general it's nearly impossible to do "bullet proof" database # cleanup with SQLAlchemy that will work on a range of databases # and they configurations. # # Following approaches were considered. # # 1. Drop Buildbot Model schema: # # model.Model.metadata.drop_all(bind=conn, checkfirst=True) # # Dropping schema from model is correct and working operation only # if database schema is exactly corresponds to the model schema. # # If it is not (e.g. migration script failed or migration results in # old version of model), then some tables outside model schema may be # present, which may reference tables in the model schema. # In this case either dropping model schema will fail (if database # enforces referential integrity, e.g. PostgreSQL), or # dropping left tables in the code below will fail (if database allows # removing of tables on which other tables have references, # e.g. SQLite). # # 2. Introspect database contents and drop found tables. # # meta = MetaData(bind=conn) # meta.reflect() # meta.drop_all() # # May fail if schema contains reference cycles (and Buildbot schema # has them). Reflection looses metadata about how reference cycles # can be teared up (e.g. use_alter=True). # Introspection may fail if schema has invalid references # (e.g. possible in SQLite). # # 3. What is actually needed here is accurate code for each engine # and each engine configuration that will drop all tables, # indexes, constraints, etc in proper order or in a proper way # (using tables alternation, or DROP TABLE ... CASCADE, etc). # # Conclusion: use approach 2 with manually teared apart known # reference cycles. try: meta = MetaData(bind=conn) # Reflect database contents. May fail, e.g. if table references # non-existent table in SQLite. meta.reflect() # Table.foreign_key_constraints introduced in SQLAlchemy 1.0. if sa_version()[:2] >= (1, 0): # Restore `use_alter` settings to break known reference cycles. # Main goal of this part is to remove SQLAlchemy warning # about reference cycle. # Looks like it's OK to do it only with SQLAlchemy >= 1.0.0, # since it's not issued in SQLAlchemy == 0.8.0 # List of reference links (table_name, ref_table_name) that # should be broken by adding use_alter=True. table_referenced_table_links = [ ('buildsets', 'builds'), ('builds', 'buildrequests')] for table_name, ref_table_name in table_referenced_table_links: if table_name in meta.tables: table = meta.tables[table_name] for fkc in table.foreign_key_constraints: if fkc.referred_table.name == ref_table_name: fkc.use_alter = True # Drop all reflected tables and indices. May fail, e.g. if # SQLAlchemy wouldn't be able to break circular references. # Sqlalchemy fk support with sqlite is not yet perfect, so we must deactivate fk during that # operation, even though we made our possible to use use_alter with withoutSqliteForeignKeys(conn.engine, conn): meta.drop_all() except Exception: # sometimes this goes badly wrong; being able to see the schema # can be a big help if conn.engine.dialect.name == 'sqlite': r = conn.execute("select sql from sqlite_master " "where type='table'") log.msg("Current schema:") for row in r.fetchall(): log.msg(row.sql) raise
def patch_sqlalchemy2189(): # fix for SQLAlchemy bug 2189 if sautils.sa_version() <= (0, 7, 1): from buildbot.monkeypatches import sqlalchemy2189 sqlalchemy2189.patch()
def patch_sqlalchemy2364(): # fix for SQLAlchemy bug 2364 if sautils.sa_version() < (0, 7, 5): from buildbot.monkeypatches import sqlalchemy2364 sqlalchemy2364.patch()
def __thd_clean_database(self, conn): # In general it's nearly impossible to do "bullet proof" database # cleanup with SQLAlchemy that will work on a range of databases # and they configurations. # # Following approaches were considered. # # 1. Drop Buildbot Model schema: # # model.Model.metadata.drop_all(bind=conn, checkfirst=True) # # Dropping schema from model is correct and working operation only # if database schema is exactly corresponds to the model schema. # # If it is not (e.g. migration script failed or migration results in # old version of model), then some tables outside model schema may be # present, which may reference tables in the model schema. # In this case either dropping model schema will fail (if database # enforces referential integrity, e.g. PostgreSQL), or # dropping left tables in the code below will fail (if database allows # removing of tables on which other tables have references, # e.g. SQLite). # # 2. Introspect database contents and drop found tables. # # meta = MetaData(bind=conn) # meta.reflect() # meta.drop_all() # # May fail if schema contains reference cycles (and Buildbot schema # has them). Reflection looses metadata about how reference cycles # can be teared up (e.g. use_alter=True). # Introspection may fail if schema has invalid references # (e.g. possible in SQLite). # # 3. What is actually needed here is accurate code for each engine # and each engine configuration that will drop all tables, # indexes, constraints, etc in proper order or in a proper way # (using tables alternation, or DROP TABLE ... CASCADE, etc). # # Conclusion: use approach 2 with manually teared apart known # reference cycles. try: meta = MetaData(bind=conn) # Reflect database contents. May fail, e.g. if table references # non-existent table in SQLite. meta.reflect() # Table.foreign_key_constraints introduced in SQLAlchemy 1.0. if sa_version()[:2] >= (1, 0): # Restore `use_alter` settings to break known reference cycles. # Main goal of this part is to remove SQLAlchemy warning # about reference cycle. # Looks like it's OK to do it only with SQLAlchemy >= 1.0.0, # since it's not issued in SQLAlchemy == 0.8.0 # List of reference links (table_name, ref_table_name) that # should be broken by adding use_alter=True. table_referenced_table_links = [ ('buildsets', 'builds'), ('builds', 'buildrequests')] for table_name, ref_table_name in table_referenced_table_links: if table_name in meta.tables: table = meta.tables[table_name] for fkc in table.foreign_key_constraints: if fkc.referred_table.name == ref_table_name: fkc.use_alter = True # Drop all reflected tables and indices. May fail, e.g. if # SQLAlchemy wouldn't be able to break circular references. # Sqlalchemy fk support with sqlite is not yet perfect, so we must deactivate fk during that # operation, even though we made our possible to use use_alter with withoutSqliteForeignKeys(conn.engine, conn): meta.drop_all() except Exception: # sometimes this goes badly wrong; being able to see the schema # can be a big help if conn.engine.dialect.name == 'sqlite': r = conn.execute("select sql from sqlite_master " "where type='table'") log.msg("Current schema:") for row in r.fetchall(): log.msg(row.sql) raise
def patch_sqlalchemy2189(): # fix for SQLAlchemy bug 2189 if sautils.sa_version() <= (0,7,1): from buildbot.monkeypatches import sqlalchemy2189 sqlalchemy2189.patch()
def test_sa_version(self): self.assertTrue(sautils.sa_version() > (0, 5, 0))
def patch_sqlalchemy2364(): # fix for SQLAlchemy bug 2364 if sautils.sa_version() < (0,7,5): from buildbot.monkeypatches import sqlalchemy2364 sqlalchemy2364.patch()
class BuildbotEngineStrategy(strategies.ThreadLocalEngineStrategy): # A subclass of the ThreadLocalEngineStrategy that can effectively interact # with Buildbot. # # This adjusts the passed-in parameters to ensure that we get the behaviors # Buildbot wants from particular drivers, and wraps the outgoing Engine # object so that its methods run in threads and return deferreds. name = 'buildbot' def special_case_sqlite(self, u, kwargs): """For sqlite, percent-substitute %(basedir)s and use a full path to the basedir. If using a memory database, force the pool size to be 1.""" max_conns = None # when given a database path, stick the basedir in there if u.database: # Use NullPool instead of the sqlalchemy-0.6.8-default # SingletonThreadpool for sqlite to suppress the error in # http://groups.google.com/group/sqlalchemy/msg/f8482e4721a89589, # which also explains that NullPool is the new default in # sqlalchemy 0.7 for non-memory SQLite databases. kwargs.setdefault('poolclass', NullPool) u.database = u.database % dict(basedir=kwargs['basedir']) if not os.path.isabs(u.database[0]): u.database = os.path.join(kwargs['basedir'], u.database) # in-memory databases need exactly one connection if not u.database: kwargs['pool_size'] = 1 max_conns = 1 # allow serializing access to the db if 'serialize_access' in u.query: u.query.pop('serialize_access') max_conns = 1 return u, kwargs, max_conns def set_up_sqlite_engine(self, u, engine): """Special setup for sqlite engines""" # try to enable WAL logging if u.database: def connect_listener(connection, record): connection.execute("pragma checkpoint_fullfsync = off") if sautils.sa_version() < (0, 7, 0): class CheckpointFullfsyncDisabler(object): pass disabler = CheckpointFullfsyncDisabler() disabler.connect = connect_listener engine.pool.add_listener(disabler) else: sa.event.listen(engine.pool, 'connect', connect_listener) log.msg("setting database journal mode to 'wal'") try: engine.execute("pragma journal_mode = wal") except: log.msg("failed to set journal mode - database may fail") def special_case_mysql(self, u, kwargs): """For mysql, take max_idle out of the query arguments, and use its value for pool_recycle. Also, force use_unicode and charset to be True and 'utf8', failing if they were set to anything else.""" kwargs['pool_recycle'] = int(u.query.pop('max_idle', 3600)) # default to the InnoDB storage engine storage_engine = u.query.pop('storage_engine', 'MyISAM') kwargs['connect_args'] = { 'init_command': 'SET storage_engine=%s' % storage_engine, } if 'use_unicode' in u.query: if u.query['use_unicode'] != "True": raise TypeError("Buildbot requires use_unicode=True " + "(and adds it automatically)") else: u.query['use_unicode'] = True if 'charset' in u.query: if u.query['charset'] != "utf8": raise TypeError("Buildbot requires charset=utf8 " + "(and adds it automatically)") else: u.query['charset'] = 'utf8' return u, kwargs, None def set_up_mysql_engine(self, u, engine): """Special setup for mysql engines""" # add the reconnecting PoolListener that will detect a # disconnected connection and automatically start a new # one. This provides a measure of additional safety over # the pool_recycle parameter, and is useful when e.g., the # mysql server goes away def checkout_listener(dbapi_con, con_record, con_proxy): try: cursor = dbapi_con.cursor() cursor.execute("SELECT 1") except dbapi_con.OperationalError, ex: if ex.args[0] in (2006, 2013, 2014, 2045, 2055): # sqlalchemy will re-create the connection raise sa.exc.DisconnectionError() raise # older versions of sqlalchemy require the listener to be specified # in the kwargs, in a class instance if sautils.sa_version() < (0, 7, 0): class ReconnectingListener(object): pass rcl = ReconnectingListener() rcl.checkout = checkout_listener engine.pool.add_listener(rcl) else: sa.event.listen(engine.pool, 'checkout', checkout_listener)
def test_sa_version(self): self.failUnless(sautils.sa_version() > (0, 5, 0))
def test_sa_version(self): self.failUnless(sautils.sa_version() > (0,5,0))
def test_sa_version(self): self.assertTrue(sautils.sa_version() > (0, 5, 0))