def upgrade(engine):
schema = ControlledSchema(engine, self.repo_path)
changeset = schema.changeset(None)
with sautils.withoutSqliteForeignKeys(engine):
for version, change in changeset:
log.msg('migrating schema version %s -> %d' %
(version, version + 1))
schema.runchange(version, change, 1)
def upgrade(engine):
schema = ControlledSchema(engine, self.repo_path)
changeset = schema.changeset(None)
with sautils.withoutSqliteForeignKeys(engine):
for version, change in changeset:
log.msg('migrating schema version %s -> %d'
% (version, version + 1))
schema.runchange(version, change, 1)
def upgrade_thd(engine):
with querylog.log_queries():
schema = migrate.versioning.schema.ControlledSchema(
engine, self.db.model.repo_path)
changeset = schema.changeset(target_version)
with sautils.withoutSqliteForeignKeys(engine):
for version, change in changeset:
log.msg('upgrading to schema version %d' % (version + 1))
schema.runchange(version, change, 1)
def upgrade_thd(engine):
with querylog.log_queries():
schema = migrate.versioning.schema.ControlledSchema(
engine, self.db.model.repo_path)
changeset = schema.changeset(target_version)
with sautils.withoutSqliteForeignKeys(engine):
for version, change in changeset:
log.msg('upgrading to schema version %d' %
(version + 1))
schema.runchange(version, change, 1)
def thd(conn):
alembic_scripts = self.alembic_get_scripts()
current_script_rev_head = alembic_scripts.get_current_head()
if self.table_exists(conn, 'version'):
raise UpgradeFromBefore0p9Error()
if self.table_exists(conn, 'migrate_version'):
version = self.migrate_get_version(conn)
if version < 40:
raise UpgradeFromBefore0p9Error()
last_sqlalchemy_migrate_version = 58
if version != last_sqlalchemy_migrate_version:
raise UpgradeFromBefore3p0Error()
self.alembic_stamp(conn, alembic_scripts,
alembic_scripts.get_base())
conn.execute('drop table migrate_version')
if not self.table_exists(conn, 'alembic_version'):
log.msg("Initializing empty database")
# Do some tests first
test_unicode(conn)
Model.metadata.create_all(conn)
self.alembic_stamp(conn, alembic_scripts,
current_script_rev_head)
return
context = alembic.runtime.migration.MigrationContext.configure(
conn)
current_rev = context.get_current_revision()
if current_rev == current_script_rev_head:
log.msg(
'Upgrading database: the current database schema is already the newest'
)
return
log.msg('Upgrading database')
with sautils.withoutSqliteForeignKeys(conn):
with context.begin_transaction():
context.run_migrations()
log.msg('Upgrading database: done')
def upgrade_thd(engine):
with querylog.log_queries():
with sautils.withoutSqliteForeignKeys(engine):
with engine.connect() as conn:
def upgrade(rev, context):
log.msg(
f'Upgrading from {rev} to {target_revision}')
return alembic_scripts._upgrade_revs(
target_revision, rev)
context = MigrationContext.configure(
conn, opts={'fn': upgrade})
with context.begin_transaction():
context.run_migrations()
def __thd_clean_database(self, conn):
# In general it's nearly impossible to do "bullet proof" database
# cleanup with SQLAlchemy that will work on a range of databases
# and they configurations.
#
# Following approaches were considered.
#
# 1. Drop Buildbot Model schema:
#
# model.Model.metadata.drop_all(bind=conn, checkfirst=True)
#
# Dropping schema from model is correct and working operation only
# if database schema is exactly corresponds to the model schema.
#
# If it is not (e.g. migration script failed or migration results in
# old version of model), then some tables outside model schema may be
# present, which may reference tables in the model schema.
# In this case either dropping model schema will fail (if database
# enforces referential integrity, e.g. PostgreSQL), or
# dropping left tables in the code below will fail (if database allows
# removing of tables on which other tables have references,
# e.g. SQLite).
#
# 2. Introspect database contents and drop found tables.
#
# meta = MetaData(bind=conn)
# meta.reflect()
# meta.drop_all()
#
# May fail if schema contains reference cycles (and Buildbot schema
# has them). Reflection looses metadata about how reference cycles
# can be teared up (e.g. use_alter=True).
# Introspection may fail if schema has invalid references
# (e.g. possible in SQLite).
#
# 3. What is actually needed here is accurate code for each engine
# and each engine configuration that will drop all tables,
# indexes, constraints, etc in proper order or in a proper way
# (using tables alternation, or DROP TABLE ... CASCADE, etc).
#
# Conclusion: use approach 2 with manually teared apart known
# reference cycles.
# pylint: disable=too-many-nested-blocks
try:
meta = MetaData(bind=conn)
# Reflect database contents. May fail, e.g. if table references
# non-existent table in SQLite.
meta.reflect()
# Table.foreign_key_constraints introduced in SQLAlchemy 1.0.
if sa_version()[:2] >= (1, 0):
# Restore `use_alter` settings to break known reference cycles.
# Main goal of this part is to remove SQLAlchemy warning
# about reference cycle.
# Looks like it's OK to do it only with SQLAlchemy >= 1.0.0,
# since it's not issued in SQLAlchemy == 0.8.0
# List of reference links (table_name, ref_table_name) that
# should be broken by adding use_alter=True.
table_referenced_table_links = [('buildsets', 'builds'),
('builds', 'buildrequests')]
for table_name, ref_table_name in table_referenced_table_links:
if table_name in meta.tables:
table = meta.tables[table_name]
for fkc in table.foreign_key_constraints:
if fkc.referred_table.name == ref_table_name:
fkc.use_alter = True
# Drop all reflected tables and indices. May fail, e.g. if
# SQLAlchemy wouldn't be able to break circular references.
# Sqlalchemy fk support with sqlite is not yet perfect, so we must deactivate fk during that
# operation, even though we made our possible to use use_alter
with withoutSqliteForeignKeys(conn.engine, conn):
meta.drop_all()
except Exception:
# sometimes this goes badly wrong; being able to see the schema
# can be a big help
if conn.engine.dialect.name == 'sqlite':
r = conn.execute("select sql from sqlite_master "
"where type='table'")
log.msg("Current schema:")
for row in r.fetchall():
log.msg(row.sql)
raise
def verify_thd(engine):
with sautils.withoutSqliteForeignKeys(engine):
verify_thd_cb(engine)
def __thd_clean_database(self, conn):
# In general it's nearly impossible to do "bullet proof" database
# cleanup with SQLAlchemy that will work on a range of databases
# and they configurations.
#
# Following approaches were considered.
#
# 1. Drop Buildbot Model schema:
#
# model.Model.metadata.drop_all(bind=conn, checkfirst=True)
#
# Dropping schema from model is correct and working operation only
# if database schema is exactly corresponds to the model schema.
#
# If it is not (e.g. migration script failed or migration results in
# old version of model), then some tables outside model schema may be
# present, which may reference tables in the model schema.
# In this case either dropping model schema will fail (if database
# enforces referential integrity, e.g. PostgreSQL), or
# dropping left tables in the code below will fail (if database allows
# removing of tables on which other tables have references,
# e.g. SQLite).
#
# 2. Introspect database contents and drop found tables.
#
# meta = MetaData(bind=conn)
# meta.reflect()
# meta.drop_all()
#
# May fail if schema contains reference cycles (and Buildbot schema
# has them). Reflection looses metadata about how reference cycles
# can be teared up (e.g. use_alter=True).
# Introspection may fail if schema has invalid references
# (e.g. possible in SQLite).
#
# 3. What is actually needed here is accurate code for each engine
# and each engine configuration that will drop all tables,
# indexes, constraints, etc in proper order or in a proper way
# (using tables alternation, or DROP TABLE ... CASCADE, etc).
#
# Conclusion: use approach 2 with manually teared apart known
# reference cycles.
try:
meta = MetaData(bind=conn)
# Reflect database contents. May fail, e.g. if table references
# non-existent table in SQLite.
meta.reflect()
# Table.foreign_key_constraints introduced in SQLAlchemy 1.0.
if sa_version()[:2] >= (1, 0):
# Restore `use_alter` settings to break known reference cycles.
# Main goal of this part is to remove SQLAlchemy warning
# about reference cycle.
# Looks like it's OK to do it only with SQLAlchemy >= 1.0.0,
# since it's not issued in SQLAlchemy == 0.8.0
# List of reference links (table_name, ref_table_name) that
# should be broken by adding use_alter=True.
table_referenced_table_links = [
('buildsets', 'builds'), ('builds', 'buildrequests')]
for table_name, ref_table_name in table_referenced_table_links:
if table_name in meta.tables:
table = meta.tables[table_name]
for fkc in table.foreign_key_constraints:
if fkc.referred_table.name == ref_table_name:
fkc.use_alter = True
# Drop all reflected tables and indices. May fail, e.g. if
# SQLAlchemy wouldn't be able to break circular references.
# Sqlalchemy fk support with sqlite is not yet perfect, so we must deactivate fk during that
# operation, even though we made our possible to use use_alter
with withoutSqliteForeignKeys(conn.engine, conn):
meta.drop_all()
except Exception:
# sometimes this goes badly wrong; being able to see the schema
# can be a big help
if conn.engine.dialect.name == 'sqlite':
r = conn.execute("select sql from sqlite_master "
"where type='table'")
log.msg("Current schema:")
for row in r.fetchall():
log.msg(row.sql)
raise
def verify_thd(engine):
with sautils.withoutSqliteForeignKeys(engine):
verify_thd_cb(engine)
