def schema_update(self):
"""Examines the Alembic schema version and performs database
migration if needed
Returns:
result (dict): summary of changes made
status = ok if no errors
"""
results = []
errors = 0
try:
version = self.session.query(AlembicVersion).one().version_num
except (sqlalchemy.orm.exc.NoResultFound,
sqlalchemy.exc.OperationalError,
sqlalchemy.exc.ProgrammingError) as ex:
Syslog.logger.warn('DB schema does not yet exist: %s' % str(ex))
version = None
cfg = alembic.config.Config()
cfg.set_main_option(
'script_location',
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'alembic'))
cfg.set_main_option('url', str(self.engine.url))
script = alembic.script.ScriptDirectory.from_config(cfg)
env = EnvironmentContext(cfg, script)
if (version == script.get_heads()[0]):
Syslog.logger.info('action=schema-update version=%s is current, '
'skipping' % version)
results.append(dict(name=version, action='skipped'))
else:
def _do_upgrade(revision, context):
return script._upgrade_revs(script.get_heads(), revision)
conn = self.engine.connect()
env.configure(connection=conn,
target_metadata=metadata,
fn=_do_upgrade)
with env.begin_transaction():
env.run_migrations()
results.append(dict(name=script.get_heads()[0], action='migrated'))
Syslog.logger.info('action=schema-update finished migration, '
'version=%s' % script.get_heads()[0])
if (version is None):
# Seed a new db with host and volume records
record = Host(hostname=self.backup_host)
self.session.add(record)
self.session.flush()
self.session.add(
Volume(volume=Constants.DEFAULT_VOLUME,
path=Constants.SNAPSHOT_ROOT,
host_id=record.id))
self.session.commit()
return {
'status': 'ok' if errors == 0 else 'error',
'schema-update': results
}
def check_db_state():
with app.app_context():
config = app.extensions["migrate"].migrate.get_config()
script = alembic.script.ScriptDirectory.from_config(config)
heads = script.get_revisions(script.get_heads())
head_revs = frozenset(rev.revision for rev in heads)
def check(rev, context):
db_revs = frozenset(
rev.revision for rev in script.get_all_current(rev))
if db_revs ^ head_revs:
config.print_stdout(
"Current revision(s) for %s %s do not match the heads %s\n.Run ./manage.sh db upgrade.",
alembic.util.obfuscate_url_pw(
context.connection.engine.url),
tuple(db_revs),
tuple(head_revs),
)
sys.exit(1)
return []
with alembic.runtime.environment.EnvironmentContext(
config, script, fn=check):
script.run_env()
def setUp(self, mock_log):
super(TestSchemaUpdate, self).setUp()
test_config = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'..', '..', 'etc', 'backup-daily.conf')
self.cli.update({
'db-url': 'sqlite:///:memory:',
'rsnapshot-conf': test_config
})
Syslog.logger = Syslog(self.cli)
cfg = alembic.config.Config()
cfg.set_main_option(
'script_location',
os.path.join(os.path.abspath(os.path.dirname(__file__)), '..',
'secondshot', 'alembic'))
cfg.set_main_option('url', str(self.engine.url))
script = alembic.script.ScriptDirectory.from_config(cfg)
self.alembic_ver = script.get_heads()[0]
def _do_upgrade(revision, context):
return script._upgrade_revs(script.get_heads(), revision)
def alembic_migrate(models, version, script_location, migrate=False,
db_session=None, schema='apicrud', schema_maxtime=0,
seed_func=None):
"""run schema migrations
Args:
models (obj): the models file object
version (str): schema version expected after migration
script_location (str): relative path name of alembic's env.py
migrate (bool): whether to run alembic migrations
db_session (obj): existing db session
schema (str): name of schema
schema_maxtime (int): how long to wait for migration
seed_func (function): function to seed initial records in blank db
"""
start = utcnow().timestamp()
cfg = alembic.config.Config()
cfg.set_main_option('script_location', script_location)
script = alembic.script.ScriptDirectory.from_config(cfg)
env = EnvironmentContext(cfg, script)
logmsg = dict(action='schema_update', name=schema, version=version)
if (version == script.get_heads()[0]):
logging.info(dict(message='is current', duration='%.3f' %
(utcnow().timestamp() - start), **logmsg))
elif migrate:
def _do_upgrade(revision, context):
return script._upgrade_revs(script.get_heads(), revision)
conn = None
while schema_maxtime > 0:
try:
conn = db_engine.connect()
break
except OperationalError as ex:
logging.info('action=alembic_migrate status=waiting message=%s'
% str(ex))
schema_maxtime -= 10
time.sleep(10)
if not conn:
logging.error('action=alembic_migrate status=timeout')
sys.exit(1)
if db_engine.dialect.name == 'sqlite' and spatialite_loaded:
conn.execute(select([func.InitSpatialMetaData(1)]))
env.configure(connection=conn, target_metadata=Base.metadata,
verbose=True, fn=_do_upgrade,
version_table='alembic_version_%s' % schema)
with env.begin_transaction():
env.run_migrations()
logging.info(dict(message='finished migration', duration='%0.3f' %
(utcnow().timestamp() - start), **logmsg))
else:
# Not migrating: must wait
wait_time = schema_maxtime
while version != script.get_heads()[0] and wait_time:
time.sleep(5)
wait_time -= 5
if (version is None and seed_func and
schema == ServiceConfig().config.DB_SCHEMAS[-1]):
if not db_session:
db_session = get_session(scoped=True)
seed_func(db_session)
db_session.close()