def _migrate_db_sync(engine, abs_path, version=None, init_version=0):
"""Upgrade or downgrade a database.
Function runs the upgrade() or downgrade() functions in change scripts.
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository.
:param version: Database will upgrade/downgrade until this version.
If None - database will update to the latest available version.
:param init_version: Initial database version
"""
if version is not None:
try:
version = int(version)
except ValueError:
msg = _("version should be an integer")
raise db_exception.DBMigrationError(msg)
current_version = _migrate_db_version(engine, abs_path, init_version)
repository = _find_migrate_repo(abs_path)
if version is None or version > current_version:
try:
return migrate_api.upgrade(engine, repository, version)
except Exception as ex:
raise db_exception.DBMigrationError(ex)
else:
return migrate_api.downgrade(engine, repository, version)
def db_sync(engine, abs_path, version=None, init_version=0, sanity_check=True):
"""Upgrade or downgrade a database.
Function runs the upgrade() or downgrade() functions in change scripts.
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository.
:param version: Database will upgrade/downgrade until this version.
If None - database will update to the latest
available version.
:param init_version: Initial database version
:param sanity_check: Require schema sanity checking for all tables
"""
if version is not None:
try:
version = int(version)
except ValueError:
raise exception.DBMigrationError(_("version should be an integer"))
current_version = db_version(engine, abs_path, init_version)
repository = _find_migrate_repo(abs_path)
if sanity_check:
_db_schema_sanity_check(engine)
if version is None or version > current_version:
try:
migration = versioning_api.upgrade(engine, repository, version)
except Exception as ex:
raise exception.DBMigrationError(ex)
else:
migration = versioning_api.downgrade(engine, repository, version)
if sanity_check:
_db_schema_sanity_check(engine)
return migration
def validate_upgrade_order(repo_name, target_repo_version=None):
"""Validate the state of the migration repositories.
This is run before allowing the db_sync command to execute. Ensure the
upgrade step and version specified by the operator remains consistent with
the upgrade process. I.e. expand's version is greater or equal to
migrate's, migrate's version is greater or equal to contract's.
:param repo_name: The name of the repository that the user is trying to
upgrade.
:param target_repo_version: The version to upgrade the repo. Otherwise, the
version will be upgraded to the latest version
available.
"""
# Initialize a dict to have each key assigned a repo with their value being
# the repo that comes before.
db_sync_order = {
DATA_MIGRATION_REPO: EXPAND_REPO,
CONTRACT_REPO: DATA_MIGRATION_REPO
}
if repo_name == LEGACY_REPO:
return
# If expand is being run, we validate that Legacy repo is at the maximum
# version before running the additional schema expansions.
elif repo_name == EXPAND_REPO:
abs_path = find_repo(LEGACY_REPO)
repo = migrate.versioning.repository.Repository(abs_path)
if int(repo.latest) != get_db_version():
raise db_exception.DBMigrationError(
'Your Legacy repo version is not up to date. Please refer to '
'https://docs.openstack.org/keystone/latest/admin/'
'identity-upgrading.html '
'to see the proper steps for rolling upgrades.')
return
# find the latest version that the current command will upgrade to if there
# wasn't a version specified for upgrade.
if not target_repo_version:
abs_path = find_repo(repo_name)
repo = migrate.versioning.repository.Repository(abs_path)
target_repo_version = int(repo.latest)
# get current version of the command that runs before the current command.
dependency_repo_version = get_db_version(repo=db_sync_order[repo_name])
if dependency_repo_version < target_repo_version:
raise db_exception.DBMigrationError(
'You are attempting to upgrade %s ahead of %s. Please refer to '
'https://docs.openstack.org/keystone/latest/admin/'
'identity-upgrading.html '
'to see the proper steps for rolling upgrades.' %
(repo_name, db_sync_order[repo_name]))
def _migrate_db_version(engine, abs_path, init_version):
"""Show the current version of the repository.
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository
:param init_version: Initial database version
"""
repository = _find_migrate_repo(abs_path)
try:
return migrate_api.db_version(engine, repository)
except migrate_exceptions.DatabaseNotControlledError:
pass
meta = sa.MetaData()
meta.reflect(bind=engine)
tables = meta.tables
if (
len(tables) == 0 or
'alembic_version' in tables or
'migrate_version' in tables
):
_migrate_db_version_control(engine, abs_path, version=init_version)
return migrate_api.db_version(engine, repository)
msg = _(
"The database is not under version control, but has tables. "
"Please stamp the current version of the schema manually."
)
raise db_exception.DBMigrationError(msg)
def _find_migrate_repo(abs_path):
"""Get the project's change script repository
:param abs_path: Absolute path to migrate repository
"""
if not os.path.exists(abs_path):
raise exception.DBMigrationError("Path %s not found" % abs_path)
return Repository(abs_path)
def get_manager():
storage_db = CONF['storage:sqlalchemy'].connection
pool_manager_cache_db = CONF['pool_manager_cache:sqlalchemy'].connection
if storage_db == pool_manager_cache_db:
raise exception.DBMigrationError(
message=("Pool Manager Cache requires its own database."
" Please check your config file."))
else:
return utils.get_migration_manager(REPOSITORY, pool_manager_cache_db)
def db_version_control(engine, abs_path, version=None):
"""Mark a database as under this repository's version control.
Once a database is under version control, schema changes should
only be done via change scripts in this repository.
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository
:param version: Initial database version
"""
repository = _find_migrate_repo(abs_path)
try:
versioning_api.version_control(engine, repository, version)
except versioning_exceptions.InvalidVersionError as ex:
raise exception.DBMigrationError("Invalid version : %s" % ex)
except versioning_exceptions.DatabaseAlreadyControlledError:
raise exception.DBMigrationError("Database is already controlled.")
return version
def create_schema(config=None, engine=None):
"""Create database schema from models description.
Can be used for initial installation instead of upgrade('head').
"""
if engine is None:
engine = enginefacade.get_legacy_facade().get_engine()
if version(engine=engine) is not None:
raise db_exc.DBMigrationError("DB schema is already under version"
" control. Use upgrade() instead")
models.Base.metadata.create_all(engine)
stamp('head', config=config)
def create_schema(config=None, engine=None):
"""Create database schema from models description.
Can be used for initial installation instead of upgrade('head').
"""
if engine is None:
engine = enginefacade.get_legacy_facade().get_engine()
# NOTE(viktors): If we will use metadata.create_all() for non empty db
# schema, it will only add the new tables, but leave
# existing as is. So we should avoid of this situation.
if version(engine=engine) is not None:
raise db_exc.DBMigrationError("DB schema is already under version"
" control. Use upgrade() instead")
models.Base.metadata.create_all(engine)
stamp('head', config=config)
def upgrade(self, revision):
"""Upgrade database with all available backends."""
# a revision exists only in a single plugin. Until we reached it, we
# should upgrade to the plugins' heads.
# revision=None is a special case meaning latest revision.
rev_in_plugins = [p.has_revision(revision) for p in self._plugins]
if not any(rev_in_plugins) and revision is not None:
raise exception.DBMigrationError('Revision does not exist')
results = []
for plugin, has_revision in zip(self._plugins, rev_in_plugins):
if not has_revision or revision is None:
results.append(plugin.upgrade(None))
else:
results.append(plugin.upgrade(revision))
break
return results
def schema_create(self, config=None, engine=None):
"""Create database schema from models description.
Can be used for initial installation instead of upgrade('head').
:param config: Instance of alembic config
:param engine: Instance of DB engine
"""
engine = engine or get_engine()
# NOTE(viktors): If we will use metadata.create_all() for non empty db
# schema, it will only add the new tables, but leave
# existing as is. So we should avoid of this situation.
if self.schema_revision(engine=engine) is not None:
raise db_exc.DBMigrationError("DB schema is already under version"
" control. Use upgrade() instead")
models.BASE.metadata.create_all(engine)
self.schema_stamp("head", config=config)
def downgrade(self, revision):
"""Downgrade database with available backends."""
# a revision exists only in a single plugin. Until we reached it, we
# should upgrade to the plugins' first revision.
# revision=None is a special case meaning initial revision.
rev_in_plugins = [p.has_revision(revision) for p in self._plugins]
if not any(rev_in_plugins) and revision is not None:
raise exception.DBMigrationError('Revision does not exist')
# downgrading should be performed in reversed order
results = []
for plugin, has_revision in zip(reversed(self._plugins),
reversed(rev_in_plugins)):
if not has_revision or revision is None:
results.append(plugin.downgrade(None))
else:
results.append(plugin.downgrade(revision))
break
return results
def _validate_upgrade_order(branch, target_repo_version=None):
"""Validate the state of the migration repositories.
This is run before allowing the db_sync command to execute. Ensure the
upgrade step and version specified by the operator remains consistent with
the upgrade process. I.e. expand's version is greater or equal to
migrate's, migrate's version is greater or equal to contract's.
:param branch: The name of the repository that the user is trying to
upgrade.
:param target_repo_version: The version to upgrade the repo. Otherwise, the
version will be upgraded to the latest version
available.
"""
# Initialize a dict to have each key assigned a repo with their value being
# the repo that comes before.
db_sync_order = {
DATA_MIGRATION_BRANCH: EXPAND_BRANCH,
CONTRACT_BRANCH: DATA_MIGRATION_BRANCH,
}
if branch == EXPAND_BRANCH:
return
# find the latest version that the current command will upgrade to if there
# wasn't a version specified for upgrade.
if not target_repo_version:
abs_path = _get_migrate_repo_path(branch)
repo = _find_migrate_repo(abs_path)
target_repo_version = int(repo.latest)
# get current version of the command that runs before the current command.
dependency_repo_version = get_db_version(branch=db_sync_order[branch])
if dependency_repo_version < target_repo_version:
raise db_exception.DBMigrationError(
'You are attempting to upgrade %s ahead of %s. Please refer to '
'https://docs.openstack.org/keystone/latest/admin/'
'identity-upgrading.html '
'to see the proper steps for rolling upgrades.' % (
branch, db_sync_order[branch]))
def db_version(engine, abs_path, init_version):
"""Show the current version of the repository.
:param engine: SQLAlchemy engine instance for a given database
:param abs_path: Absolute path to migrate repository
:param init_version: Initial database version
"""
repository = _find_migrate_repo(abs_path)
try:
return versioning_api.db_version(engine, repository)
except versioning_exceptions.DatabaseNotControlledError:
meta = sqlalchemy.MetaData()
meta.reflect(bind=engine)
tables = meta.tables
if len(tables) == 0 or 'alembic_version' in tables:
db_version_control(engine, abs_path, version=init_version)
return versioning_api.db_version(engine, repository)
else:
raise exception.DBMigrationError(
_("The database is not under version control, but has "
"tables. Please stamp the current version of the schema "
"manually."))
def _fake_upgrade_boom(*args, **kwargs):
raise exc.DBMigrationError("boom")
def test_db_commands_script_not_present(self, db_sync):
db_sync.side_effect = db_exc.DBMigrationError(None)
db_cmds = karbor_manage.DbCommands()
exit = self.assertRaises(SystemExit, db_cmds.sync, 101)
self.assertEqual(1, exit.code)
