def create_db_tables(self):
"""Populate the new database.
MySQLTestCase creates a new database for each test, but these need to
be populated with the appropriate tables. Before we can do that, we
must change the 'connection' option which the Neutron code knows to
look at.
Currently, the username and password options are hard-coded by
oslo.db and neutron/tests/functional/contrib/gate_hook.sh. Also,
we only support MySQL for now, but the groundwork for adding Postgres
is already laid.
"""
conn = ("mysql+pymysql://%(username)s:%(password)s"
"@127.0.0.1/%(db_name)s" % {
'username': test_base.DbFixture.USERNAME,
'password': test_base.DbFixture.PASSWORD,
'db_name': self.engine.url.database})
alembic_config = migration.get_neutron_config()
alembic_config.neutron_config = cfg.CONF
self.original_conn = cfg.CONF.database.connection
self.addCleanup(self._revert_connection_address)
cfg.CONF.set_override('connection', conn, group='database')
migration.do_alembic_command(alembic_config, 'upgrade', 'heads')
def test_external_tables_not_changed(self):
def block_external_tables(conn, clauseelement, multiparams, params):
if isinstance(clauseelement, sqlalchemy.sql.selectable.Select):
return
if isinstance(clauseelement, six.string_types) and any(name in clauseelement for name in external.TABLES):
self.fail("External table referenced by neutron core " "migration.")
if hasattr(clauseelement, "element"):
element = clauseelement.element
if element.name in external.TABLES or (
hasattr(clauseelement, "table") and element.table.name in external.TABLES
):
# Table 'nsxv_vdr_dhcp_bindings' was created in liberty,
# before NSXV has moved to separate repo.
if (
isinstance(clauseelement, sqlalchemy.sql.ddl.CreateTable)
and element.name == "nsxv_vdr_dhcp_bindings"
):
return
self.fail("External table referenced by neutron core " "migration.")
engine = self.get_engine()
cfg.CONF.set_override("connection", engine.url, group="database")
with engine.begin() as connection:
self.alembic_config.attributes["connection"] = connection
migration.do_alembic_command(self.alembic_config, "upgrade", "kilo")
with self._listener(engine, block_external_tables):
migration.do_alembic_command(self.alembic_config, "upgrade", "heads")
def _test_has_offline_migrations(self, revision, expected):
engine = self.get_engine()
cfg.CONF.set_override('connection', engine.url, group='database')
migration.do_alembic_command(self.alembic_config, 'upgrade', revision)
self.assertEqual(expected,
migration.has_offline_migrations(self.alembic_config,
'unused'))
def test_forbid_offline_migrations_starting_newton(self):
engine = self.get_engine()
cfg.CONF.set_override('connection', engine.url, group='database')
# the following revisions are Newton heads
for revision in ('5cd92597d11d', '5c85685d616d'):
migration.do_alembic_command(
self.alembic_config, 'upgrade', revision)
self.assertFalse(migration.has_offline_migrations(
self.alembic_config, 'unused'),
msg='Offline contract migration scripts are forbidden for Ocata+')
def _generate_schema_w_migrations(cls, engine):
alembic_configs = migration.get_alembic_configs()
with engine.connect() as conn:
for alembic_config in alembic_configs:
alembic_config.attributes['connection'] = conn
alembic_config.neutron_config = cfg.CONF
alembic_config.neutron_config.set_override(
'connection', str(engine.url), group='database')
migration.do_alembic_command(
alembic_config, 'upgrade', 'heads')
def test_forbid_offline_migrations_starting_newton(self):
engine = self.get_engine()
cfg.CONF.set_override("connection", engine.url, group="database")
# the following revisions are Newton heads
for revision in ("5cd92597d11d", "5c85685d616d"):
migration.do_alembic_command(self.alembic_config, "upgrade", revision)
self.assertFalse(
migration.has_offline_migrations(self.alembic_config, "unused"),
msg="Offline contract migration scripts are forbidden for Ocata+",
)
def _migrate_up(self, config, engine, dest, curr, with_data=False):
if with_data:
data = None
pre_upgrade = getattr(self, "_pre_upgrade_%s" % dest, None)
if pre_upgrade:
data = pre_upgrade(engine)
migration.do_alembic_command(config, "upgrade", dest)
if with_data:
check = getattr(self, "_check_%s" % dest, None)
if check and data:
check(engine, data)
def _migrate_down(self, config, engine, dest, curr, with_data=False):
# First upgrade it to current to do downgrade
if dest:
migration.do_alembic_command(config, 'downgrade', dest)
else:
meta = sqlalchemy.MetaData(bind=engine)
meta.drop_all()
if with_data:
post_downgrade = getattr(
self, "_post_downgrade_%s" % curr, None)
if post_downgrade:
post_downgrade(engine)
def test_check_mysql_engine(self):
engine = self.get_engine()
cfg.CONF.set_override('connection', engine.url, group='database')
with engine.begin() as connection:
self.alembic_config.attributes['connection'] = connection
migration.do_alembic_command(self.alembic_config, 'upgrade',
'heads')
insp = sqlalchemy.engine.reflection.Inspector.from_engine(engine)
# Test that table creation on MySQL only builds InnoDB tables
tables = insp.get_table_names()
self.assertTrue(len(tables) > 0,
"No tables found. Wrong schema?")
res = [table for table in tables if
insp.get_table_options(table)['mysql_engine'] != 'InnoDB'
and table != 'alembic_version']
self.assertEqual(0, len(res), "%s non InnoDB tables created" % res)
def test_check_mysql_engine(self):
engine = self.get_engine()
cfg.CONF.set_override("connection", engine.url, group="database")
with engine.begin() as connection:
self.alembic_config.attributes["connection"] = connection
migration.do_alembic_command(self.alembic_config, "upgrade", "heads")
insp = sqlalchemy.engine.reflection.Inspector.from_engine(engine)
# Test that table creation on MySQL only builds InnoDB tables
tables = insp.get_table_names()
self.assertGreater(len(tables), 0, "No tables found. Wrong schema?")
res = [
table
for table in tables
if insp.get_table_options(table)["mysql_engine"] != "InnoDB" and table != "alembic_version"
]
self.assertEqual(0, len(res), "%s non InnoDB tables created" % res)
def test_branches(self):
def check_expand_branch(conn, clauseelement, multiparams, params):
if isinstance(clauseelement, migration_help.DROP_OPERATIONS):
self.fail("Migration from expand branch contains drop command")
def check_contract_branch(conn, clauseelement, multiparams, params):
if isinstance(clauseelement, migration_help.CREATION_OPERATIONS):
# Skip tables that were created by mistake in contract branch
if hasattr(clauseelement, "element"):
element = clauseelement.element
if any(
[
isinstance(element, sqlalchemy.Table)
and element.name in ["ml2_geneve_allocations", "ml2_geneve_endpoints"],
isinstance(element, sqlalchemy.Index) and element.table.name == "ml2_geneve_allocations",
]
):
return
self.fail("Migration from contract branch contains create " "command")
engine = self.get_engine()
cfg.CONF.set_override("connection", engine.url, group="database")
with engine.begin() as connection:
self.alembic_config.attributes["connection"] = connection
migration.do_alembic_command(self.alembic_config, "upgrade", "kilo")
with self._listener(engine, check_expand_branch):
migration.do_alembic_command(self.alembic_config, "upgrade", "%s@head" % migration.EXPAND_BRANCH)
with self._listener(engine, check_contract_branch):
migration.do_alembic_command(self.alembic_config, "upgrade", "%s@head" % migration.CONTRACT_BRANCH)
def db_sync(self, engine):
cfg.CONF.set_override('connection', engine.url, group='database')
migration.do_alembic_command(self.alembic_config, 'upgrade', 'heads')
def db_sync(self, engine):
cfg.CONF.set_override('connection', engine.url, group='database')
for conf in migration.get_alembic_configs():
self.alembic_config = conf
self.alembic_config.neutron_config = cfg.CONF
migration.do_alembic_command(conf, 'upgrade', 'heads')
def _generate_schema_w_migrations(cls, engine):
alembic_config = migration.get_neutron_config()
with engine.connect() as conn:
alembic_config.attributes['connection'] = conn
migration.do_alembic_command(
alembic_config, 'upgrade', 'heads')
def upgrade(engine, alembic_config, branch_name='heads'):
cfg.CONF.set_override('connection', engine.url, group='database')
migration.do_alembic_command(alembic_config, 'upgrade',
branch_name)
def _generate_schema_w_migrations(cls, engine):
alembic_config = migration.get_neutron_config()
with engine.connect() as conn:
alembic_config.attributes['connection'] = conn
migration.do_alembic_command(alembic_config, 'upgrade', 'heads')
def db_sync(self, engine):
cfg.CONF.set_override("connection", engine.url, group="database")
migration.do_alembic_command(self.alembic_config, "upgrade", "heads")
def test_branches(self):
drop_exceptions = collections.defaultdict(list)
creation_exceptions = collections.defaultdict(list)
def find_migration_exceptions():
# Due to some misunderstandings and some conscious decisions,
# there may be some expand migrations which drop elements and
# some contract migrations which create elements. These excepted
# elements must be returned by a method in the script itself.
# The names of the method must be 'contract_creation_exceptions'
# or 'expand_drop_exceptions'. The methods must have a docstring
# explaining the reason for the exception.
#
# Here we build lists of the excepted elements and verify that
# they are documented.
script = alembic_script.ScriptDirectory.from_config(
self.alembic_config)
for m in list(script.walk_revisions(base='base', head='heads')):
branches = m.branch_labels or [None]
if migration.CONTRACT_BRANCH in branches:
method_name = 'contract_creation_exceptions'
exceptions_dict = creation_exceptions
elif migration.EXPAND_BRANCH in branches:
method_name = 'expand_drop_exceptions'
exceptions_dict = drop_exceptions
else:
continue
get_excepted_elements = getattr(m.module, method_name, None)
if not get_excepted_elements:
continue
explanation = getattr(get_excepted_elements, '__doc__', "")
if len(explanation) < 1:
self.fail(
"%s() requires docstring with explanation" % '.'.join([
m.module.__name__, get_excepted_elements.__name__
]))
for sa_type, elements in get_excepted_elements().items():
exceptions_dict[sa_type].extend(elements)
def is_excepted(clauseelement, exceptions):
# Identify elements that are an exception for the branch
element = clauseelement.element
element_name = element.name
if isinstance(element, sqlalchemy.Index):
element_name = element.table.name
for sa_type_, excepted_names in exceptions.items():
if isinstance(element, sa_type_):
if element_name in excepted_names:
return True
def check_expand_branch(conn, clauseelement, multiparams, params):
if not (isinstance(clauseelement, migration_help.DROP_OPERATIONS)
and hasattr(clauseelement, 'element')):
return
# Skip drops that have been explicitly excepted
if is_excepted(clauseelement, drop_exceptions):
return
self.fail("Migration in expand branch contains drop command")
def check_contract_branch(conn, clauseelement, multiparams, params):
if not (isinstance(clauseelement,
migration_help.CREATION_OPERATIONS)
and hasattr(clauseelement, 'element')):
return
# Skip creations that have been explicitly excepted
if is_excepted(clauseelement, creation_exceptions):
return
self.fail("Migration in contract branch contains create command")
find_migration_exceptions()
engine = self.get_engine()
cfg.CONF.set_override('connection', engine.url, group='database')
with engine.begin() as connection:
self.alembic_config.attributes['connection'] = connection
migration.do_alembic_command(self.alembic_config, 'upgrade',
'kilo')
with self._listener(engine, check_expand_branch):
migration.do_alembic_command(
self.alembic_config, 'upgrade',
'%s@head' % migration.EXPAND_BRANCH)
with self._listener(engine, check_contract_branch):
migration.do_alembic_command(
self.alembic_config, 'upgrade',
'%s@head' % migration.CONTRACT_BRANCH)
def upgrade(engine, alembic_config, branch_name="heads"):
cfg.CONF.set_override("connection", engine.url, group="database")
migration.do_alembic_command(alembic_config, "upgrade", branch_name)
def db_sync(self, engine):
cfg.CONF.set_override('connection', engine.url, group='database')
for conf in migration.get_alembic_configs():
self.alembic_config = conf
self.alembic_config.neutron_config = cfg.CONF
migration.do_alembic_command(conf, 'upgrade', 'heads')
def _test_has_offline_migrations(self, revision, expected):
engine = self.get_engine()
cfg.CONF.set_override("connection", engine.url, group="database")
migration.do_alembic_command(self.alembic_config, "upgrade", revision)
self.assertEqual(expected, migration.has_offline_migrations(self.alembic_config, "unused"))
def upgrade(engine, alembic_config, branch_name='heads'):
cfg.CONF.set_override('connection', engine.url, group='database')
migration.do_alembic_command(alembic_config, 'upgrade', branch_name)
def test_branches(self):
drop_exceptions = collections.defaultdict(list)
creation_exceptions = collections.defaultdict(list)
def find_migration_exceptions():
# Due to some misunderstandings and some conscious decisions,
# there may be some expand migrations which drop elements and
# some contract migrations which create elements. These excepted
# elements must be returned by a method in the script itself.
# The names of the method must be 'contract_creation_exceptions'
# or 'expand_drop_exceptions'. The methods must have a docstring
# explaining the reason for the exception.
#
# Here we build lists of the excepted elements and verify that
# they are documented.
script = alembic_script.ScriptDirectory.from_config(
self.alembic_config)
for m in list(script.walk_revisions(base='base', head='heads')):
branches = m.branch_labels or [None]
if migration.CONTRACT_BRANCH in branches:
method_name = 'contract_creation_exceptions'
exceptions_dict = creation_exceptions
elif migration.EXPAND_BRANCH in branches:
method_name = 'expand_drop_exceptions'
exceptions_dict = drop_exceptions
else:
continue
get_excepted_elements = getattr(m.module, method_name, None)
if not get_excepted_elements:
continue
explanation = getattr(get_excepted_elements, '__doc__', "")
if len(explanation) < 1:
self.fail("%s() requires docstring with explanation" %
'.'.join([m.module.__name__,
get_excepted_elements.__name__]))
for sa_type, elements in get_excepted_elements().items():
exceptions_dict[sa_type].extend(elements)
def is_excepted(clauseelement, exceptions):
# Identify elements that are an exception for the branch
element = clauseelement.element
element_name = element.name
if isinstance(element, sqlalchemy.Index):
element_name = element.table.name
for sa_type_, excepted_names in exceptions.items():
if isinstance(element, sa_type_):
if element_name in excepted_names:
return True
def check_expand_branch(conn, clauseelement, multiparams, params):
if not (isinstance(clauseelement,
migration_help.DROP_OPERATIONS) and
hasattr(clauseelement, 'element')):
return
# Skip drops that have been explicitly excepted
if is_excepted(clauseelement, drop_exceptions):
return
self.fail("Migration in expand branch contains drop command")
def check_contract_branch(conn, clauseelement, multiparams, params):
if not (isinstance(clauseelement,
migration_help.CREATION_OPERATIONS) and
hasattr(clauseelement, 'element')):
return
# Skip creations that have been explicitly excepted
if is_excepted(clauseelement, creation_exceptions):
return
self.fail("Migration in contract branch contains create command")
find_migration_exceptions()
engine = self.get_engine()
cfg.CONF.set_override('connection', engine.url, group='database')
with engine.begin() as connection:
self.alembic_config.attributes['connection'] = connection
migration.do_alembic_command(self.alembic_config, 'upgrade',
'kilo')
with self._listener(engine, check_expand_branch):
migration.do_alembic_command(
self.alembic_config, 'upgrade',
'%s@head' % migration.EXPAND_BRANCH)
with self._listener(engine, check_contract_branch):
migration.do_alembic_command(
self.alembic_config, 'upgrade',
'%s@head' % migration.CONTRACT_BRANCH)
def db_sync(self, engine):
cfg.CONF.set_override('connection', engine.url, group='database')
migration.do_alembic_command(self.alembic_config, 'upgrade', 'heads')
def test_branches(self):
drop_exceptions = collections.defaultdict(list)
creation_exceptions = collections.defaultdict(list)
def find_migration_exceptions():
# Due to some misunderstandings and some conscious decisions,
# there may be some expand migrations which drop elements and
# some contract migrations which create elements. These excepted
# elements must be returned by a method in the script itself.
# The names of the method must be 'contract_creation_exceptions'
# or 'expand_drop_exceptions'. The methods must have a docstring
# explaining the reason for the exception.
#
# Here we build lists of the excepted elements and verify that
# they are documented.
script = alembic_script.ScriptDirectory.from_config(
self.alembic_config)
for m in list(script.walk_revisions(base='base', head='heads')):
branches = m.branch_labels or []
if migration.CONTRACT_BRANCH in branches:
method_name = 'contract_creation_exceptions'
exceptions_dict = creation_exceptions
elif migration.EXPAND_BRANCH in branches:
method_name = 'expand_drop_exceptions'
exceptions_dict = drop_exceptions
else:
continue
get_excepted_elements = getattr(m.module, method_name, None)
if not get_excepted_elements:
continue
explanation = getattr(get_excepted_elements, '__doc__', "")
if len(explanation) < 1:
self.fail("%s() requires docstring with explanation" %
'.'.join([m.module.__name__,
get_excepted_elements.__name__]))
for sa_type, elements in get_excepted_elements().items():
exceptions_dict[sa_type].extend(elements)
def is_excepted_sqla(clauseelement, exceptions):
"""Identify excepted operations that are allowed for the branch."""
element = clauseelement.element
element_name = element.name
if isinstance(element, sqlalchemy.Index):
element_name = element.table.name
for sa_type_, excepted_names in exceptions.items():
if isinstance(element, sa_type_):
if element_name in excepted_names:
return True
def is_excepted_alembic(clauseelement, exceptions):
"""Identify excepted operations that are allowed for the branch."""
# For alembic the clause is AddColumn or DropColumn
column = clauseelement.column.name
table = clauseelement.column.table.name
element_name = '.'.join([table, column])
for alembic_type, excepted_names in exceptions.items():
if alembic_type == sqlalchemy.Column:
if element_name in excepted_names:
return True
def is_allowed(clauseelement, exceptions, disallowed_ops):
if (isinstance(clauseelement, disallowed_ops['sqla']) and
hasattr(clauseelement, 'element')):
return is_excepted_sqla(clauseelement, exceptions)
if isinstance(clauseelement, disallowed_ops['alembic']):
return is_excepted_alembic(clauseelement, exceptions)
return True
def check_expand_branch(conn, clauseelement, multiparams, params):
if not is_allowed(clauseelement, drop_exceptions, DROP_OPERATIONS):
self.fail("Migration in expand branch contains drop command")
def check_contract_branch(conn, clauseelement, multiparams, params):
if not is_allowed(clauseelement, creation_exceptions,
CREATION_OPERATIONS):
self.fail("Migration in contract branch contains create "
"command")
find_migration_exceptions()
engine = self.engine
cfg.CONF.set_override('connection', engine.url, group='database')
with engine.begin() as connection:
self.alembic_config.attributes['connection'] = connection
# upgrade to latest release first; --expand users are expected to
# apply all alembic scripts from previous releases before applying
# the new ones
for release in migration_root.NEUTRON_MILESTONES:
release_revisions = migration._find_milestone_revisions(
self.alembic_config, release)
for rev in release_revisions:
migration.do_alembic_command(
self.alembic_config, 'upgrade', rev[0])
with self._listener(engine, check_expand_branch):
migration.do_alembic_command(
self.alembic_config, 'upgrade',
'%s@head' % migration.EXPAND_BRANCH)
with self._listener(engine, check_contract_branch):
migration.do_alembic_command(
self.alembic_config, 'upgrade',
'%s@head' % migration.CONTRACT_BRANCH)
def db_sync(self, engine):
cfg.CONF.set_override("connection", engine.url, group="database")
for conf in migration.get_alembic_configs():
self.alembic_config = conf
self.alembic_config.neutron_config = cfg.CONF
migration.do_alembic_command(conf, "upgrade", "heads")
def test_branches(self):
drop_exceptions = collections.defaultdict(list)
creation_exceptions = collections.defaultdict(list)
def find_migration_exceptions():
# Due to some misunderstandings and some conscious decisions,
# there may be some expand migrations which drop elements and
# some contract migrations which create elements. These excepted
# elements must be returned by a method in the script itself.
# The names of the method must be 'contract_creation_exceptions'
# or 'expand_drop_exceptions'. The methods must have a docstring
# explaining the reason for the exception.
#
# Here we build lists of the excepted elements and verify that
# they are documented.
script = alembic_script.ScriptDirectory.from_config(
self.alembic_config)
for m in list(script.walk_revisions(base='base', head='heads')):
branches = m.branch_labels or []
if migration.CONTRACT_BRANCH in branches:
method_name = 'contract_creation_exceptions'
exceptions_dict = creation_exceptions
elif migration.EXPAND_BRANCH in branches:
method_name = 'expand_drop_exceptions'
exceptions_dict = drop_exceptions
else:
continue
get_excepted_elements = getattr(m.module, method_name, None)
if not get_excepted_elements:
continue
explanation = getattr(get_excepted_elements, '__doc__', "")
if len(explanation) < 1:
self.fail(
"%s() requires docstring with explanation" % '.'.join([
m.module.__name__, get_excepted_elements.__name__
]))
for sa_type, elements in get_excepted_elements().items():
exceptions_dict[sa_type].extend(elements)
def is_excepted_sqla(clauseelement, exceptions):
"""Identify excepted operations that are allowed for the branch."""
element = clauseelement.element
element_name = element.name
if isinstance(element, sqlalchemy.Index):
element_name = element.table.name
for sa_type_, excepted_names in exceptions.items():
if isinstance(element, sa_type_):
if element_name in excepted_names:
return True
def is_excepted_alembic(clauseelement, exceptions):
"""Identify excepted operations that are allowed for the branch."""
# For alembic the clause is AddColumn or DropColumn
column = clauseelement.column.name
table = clauseelement.column.table.name
element_name = '.'.join([table, column])
for alembic_type, excepted_names in exceptions.items():
if alembic_type == sqlalchemy.Column:
if element_name in excepted_names:
return True
def is_allowed(clauseelement, exceptions, disallowed_ops):
if (isinstance(clauseelement, disallowed_ops['sqla'])
and hasattr(clauseelement, 'element')):
return is_excepted_sqla(clauseelement, exceptions)
if isinstance(clauseelement, disallowed_ops['alembic']):
return is_excepted_alembic(clauseelement, exceptions)
return True
def check_expand_branch(conn, clauseelement, multiparams, params):
if not is_allowed(clauseelement, drop_exceptions, DROP_OPERATIONS):
self.fail("Migration in expand branch contains drop command")
def check_contract_branch(conn, clauseelement, multiparams, params):
if not is_allowed(clauseelement, creation_exceptions,
CREATION_OPERATIONS):
self.fail("Migration in contract branch contains create "
"command")
find_migration_exceptions()
engine = self.engine
cfg.CONF.set_override('connection', engine.url, group='database')
with engine.begin() as connection:
self.alembic_config.attributes['connection'] = connection
# upgrade to latest release first; --expand users are expected to
# apply all alembic scripts from previous releases before applying
# the new ones
for release in migration_root.NEUTRON_MILESTONES:
release_revisions = migration._find_milestone_revisions(
self.alembic_config, release)
for rev in release_revisions:
migration.do_alembic_command(self.alembic_config,
'upgrade', rev[0])
with self._listener(engine, check_expand_branch):
migration.do_alembic_command(
self.alembic_config, 'upgrade',
'%s@head' % migration.EXPAND_BRANCH)
with self._listener(engine, check_contract_branch):
migration.do_alembic_command(
self.alembic_config, 'upgrade',
'%s@head' % migration.CONTRACT_BRANCH)
