async def test_dbschema_update_legacy_contained( postgresql_client: asyncpg.Connection, get_columns_in_db_table, hard_clean_db, hard_clean_db_post): """ Verify that the legacy row migration is contained to the instance it is called on. """ await postgresql_client.execute(""" -- Table: public.schemamanager CREATE TABLE IF NOT EXISTS public.schemamanager( name varchar PRIMARY KEY, current_version integer NOT NULL ); """) await postgresql_client.execute( "INSERT INTO public.schemamanager(name, current_version) VALUES ($1, $2);", "myslice", 1) await postgresql_client.execute( "INSERT INTO public.schemamanager(name, current_version) VALUES ($1, $2);", "otherslice", 3) dbm1 = schema.DBSchema("myslice", versions, postgresql_client) dbm2 = schema.DBSchema("otherslice", versions, postgresql_client) await dbm1.ensure_self_update() assert await dbm1.get_legacy_version() == 1 assert await dbm1.get_installed_versions() == {1} assert await dbm2.get_legacy_version() == 3 assert await dbm2.get_installed_versions() == set() await dbm2.ensure_self_update() assert await dbm1.get_legacy_version() == 1 assert await dbm1.get_installed_versions() == {1} assert await dbm2.get_legacy_version() == 3 assert await dbm2.get_installed_versions() == {1, 3}
async def get_core_versions(postgresql_client) -> Set[int]: dbm = schema.DBSchema(CORE_SCHEMA_NAME, inmanta.db.versions, postgresql_client) try: return await dbm.get_installed_versions() except TableNotFound: return set()
async def test_dbschema_update_db_schema(postgresql_client, get_columns_in_db_table, hard_clean_db, hard_clean_db_post): db_schema = schema.DBSchema("test_dbschema_update_db_schema", inmanta.db.versions, postgresql_client) await db_schema.ensure_self_update() await run_updates_and_verify(get_columns_in_db_table, db_schema, set()) db_schema = schema.DBSchema("test_dbschema_update_db_schema_1", inmanta.db.versions, postgresql_client) await run_updates_and_verify(get_columns_in_db_table, db_schema, set(), prefix="c")
async def test_dbschema_get_dct_filter_disabled(): db_schema = schema.DBSchema(CORE_SCHEMA_NAME, versions, None) update_function_map = db_schema._get_update_functions() assert {v.version for v in update_function_map} == {1, 3} for version in update_function_map: assert version.version >= 0 assert isinstance(version.function, types.FunctionType) assert version.function.__name__ == "update" assert inspect.getfullargspec(version.function)[0] == ["connection"]
async def test_dbschema_update_legacy_table_concurrent(postgres_db, database_name, get_columns_in_db_table, hard_clean_db, hard_clean_db_post): """ Verify that no conflicts arise from multiple concurrent processes trying to migrate from the legacy table. """ client1 = await asyncpg.connect( host=postgres_db.host, port=postgres_db.port, user=postgres_db.user, password=postgres_db.password, database=database_name, ) client2 = await asyncpg.connect( host=postgres_db.host, port=postgres_db.port, user=postgres_db.user, password=postgres_db.password, database=database_name, ) await client1.execute(""" -- Table: public.schemamanager CREATE TABLE IF NOT EXISTS public.schemamanager( name varchar PRIMARY KEY, current_version integer NOT NULL ); """) dbm1 = schema.DBSchema("myslice", inmanta.db.versions, client1) dbm2 = schema.DBSchema("otherslice", inmanta.db.versions, client2) assert set(await get_columns_in_db_table("schemamanager")) == { "name", "current_version" } await asyncio.gather( dbm1._legacy_migration_table(), dbm2._legacy_migration_table(), ) assert set(await get_columns_in_db_table("schemamanager")) == { "name", "legacy_version", "installed_versions" } assert await dbm1.get_installed_versions() == set() assert await dbm2.get_installed_versions() == set()
async def assert_core_untouched(postgresql_client, corev: Optional[Set[int]] = None): """ Verify abscence of side-effect leaks to other cases """ if corev is None: corev = set() dbm = schema.DBSchema(CORE_SCHEMA_NAME, inmanta.db.versions, postgresql_client) current_db_versions = await dbm.get_installed_versions() assert current_db_versions == corev
async def test_dbschema_partial_update_db_schema_failure( postgresql_client, get_columns_in_db_table): corev: Set[int] = await get_core_versions(postgresql_client) db_schema = schema.DBSchema( "test_dbschema_partial_update_db_schema_failure", inmanta.db.versions, postgresql_client) await db_schema.ensure_self_update() async def update_function_good(connection): # Fix syntax issue await connection.execute( "CREATE TABLE public.taba(id integer primary key, val varchar NOT NULL);" ) async def update_function_bad(connection): # Syntax error should trigger database rollback await connection.execute( "CREATE TABLE public.tabb(id integer primary key, val varchar NOT NULL);" ) await connection.execute( "CREATE TABE public.tab(id integer primary key, val varchar NOT NULL);" ) async def update_function_good2(connection): # Fix syntax issue await connection.execute( "CREATE TABLE public.tabc(id integer primary key, val varchar NOT NULL);" ) current_db_versions: Set[int] = await db_schema.get_installed_versions() assert len(current_db_versions) == 0 update_function_map = make_versions(1, update_function_good, update_function_bad, update_function_good2) with pytest.raises(PostgresSyntaxError): await db_schema._update_db_schema(update_function_map) # Assert full rollback assert (await db_schema.get_installed_versions()) == set() assert (await postgresql_client.fetchval( "SELECT table_name FROM information_schema.tables " "WHERE table_schema='public' AND table_name='taba'")) is None assert (await postgresql_client.fetchval( "SELECT table_name FROM information_schema.tables " "WHERE table_schema='public' AND table_name='tabb'")) is None assert (await postgresql_client.fetchval( "SELECT table_name FROM information_schema.tables " "WHERE table_schema='public' AND table_name='tabc'")) is None await assert_core_untouched(postgresql_client, corev)
async def test_dbschema_clean(postgresql_client: asyncpg.Connection, get_columns_in_db_table, hard_clean_db, caplog): with caplog.at_level(logging.INFO): dbm = schema.DBSchema("test_dbschema_clean", inmanta.db.versions, postgresql_client) with pytest.raises(TableNotFound): await dbm.get_installed_versions() await dbm.ensure_self_update() await run_updates_and_verify(get_columns_in_db_table, dbm, set()) await assert_core_untouched(postgresql_client) log_contains(caplog, "inmanta.data.schema.schema:test_dbschema_clean", logging.INFO, "Creating schema version table")
async def test_dbschema_get_dct_filter_invalid_names(caplog): db_schema = schema.DBSchema(CORE_SCHEMA_NAME, invalid_versions, None) update_function_map = db_schema._get_update_functions() assert len(update_function_map) == 0 log_contains(caplog, "inmanta.data.schema", logging.WARNING, "V2 doesn't match the expected pattern") log_contains(caplog, "inmanta.data.schema", logging.WARNING, "ver1 doesn't match the expected pattern") log_contains(caplog, "inmanta.data.schema", logging.WARNING, "ver1a doesn't match the expected pattern") log_contains(caplog, "inmanta.data.schema", logging.WARNING, "version3 doesn't match the expected pattern") log_contains(caplog, "inmanta.data.schema", logging.WARNING, "v1b doesn't match the expected pattern")
async def test_dbschema_unclean(postgresql_client: asyncpg.Connection, get_columns_in_db_table, hard_clean_db): dbm = schema.DBSchema("test_dbschema_unclean", inmanta.db.versions, postgresql_client) await dbm.ensure_self_update() assert await dbm.get_installed_versions() == set() await dbm.set_installed_version(5) current_versions: Set[int] = await dbm.get_installed_versions() assert current_versions == {5} await run_updates_and_verify(get_columns_in_db_table, dbm, current_versions) await assert_core_untouched(postgresql_client)
async def test_dbschema_update_db_schema_failure(postgresql_client, get_columns_in_db_table): corev: Set[int] = await get_core_versions(postgresql_client) db_schema = schema.DBSchema("test_dbschema_update_db_schema_failure", inmanta.db.versions, postgresql_client) await db_schema.ensure_self_update() async def update_function(connection): # Syntax error should trigger database rollback await connection.execute( "CREATE TABE public.tab(id integer primary key, val varchar NOT NULL);" ) current_db_versions: Set[int] = await db_schema.get_installed_versions() assert len(current_db_versions) == 0 new_db_version = 1 update_function_map = [Version(f"v{new_db_version}", update_function)] with pytest.raises(PostgresSyntaxError): await db_schema._update_db_schema(update_function_map) # Assert rollback assert (await db_schema.get_installed_versions()) == current_db_versions assert (await postgresql_client.fetchval( "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_name='tab'" )) is None async def update_function2(connection): # Fix syntax issue await connection.execute( "CREATE TABLE public.tab(id integer primary key, val varchar NOT NULL);" ) update_function_map = [Version(f"v{new_db_version}", update_function2)] await db_schema._update_db_schema(update_function_map) # Assert update assert (await db_schema.get_installed_versions()) == current_db_versions.union( {new_db_version}) assert sorted(["id", "val"]) == sorted(await get_columns_in_db_table("tab")) await assert_core_untouched(postgresql_client, corev)
async def test_dbschema_update_legacy_2(postgresql_client: asyncpg.Connection, get_columns_in_db_table, hard_clean_db, hard_clean_db_post): await postgresql_client.execute(""" -- Table: public.schemamanager CREATE TABLE IF NOT EXISTS public.schemamanager( name varchar PRIMARY KEY, current_version integer NOT NULL ); """) await postgresql_client.execute( "INSERT INTO public.schemamanager(name, current_version) VALUES ($1, $2);", "myslice", 1) dbm = schema.DBSchema("myslice", inmanta.db.versions, postgresql_client) await dbm._legacy_migration_table() current_db_version = await dbm.get_legacy_version() assert current_db_version == 1 await dbm._legacy_migration_row({0, 1, 2, 3}) await run_updates_and_verify(get_columns_in_db_table, dbm, {0, 1})
def test_dbschema_get_dct_with_update_functions(): module_names = [ modname for _, modname, ispkg in pkgutil.iter_modules( data.PACKAGE_WITH_UPDATE_FILES.__path__) if not ispkg ] for module_name in module_names: module = __import__(data.PACKAGE_WITH_UPDATE_FILES.__name__ + "." + module_name, fromlist=["update"]) if module.DISABLED: module_names.remove(module_name) all_versions = [int(mod_name[1:]) for mod_name in module_names] db_schema = schema.DBSchema(CORE_SCHEMA_NAME, data.PACKAGE_WITH_UPDATE_FILES, None) update_function_map = db_schema._get_update_functions() assert sorted(all_versions) == [v.version for v in update_function_map] for version in update_function_map: assert version.version >= 0 assert isinstance(version.function, types.FunctionType) assert version.function.__name__ == "update" assert inspect.getfullargspec(version.function)[0] == ["connection"]
async def test_dbschema_ensure_self_update( postgresql_client: asyncpg.Connection, get_columns_in_db_table, hard_clean_db, hard_clean_db_post): await postgresql_client.execute(""" -- Table: public.schemamanager CREATE TABLE IF NOT EXISTS public.schemamanager( name varchar PRIMARY KEY, current_version integer NOT NULL ); """) await postgresql_client.execute( "INSERT INTO public.schemamanager(name, current_version) VALUES ($1, $2);", "myslice", 3) dbm = schema.DBSchema("myslice", versions, postgresql_client) await dbm.ensure_self_update() assert await dbm.get_installed_versions() == {1, 3} # make sure legacy update gets executed only once await postgresql_client.execute( "UPDATE public.schemamanager SET legacy_version=$1 WHERE name=$2", 2, "myslice") dbm.ensure_self_update() assert await dbm.get_installed_versions() == {1, 3}
async def test_dbschema_update_legacy_1(postgresql_client: asyncpg.Connection, get_columns_in_db_table, hard_clean_db, hard_clean_db_post): await postgresql_client.execute(""" -- Table: public.schemamanager CREATE TABLE IF NOT EXISTS public.schemamanager( name varchar PRIMARY KEY, current_version integer NOT NULL ); """) dbm = schema.DBSchema("test_l1", inmanta.db.versions, postgresql_client) assert set(await get_columns_in_db_table("schemamanager")) == { "name", "current_version" } await dbm._legacy_migration_table() assert set(await get_columns_in_db_table("schemamanager")) == { "name", "legacy_version", "installed_versions" } current_db_version = await dbm.get_legacy_version() assert current_db_version == 0 await dbm._legacy_migration_row({0, 1, 2}) await run_updates_and_verify(get_columns_in_db_table, dbm, set())
async def test_multi_upgrade_lockout(postgresql_pool, get_columns_in_db_table, hard_clean_db): async with postgresql_pool.acquire() as postgresql_client: async with postgresql_pool.acquire() as postgresql_client2: # schedule 3 updates, hang on second, unblock one, verify, unblock other, verify corev: Set[int] = await get_core_versions(postgresql_client) db_schema = schema.DBSchema("test_multi_upgrade_lockout", inmanta.db.versions, postgresql_client) db_schema2 = schema.DBSchema("test_multi_upgrade_lockout", inmanta.db.versions, postgresql_client2) await db_schema.ensure_self_update() lock = Semaphore(0) async def update_function_a(connection): # Fix syntax issue await connection.execute( "CREATE TABLE public.taba(id integer primary key, val varchar NOT NULL);" ) async def update_function_b(connection): # Syntax error should trigger database rollback await lock.acquire() await connection.execute( "CREATE TABLE public.tabb(id integer primary key, val varchar NOT NULL);" ) async def update_function_c(connection): # Fix syntax issue await connection.execute( "CREATE TABLE public.tabc(id integer primary key, val varchar NOT NULL);" ) current_db_versions: Set[ int] = await db_schema.get_installed_versions() assert len(current_db_versions) == 0 update_function_map = make_versions(1, update_function_a, update_function_b, update_function_c) r1 = asyncio.ensure_future( db_schema._update_db_schema(update_function_map)) r2 = asyncio.ensure_future( db_schema2._update_db_schema(update_function_map)) both = asyncio.as_completed([r1, r2]).__iter__() await asyncio.sleep(0.1) first = next(both) lock.release() await first # second one doesn't even hit the lock, as it never sees schema version 0 # lock.release() second = next(both) await second # Assert done assert (await db_schema.get_installed_versions()) == {1, 2, 3} assert (await postgresql_client.fetchval( "SELECT table_name FROM information_schema.tables " "WHERE table_schema='public' AND table_name='taba'" )) is not None assert (await postgresql_client.fetchval( "SELECT table_name FROM information_schema.tables " "WHERE table_schema='public' AND table_name='tabb'" )) is not None assert (await postgresql_client.fetchval( "SELECT table_name FROM information_schema.tables " "WHERE table_schema='public' AND table_name='tabc'" )) is not None await assert_core_untouched(postgresql_client, corev)