async def server(server_pre_start): """ :param event_loop: explicitly include event_loop to make sure event loop started before and closed after this fixture. May not be required """ # fix for fact that pytest_tornado never set IOLoop._instance, the IOLoop of the main thread # causes handler failure ibl = InmantaBootloader() try: await ibl.start() except SliceStartupException as e: port = config.Config.get("server", "bind-port") output = subprocess.check_output(["ss", "-antp"]) output = output.decode("utf-8") logger.debug(f"Port: {port}") logger.debug(f"Port usage: \n {output}") raise e yield ibl.restserver try: await asyncio.wait_for(ibl.stop(), 15) except concurrent.futures.TimeoutError: logger.exception("Timeout during stop of the server in teardown") logger.info("Server clean up done")
def start_server(options: argparse.Namespace) -> None: if options.config_file and not os.path.exists(options.config_file): LOGGER.warning("Config file %s doesn't exist", options.config_file) if options.config_dir and not os.path.isdir(options.config_dir): LOGGER.warning("Config directory %s doesn't exist", options.config_dir) ibl = InmantaBootloader() setup_signal_handlers(ibl.stop) ioloop = IOLoop.current() # handle startup exceptions def _handle_startup_done(fut: asyncio.Future) -> None: if fut.cancelled(): safe_shutdown(ioloop, ibl.stop) else: exc = fut.exception() if exc is not None: LOGGER.exception("Server setup failed", exc_info=exc) traceback.print_exception(type(exc), exc, exc.__traceback__) safe_shutdown(ioloop, ibl.stop) else: LOGGER.info("Server startup complete") ensure_future(ibl.start()).add_done_callback(_handle_startup_done) ioloop.start() LOGGER.info("Server shutdown complete") if not ibl.started: exit(EXIT_START_FAILED)
async def test_custom_feature_manager(tmp_path, inmanta_config, postgres_db, database_name, clean_reset, unused_tcp_port_factory, async_finalizer): with splice_extension_in("test_module_path"): state_dir = str(tmp_path) port = str(unused_tcp_port_factory()) config.Config.set("database", "name", database_name) config.Config.set("database", "host", "localhost") config.Config.set("database", "port", str(postgres_db.port)) config.Config.set("database", "username", postgres_db.user) config.Config.set("database", "password", postgres_db.password) config.Config.set("database", "connection_timeout", str(1)) config.Config.set("config", "state-dir", state_dir) config.Config.set("config", "log-dir", os.path.join(state_dir, "logs")) config.Config.set("agent_rest_transport", "port", port) config.Config.set("compiler_rest_transport", "port", port) config.Config.set("client_rest_transport", "port", port) config.Config.set("cmdline_rest_transport", "port", port) config.Config.set("server", "bind-port", port) config.Config.set("server", "bind-address", "127.0.0.1") config.server_enabled_extensions.set("testfm") ibl = InmantaBootloader() async_finalizer.add(partial(ibl.stop, timeout=15)) await ibl.start() server = ibl.restserver fm = server.get_slice(SLICE_SERVER).feature_manager assert not fm.enabled(None) assert not fm.enabled("a")
async def server_with_frequent_cleanups(server_pre_start, server_config, async_finalizer): config.Config.set("server", "compiler-report-retention", "60") config.Config.set("server", "cleanup-compiler-reports_interval", "1") ibl = InmantaBootloader() await ibl.start() yield ibl.restserver await ibl.stop(timeout=15)
def test_end_to_end(): with splice_extension_in("test_module_path"): ibl = InmantaBootloader() config.server_enabled_extensions.set("testplugin") ctx = ibl.load_slices() byname = {sl.name: sl for sl in ctx.get_slices()} assert "testplugin.testslice" in byname
async def migrate_v2_to_v3(hard_clean_db, hard_clean_db_post, postgresql_client: Connection, async_finalizer, server_config): # Get old tables with open(os.path.join(os.path.dirname(__file__), "dumps/v2.sql"), "r") as fh: await PGRestore(fh.readlines(), postgresql_client).run() ibl = InmantaBootloader() await ibl.start() yield await ibl.stop()
async def startable_server(server_config): """ This fixture returns the bootloader of a server which is not yet started. """ bootloader = InmantaBootloader() yield bootloader try: await bootloader.stop(timeout=15) except concurrent.futures.TimeoutError: logger.exception("Timeout during stop of the server in teardown")
def test_end_to_end_2(): with splice_extension_in("bad_module_path"): config.server_enabled_extensions.set("badplugin") ibl = InmantaBootloader() all = ibl._load_extensions() print(all) assert "badplugin" in all config.server_enabled_extensions.set("") all = ibl._load_extensions() assert "badplugin" not in all
async def test_server_logs_address(server_config, caplog): with caplog.at_level(logging.INFO): ibl = InmantaBootloader() await ibl.start() client = Client("client") result = await client.create_project("env-test") assert result.code == 200 address = "127.0.0.1" await ibl.stop() log_contains(caplog, "protocol.rest", logging.INFO, f"Server listening on {address}:")
def test_phase_1(caplog): with splice_extension_in("test_module_path"): ibl = InmantaBootloader() config.server_enabled_extensions.set("testplugin,noext") all = ibl._load_extensions() assert "testplugin" in all assert all["testplugin"] == inmanta_ext.testplugin.extension.setup log_contains(caplog, "inmanta.server.bootloader", logging.WARNING, "Could not load extension inmanta_ext.noext")
async def migrate_v4_to_v5( hard_clean_db, hard_clean_db_post, postgresql_client: Connection, async_finalizer, server_config ) -> AsyncIterator[None]: # Get old tables with open(os.path.join(os.path.dirname(__file__), "dumps/v4.sql"), "r") as fh: await PGRestore(fh.readlines(), postgresql_client).run() ibl = InmantaBootloader() await ibl.start() # When the bootloader is started, it also executes the migration to v5 yield await ibl.stop(timeout=15)
async def migrate_v202111260_to_v202203140( hard_clean_db, hard_clean_db_post, postgresql_client: Connection, server_config) -> AsyncIterator[Callable[[], Awaitable[None]]]: """ Returns a callable that performs a v202111260 database restore and migrates to v202203140. """ # Get old tables with open(os.path.join(os.path.dirname(__file__), "dumps/v202111260.sql"), "r") as fh: await PGRestore(fh.readlines(), postgresql_client).run() ibl = InmantaBootloader() # When the bootloader is started, it also executes the migration to v202203140 yield ibl.start await ibl.stop(timeout=15)
async def test_startup_failure(async_finalizer, server_config): with splice_extension_in("bad_module_path"): config.server_enabled_extensions.set("badplugin") ibl = InmantaBootloader() async_finalizer.add(partial(ibl.stop, timeout=15)) with pytest.raises(Exception) as e: await ibl.start() print(e.value) assert str( e.value ) == "Slice badplugin.badslice failed to start because: Too bad, this plugin is broken" config.server_enabled_extensions.set("") all = ibl._load_extensions() assert "badplugin" not in all
def test_phase_2(): with splice_extension_in("test_module_path"): import inmanta_ext.testplugin.extension ibl = InmantaBootloader() all = {"testplugin": inmanta_ext.testplugin.extension.setup} ctx = ibl._collect_slices(all) byname = {sl.name: sl for sl in ctx._slices} assert "testplugin.testslice" in byname # load slice in wrong namespace with pytest.raises(InvalidSliceNameException): all = {"test": inmanta_ext.testplugin.extension.setup} ctx = ibl._collect_slices(all)
def test_load_and_filter(caplog): caplog.set_level(logging.INFO) with splice_extension_in("test_module_path"): ibl = InmantaBootloader() plugin_pkgs = ibl._discover_plugin_packages() assert "inmanta_ext.core" in plugin_pkgs assert len(plugin_pkgs) == 1 # When extensions are available but not enabled, log a message with the correct option log_contains(caplog, "inmanta.server.bootloader", logging.INFO, "Load extensions by setting configuration option") with pytest.raises(PluginLoadFailed): config.server_enabled_extensions.set("unknown") plugin_pkgs = ibl._discover_plugin_packages()
def print_versions_installed_components_and_exit() -> None: bootloader = InmantaBootloader() app_context = bootloader.load_slices() product_metadata = app_context.get_product_metadata() extension_statuses = app_context.get_extension_statuses() if product_metadata.version: print(f"{product_metadata.product} ({product_metadata.edition}): {product_metadata.version}") else: print(f"{product_metadata.product} ({product_metadata.edition}): version unknown") print(f"Compiler version: {get_compiler_version()}") if extension_statuses: print("Extensions:") for ext_status in extension_statuses: print(f" * {ext_status.name}: {ext_status.version}") else: print("Extensions: No extensions found") sys.exit(0)
async def test_server_status_database_down(server_config, server_pre_start, postgres_db, ensure_running_postgres_db_post, async_finalizer): ibl = InmantaBootloader() await ibl.start() async_finalizer.add(ibl.stop) postgres_db.stop() client = protocol.Client("client") result = await client.get_server_status() assert result.code == 200 database_slice = None for slice in result.result["data"]["slices"]: if slice["name"] == "core.database": database_slice = slice assert database_slice assert not database_slice["status"]["connected"]
async def migrate_v7_to_v17(hard_clean_db, hard_clean_db_post, postgresql_client: Connection, server_config) -> AsyncIterator[None]: """ Performs a v7 database restore and migrates to v17. """ # Get old tables with open(os.path.join(os.path.dirname(__file__), "dumps/v7.sql"), "r") as fh: await PGRestore(fh.readlines(), postgresql_client).run() ibl = InmantaBootloader() await ibl.start() # When the bootloader is started, it also executes the migration to v18 yield await ibl.stop()
def test_discover_and_load(): with splice_extension_in("test_module_path"): config.server_enabled_extensions.set("testplugin") ibl = InmantaBootloader() print("plugins: ", ibl._discover_plugin_packages()) assert "inmanta_ext.testplugin" in ibl._discover_plugin_packages() tpl = ibl._load_extension("inmanta_ext.testplugin") assert tpl == inmanta_ext.testplugin.extension.setup with pytest.raises(PluginLoadFailed): ibl._load_extension("inmanta_ext.noext") with pytest.raises(PluginLoadFailed): ibl._load_extension("inmanta_ext.noinit")
async def migrate_v3_to_v4(hard_clean_db, hard_clean_db_post, postgresql_client: Connection, async_finalizer, server_config): # Get old tables with open(os.path.join(os.path.dirname(__file__), "dumps/v3.sql"), "r") as fh: await PGRestore(fh.readlines(), postgresql_client).run() for table_name in ["form", "formrecord", "resourceversionid"]: assert await does_table_exist(postgresql_client, table_name) result = await postgresql_client.fetch("SELECT action_id, resource_version_id FROM public.resourceversionid") resource_version_id_dict = defaultdict(list) for r in result: resource_version_id_dict[r["action_id"]].append(r["resource_version_id"]) ibl = InmantaBootloader() # When the bootloader is started, it also executes the migration to v4 await ibl.start() yield resource_version_id_dict await ibl.stop()
async def server_multi(server_pre_start, event_loop, inmanta_config, postgres_db, database_name, request, clean_reset, unused_tcp_port_factory): """ :param event_loop: explicitly include event_loop to make sure event loop started before and closed after this fixture. May not be required """ state_dir = tempfile.mkdtemp() ssl, auth, ca = request.param path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") if auth: config.Config.set("server", "auth", "true") for x, ct in [ ("server", None), ("agent_rest_transport", ["agent"]), ("compiler_rest_transport", ["compiler"]), ("client_rest_transport", ["api", "compiler"]), ("cmdline_rest_transport", ["api"]), ]: if ssl and not ca: config.Config.set(x, "ssl_cert_file", os.path.join(path, "server.crt")) config.Config.set(x, "ssl_key_file", os.path.join(path, "server.open.key")) config.Config.set(x, "ssl_ca_cert_file", os.path.join(path, "server.crt")) config.Config.set(x, "ssl", "True") if ssl and ca: capath = os.path.join(path, "ca", "enduser-certs") config.Config.set(x, "ssl_cert_file", os.path.join(capath, "server.crt")) config.Config.set(x, "ssl_key_file", os.path.join(capath, "server.key.open")) config.Config.set(x, "ssl_ca_cert_file", os.path.join(capath, "server.chain")) config.Config.set(x, "ssl", "True") if auth and ct is not None: token = protocol.encode_token(ct) config.Config.set(x, "token", token) port = str(unused_tcp_port_factory()) config.Config.set("database", "name", database_name) config.Config.set("database", "host", "localhost") config.Config.set("database", "port", str(postgres_db.port)) config.Config.set("database", "connection_timeout", str(3)) config.Config.set("config", "state-dir", state_dir) config.Config.set("config", "log-dir", os.path.join(state_dir, "logs")) config.Config.set("agent_rest_transport", "port", port) config.Config.set("compiler_rest_transport", "port", port) config.Config.set("client_rest_transport", "port", port) config.Config.set("cmdline_rest_transport", "port", port) config.Config.set("server", "bind-port", port) config.Config.set("server", "bind-address", "127.0.0.1") config.Config.set("config", "executable", os.path.abspath(inmanta.app.__file__)) config.Config.set("server", "agent-timeout", "2") config.Config.set("agent", "agent-repair-interval", "0") config.Config.set("server", "auto-recompile-wait", "0") ibl = InmantaBootloader() try: await ibl.start() except SliceStartupException as e: port = config.Config.get("server", "bind-port") output = subprocess.check_output(["ss", "-antp"]) output = output.decode("utf-8") logger.debug(f"Port: {port}") logger.debug(f"Port usage: \n {output}") raise e yield ibl.restserver try: await asyncio.wait_for(ibl.stop(), 15) except concurrent.futures.TimeoutError: logger.exception("Timeout during stop of the server in teardown") shutil.rmtree(state_dir)
import pytest from inmanta import config from inmanta.server.bootloader import InmantaBootloader logger = logging.getLogger(__name__) @pytest.fixture def {{ cookiecutter.extension_name }}_config(server_config, postgres_db, database_name): config.Config.set("server", "enabled_extensions", "{{ cookiecutter.extension_name }}") @pytest.fixture async def server({{ cookiecutter.extension_name }}_config, server_config): """ Override standard inmanta server to allow more config to be injected """ ibl = InmantaBootloader() await ibl.start() yield ibl.restserver try: await asyncio.wait_for(ibl.stop(), 15) except concurrent.futures.TimeoutError: logger.exception("Timeout during stop of the server in teardown") logger.info("Server clean up done")
def get_product_meta_data() -> ProductMetadata: """Get the produce meta-data""" bootloader = InmantaBootloader() context = bootloader.load_slices() return context.get_product_metadata()