def start_server(options: argparse.Namespace) -> None:
if options.config_file and not os.path.exists(options.config_file):
LOGGER.warning("Config file %s doesn't exist", options.config_file)
if options.config_dir and not os.path.isdir(options.config_dir):
LOGGER.warning("Config directory %s doesn't exist", options.config_dir)
ibl = InmantaBootloader()
setup_signal_handlers(ibl.stop)
ioloop = IOLoop.current()
# handle startup exceptions
def _handle_startup_done(fut: asyncio.Future) -> None:
if fut.cancelled():
safe_shutdown(ioloop, ibl.stop)
else:
exc = fut.exception()
if exc is not None:
LOGGER.exception("Server setup failed", exc_info=exc)
traceback.print_exception(type(exc), exc, exc.__traceback__)
safe_shutdown(ioloop, ibl.stop)
else:
LOGGER.info("Server startup complete")
ensure_future(ibl.start()).add_done_callback(_handle_startup_done)
ioloop.start()
LOGGER.info("Server shutdown complete")
if not ibl.started:
exit(EXIT_START_FAILED)
async def server(server_pre_start):
"""
:param event_loop: explicitly include event_loop to make sure event loop started before and closed after this fixture.
May not be required
"""
# fix for fact that pytest_tornado never set IOLoop._instance, the IOLoop of the main thread
# causes handler failure
ibl = InmantaBootloader()
try:
await ibl.start()
except SliceStartupException as e:
port = config.Config.get("server", "bind-port")
output = subprocess.check_output(["ss", "-antp"])
output = output.decode("utf-8")
logger.debug(f"Port: {port}")
logger.debug(f"Port usage: \n {output}")
raise e
yield ibl.restserver
try:
await asyncio.wait_for(ibl.stop(), 15)
except concurrent.futures.TimeoutError:
logger.exception("Timeout during stop of the server in teardown")
logger.info("Server clean up done")
async def server_with_frequent_cleanups(server_pre_start, server_config,
async_finalizer):
config.Config.set("server", "compiler-report-retention", "2")
config.Config.set("server", "cleanup-compiler-reports_interval", "1")
ibl = InmantaBootloader()
await ibl.start()
yield ibl.restserver
await asyncio.wait_for(ibl.stop(), 15)
def test_end_to_end():
with splice_extension_in("test_module_path"):
ibl = InmantaBootloader()
config.server_enabled_extensions.set("testplugin")
ctx = ibl.load_slices()
byname = {sl.name: sl for sl in ctx.get_slices()}
assert "testplugin.testslice" in byname
async def startable_server(server_config):
"""
This fixture returns the bootloader of a server which is not yet started.
"""
bootloader = InmantaBootloader()
yield bootloader
try:
await asyncio.wait_for(bootloader.stop(), 15)
except concurrent.futures.TimeoutError:
logger.exception("Timeout during stop of the server in teardown")
def test_end_to_end_2():
with splice_extension_in("bad_module_path"):
config.server_enabled_extensions.set("badplugin")
ibl = InmantaBootloader()
all = ibl._load_extensions()
print(all)
assert "badplugin" in all
config.server_enabled_extensions.set("")
all = ibl._load_extensions()
assert "badplugin" not in all
def test_phase_1(caplog):
with splice_extension_in("test_module_path"):
ibl = InmantaBootloader()
config.server_enabled_extensions.set("testplugin,noext")
all = ibl._load_extensions()
assert "testplugin" in all
assert all["testplugin"] == inmanta_ext.testplugin.extension.setup
log_contains(caplog, "inmanta.server.bootloader", logging.WARNING,
"Could not load extension inmanta_ext.noext")
async def test_custom_feature_manager(tmp_path, inmanta_config, postgres_db,
database_name, clean_reset,
unused_tcp_port_factory,
async_finalizer):
with splice_extension_in("test_module_path"):
state_dir = str(tmp_path)
port = str(unused_tcp_port_factory())
config.Config.set("database", "name", database_name)
config.Config.set("database", "host", "localhost")
config.Config.set("database", "port", str(postgres_db.port))
config.Config.set("database", "username", postgres_db.user)
config.Config.set("database", "password", postgres_db.password)
config.Config.set("database", "connection_timeout", str(1))
config.Config.set("config", "state-dir", state_dir)
config.Config.set("config", "log-dir", os.path.join(state_dir, "logs"))
config.Config.set("agent_rest_transport", "port", port)
config.Config.set("compiler_rest_transport", "port", port)
config.Config.set("client_rest_transport", "port", port)
config.Config.set("cmdline_rest_transport", "port", port)
config.Config.set("server", "bind-port", port)
config.Config.set("server", "bind-address", "127.0.0.1")
config.server_enabled_extensions.set("testfm")
ibl = InmantaBootloader()
async_finalizer.add(partial(ibl.stop, timeout=15))
await ibl.start()
server = ibl.restserver
fm = server.get_slice(SLICE_SERVER).feature_manager
assert not fm.enabled(None)
assert not fm.enabled("a")
def print_versions_installed_components_and_exit() -> None:
bootloader = InmantaBootloader()
app_context = bootloader.load_slices()
product_metadata = app_context.get_product_metadata()
extension_statuses = app_context.get_extension_statuses()
if product_metadata.version:
print(f"{product_metadata.product} ({product_metadata.edition}): {product_metadata.version}")
else:
print(f"{product_metadata.product} ({product_metadata.edition}): version unknown")
print(f"Compiler version: {get_compiler_version()}")
if extension_statuses:
print("Extensions:")
for ext_status in extension_statuses:
print(f" * {ext_status.name}: {ext_status.version}")
else:
print("Extensions: No extensions found")
sys.exit(0)
def test_phase_2():
with splice_extension_in("test_module_path"):
import inmanta_ext.testplugin.extension
ibl = InmantaBootloader()
all = {"testplugin": inmanta_ext.testplugin.extension.setup}
ctx = ibl._collect_slices(all)
byname = {sl.name: sl for sl in ctx._slices}
assert "testplugin.testslice" in byname
# load slice in wrong namespace
with pytest.raises(InvalidSliceNameException):
all = {"test": inmanta_ext.testplugin.extension.setup}
ctx = ibl._collect_slices(all)
def test_load_and_filter(caplog):
caplog.set_level(logging.INFO)
with splice_extension_in("test_module_path"):
ibl = InmantaBootloader()
plugin_pkgs = ibl._discover_plugin_packages()
assert "inmanta_ext.core" in plugin_pkgs
assert len(plugin_pkgs) == 1
# When extensions are available but not enabled, log a message with the correct option
log_contains(caplog, "inmanta.server.bootloader", logging.INFO,
"Load extensions by setting configuration option")
with pytest.raises(PluginLoadFailed):
config.server_enabled_extensions.set("unknown")
plugin_pkgs = ibl._discover_plugin_packages()
async def test_startup_failure(async_finalizer, server_config):
with splice_extension_in("bad_module_path"):
config.server_enabled_extensions.set("badplugin")
ibl = InmantaBootloader()
async_finalizer.add(partial(ibl.stop, timeout=15))
with pytest.raises(Exception) as e:
await ibl.start()
print(e.value)
assert str(
e.value
) == "Slice badplugin.badslice failed to start because: Too bad, this plugin is broken"
config.server_enabled_extensions.set("")
all = ibl._load_extensions()
assert "badplugin" not in all
async def migrate_v2_to_v3(hard_clean_db, hard_clean_db_post, postgresql_client: Connection, async_finalizer, server_config):
# Get old tables
with open(os.path.join(os.path.dirname(__file__), "dumps/v2.sql"), "r") as fh:
await PGRestore(fh.readlines(), postgresql_client).run()
ibl = InmantaBootloader()
await ibl.start()
yield
await ibl.stop()
async def migrate_v4_to_v5(
hard_clean_db, hard_clean_db_post, postgresql_client: Connection, async_finalizer, server_config
) -> AsyncIterator[None]:
# Get old tables
with open(os.path.join(os.path.dirname(__file__), "dumps/v4.sql"), "r") as fh:
await PGRestore(fh.readlines(), postgresql_client).run()
ibl = InmantaBootloader()
await ibl.start()
# When the bootloader is started, it also executes the migration to v5
yield
await ibl.stop(timeout=15)
async def test_server_logs_address(server_config, caplog):
with caplog.at_level(logging.INFO):
ibl = InmantaBootloader()
await ibl.start()
client = Client("client")
result = await client.create_project("env-test")
assert result.code == 200
address = "127.0.0.1"
await ibl.stop()
log_contains(caplog, "protocol.rest", logging.INFO,
f"Server listening on {address}:")
def test_discover_and_load():
with splice_extension_in("test_module_path"):
config.server_enabled_extensions.set("testplugin")
ibl = InmantaBootloader()
print("plugins: ", ibl._discover_plugin_packages())
assert "inmanta_ext.testplugin" in ibl._discover_plugin_packages()
tpl = ibl._load_extension("inmanta_ext.testplugin")
assert tpl == inmanta_ext.testplugin.extension.setup
with pytest.raises(PluginLoadFailed):
ibl._load_extension("inmanta_ext.noext")
with pytest.raises(PluginLoadFailed):
ibl._load_extension("inmanta_ext.noinit")
async def migrate_v202111260_to_v202203140(
hard_clean_db, hard_clean_db_post, postgresql_client: Connection,
server_config) -> AsyncIterator[Callable[[], Awaitable[None]]]:
"""
Returns a callable that performs a v202111260 database restore and migrates to v202203140.
"""
# Get old tables
with open(os.path.join(os.path.dirname(__file__), "dumps/v202111260.sql"),
"r") as fh:
await PGRestore(fh.readlines(), postgresql_client).run()
ibl = InmantaBootloader()
# When the bootloader is started, it also executes the migration to v202203140
yield ibl.start
await ibl.stop(timeout=15)
async def test_server_status_database_down(server_config, server_pre_start,
postgres_db,
ensure_running_postgres_db_post,
async_finalizer):
ibl = InmantaBootloader()
await ibl.start()
async_finalizer.add(ibl.stop)
postgres_db.stop()
client = protocol.Client("client")
result = await client.get_server_status()
assert result.code == 200
database_slice = None
for slice in result.result["data"]["slices"]:
if slice["name"] == "core.database":
database_slice = slice
assert database_slice
assert not database_slice["status"]["connected"]
async def migrate_v7_to_v17(hard_clean_db, hard_clean_db_post,
postgresql_client: Connection,
server_config) -> AsyncIterator[None]:
"""
Performs a v7 database restore and migrates to v17.
"""
# Get old tables
with open(os.path.join(os.path.dirname(__file__), "dumps/v7.sql"),
"r") as fh:
await PGRestore(fh.readlines(), postgresql_client).run()
ibl = InmantaBootloader()
await ibl.start()
# When the bootloader is started, it also executes the migration to v18
yield
await ibl.stop()
async def migrate_v3_to_v4(hard_clean_db, hard_clean_db_post, postgresql_client: Connection, async_finalizer, server_config):
# Get old tables
with open(os.path.join(os.path.dirname(__file__), "dumps/v3.sql"), "r") as fh:
await PGRestore(fh.readlines(), postgresql_client).run()
for table_name in ["form", "formrecord", "resourceversionid"]:
assert await does_table_exist(postgresql_client, table_name)
result = await postgresql_client.fetch("SELECT action_id, resource_version_id FROM public.resourceversionid")
resource_version_id_dict = defaultdict(list)
for r in result:
resource_version_id_dict[r["action_id"]].append(r["resource_version_id"])
ibl = InmantaBootloader()
# When the bootloader is started, it also executes the migration to v4
await ibl.start()
yield resource_version_id_dict
await ibl.stop()
async def server_multi(server_pre_start, event_loop, inmanta_config,
postgres_db, database_name, request, clean_reset,
unused_tcp_port_factory):
"""
:param event_loop: explicitly include event_loop to make sure event loop started before and closed after this fixture.
May not be required
"""
state_dir = tempfile.mkdtemp()
ssl, auth, ca = request.param
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
if auth:
config.Config.set("server", "auth", "true")
for x, ct in [
("server", None),
("agent_rest_transport", ["agent"]),
("compiler_rest_transport", ["compiler"]),
("client_rest_transport", ["api", "compiler"]),
("cmdline_rest_transport", ["api"]),
]:
if ssl and not ca:
config.Config.set(x, "ssl_cert_file",
os.path.join(path, "server.crt"))
config.Config.set(x, "ssl_key_file",
os.path.join(path, "server.open.key"))
config.Config.set(x, "ssl_ca_cert_file",
os.path.join(path, "server.crt"))
config.Config.set(x, "ssl", "True")
if ssl and ca:
capath = os.path.join(path, "ca", "enduser-certs")
config.Config.set(x, "ssl_cert_file",
os.path.join(capath, "server.crt"))
config.Config.set(x, "ssl_key_file",
os.path.join(capath, "server.key.open"))
config.Config.set(x, "ssl_ca_cert_file",
os.path.join(capath, "server.chain"))
config.Config.set(x, "ssl", "True")
if auth and ct is not None:
token = protocol.encode_token(ct)
config.Config.set(x, "token", token)
port = str(unused_tcp_port_factory())
config.Config.set("database", "name", database_name)
config.Config.set("database", "host", "localhost")
config.Config.set("database", "port", str(postgres_db.port))
config.Config.set("database", "connection_timeout", str(3))
config.Config.set("config", "state-dir", state_dir)
config.Config.set("config", "log-dir", os.path.join(state_dir, "logs"))
config.Config.set("agent_rest_transport", "port", port)
config.Config.set("compiler_rest_transport", "port", port)
config.Config.set("client_rest_transport", "port", port)
config.Config.set("cmdline_rest_transport", "port", port)
config.Config.set("server", "bind-port", port)
config.Config.set("server", "bind-address", "127.0.0.1")
config.Config.set("config", "executable",
os.path.abspath(inmanta.app.__file__))
config.Config.set("server", "agent-timeout", "2")
config.Config.set("agent", "agent-repair-interval", "0")
config.Config.set("server", "auto-recompile-wait", "0")
ibl = InmantaBootloader()
try:
await ibl.start()
except SliceStartupException as e:
port = config.Config.get("server", "bind-port")
output = subprocess.check_output(["ss", "-antp"])
output = output.decode("utf-8")
logger.debug(f"Port: {port}")
logger.debug(f"Port usage: \n {output}")
raise e
yield ibl.restserver
try:
await asyncio.wait_for(ibl.stop(), 15)
except concurrent.futures.TimeoutError:
logger.exception("Timeout during stop of the server in teardown")
shutil.rmtree(state_dir)
def get_product_meta_data() -> ProductMetadata:
"""Get the produce meta-data"""
bootloader = InmantaBootloader()
context = bootloader.load_slices()
return context.get_product_metadata()
import pytest
from inmanta import config
from inmanta.server.bootloader import InmantaBootloader
logger = logging.getLogger(__name__)
@pytest.fixture
def {{ cookiecutter.extension_name }}_config(server_config, postgres_db, database_name):
config.Config.set("server", "enabled_extensions", "{{ cookiecutter.extension_name }}")
@pytest.fixture
async def server({{ cookiecutter.extension_name }}_config, server_config):
"""
Override standard inmanta server to allow more config to be injected
"""
ibl = InmantaBootloader()
await ibl.start()
yield ibl.restserver
try:
await asyncio.wait_for(ibl.stop(), 15)
except concurrent.futures.TimeoutError:
logger.exception("Timeout during stop of the server in teardown")
logger.info("Server clean up done")