async def test_bind_address_ipv6(async_finalizer) -> None:
@protocol.method(path="/test", operation="POST", client_types=[ClientType.api])
async def test_endpoint():
pass
class TestSlice(ServerSlice):
@protocol.handle(test_endpoint)
async def test_endpoint_handle(self):
return 200
# Get free port on all interfaces
sock = netutil.bind_sockets(0, "::", family=socket.AF_INET6)[0]
(_addr, free_port, _flowinfo, _scopeid) = sock.getsockname()
sock.close()
# Configure server
Config.load_config()
Config.set("server", "bind-port", str(free_port))
Config.set("server", "bind-address", "::1")
Config.set("client_rest_transport", "port", str(free_port))
Config.set("client_rest_transport", "host", "::1")
# Start server
rs = Server()
rs.add_slice(TestSlice("test"))
await rs.start()
async_finalizer(rs.stop)
client = protocol.Client("client")
# Check if server is reachable on loopback interface
result = await client.test_endpoint()
assert result.code == 200
async def test_agent_cannot_retrieve_autostart_agent_map(async_started_agent, startable_server, caplog):
"""
When an agent with the config option use_autostart_agent_map set to true, cannot retrieve the autostart_agent_map
from the server at startup, the process should retry. This test verifies that the retry happens correctly.
"""
client = protocol.Client("client")
def retry_occured() -> bool:
return caplog.text.count("Failed to retrieve the autostart_agent_map setting from the server.") > 2
# Agent cannot contact server, since server is not started yet.
await retry_limited(retry_occured, 10)
# Start server
await startable_server.start()
# Create project
result = await client.create_project("env-test")
assert result.code == 200
project_id = result.result["project"]["id"]
# Create environment
result = await client.create_environment(project_id=project_id, name="dev", environment_id=async_started_agent.environment)
assert result.code == 200
# Assert agent managed to establish session with the server
agent_manager = startable_server.restserver.get_slice(SLICE_AGENT_MANAGER)
await retry_limited(lambda: (async_started_agent.environment, "agent1") in agent_manager.tid_endpoint_to_session, 10)
async def test_2way_protocol(unused_tcp_port, no_tid_check, postgres_db,
database_name):
configure(unused_tcp_port, database_name, postgres_db.port)
rs = Server()
server = SessionSpy()
rs.get_slice(SLICE_SESSION_MANAGER).add_listener(server)
rs.add_slice(server)
await rs.start()
agent = Agent("agent")
await agent.add_end_point_name("agent")
agent.set_environment(uuid.uuid4())
await agent.start()
await retry_limited(lambda: len(server.get_sessions()) == 1, 10)
assert len(server.get_sessions()) == 1
await assert_agent_counter(agent, 1, 0)
client = protocol.Client("client")
status = await client.get_status_x(str(agent.environment))
assert status.code == 200
assert "agents" in status.result
assert len(status.result["agents"]) == 1
assert status.result["agents"][0]["status"], "ok"
await server.stop()
await rs.stop()
await agent.stop()
await assert_agent_counter(agent, 1, 0)
def deploy_code(self, tid, version=None):
"""
Deploy code to the server
"""
if version is None:
version = int(time.time())
def merge_dict(a, b):
"""Very specific impl to this particular data structure."""
for k, v in b.items():
if k not in a:
a[k] = v
elif isinstance(v, dict):
merge_dict(a[k], v)
else:
if a[k] != v:
raise Exception("Hash collision!", k, a[k], v)
LOGGER.info("Sending resources and handler source to server")
sources = resource.sources()
merge_dict(sources, Commander.sources())
LOGGER.info("Uploading source files")
conn = protocol.Client("compiler")
@gen.coroutine
def call():
for myresource, mysources in sources.items():
res = yield conn.upload_code(tid=tid, id=version, resource=myresource, sources=mysources)
if res is None or res.code != 200:
raise Exception("Unable to upload handler plugin code to the server (msg: %s)" % res.result)
self.run_sync(call)
async def test_environment_update(migrate_v2_to_v3, async_finalizer, server_config):
client = protocol.Client("client")
result = await client.list_environments()
names = sorted([env["name"] for env in result.result["environments"]])
name_to_id = {env["name"]: env["id"] for env in result.result["environments"]}
assert names == ["dev-1", "dev-2"]
env1 = await data.Environment.get_by_id(name_to_id["dev-1"])
assert env1.last_version == 1569583847
result = await client.create_environment(project_id=env1.project, name="dev-3")
assert result.code == 200
env3 = await data.Environment.get_by_id(result.result["environment"]["id"])
assert env3.last_version == 0
e1_next = await env1.get_next_version()
assert e1_next == 1569583848
e1_next = await env1.get_next_version()
assert e1_next == 1569583849
e3_next = await env3.get_next_version()
assert e3_next == 1
e3_next = await env3.get_next_version()
assert e3_next == 2
async def test_added_environment_columns(
migrate_v202109100_to_v202111260: Callable[[], Awaitable[None]],
get_columns_in_db_table: Callable[[str], Awaitable[List[str]]],
) -> None:
"""
Test whether the description and icon columns were added to the environment table.
"""
# The columns are not present before the migration
columns = await get_columns_in_db_table("environment")
assert "icon" not in columns
assert "description" not in columns
# Migrate DB schema
await migrate_v202109100_to_v202111260()
client = protocol.Client("client")
# The columns are added to the table
columns += ["description", "icon"]
assert (await get_columns_in_db_table("environment")) == columns
# The environment data is still ok after the migration, and has the correct default values
result = await client.environment_list()
assert result.code == 200
assert len(result.result["data"]) == 2
env_id = "982a35ab-2785-4221-9926-f4f389416ce3"
result = await client.environment_get(env_id)
assert result.code == 200
assert result.result["data"]["icon"] == ""
assert result.result["data"]["description"] == ""
def do_call():
client = protocol.Client("client")
status = yield client.get_status_x(str(agent.environment))
assert status.code == 200
assert "agents" in status.result
assert len(status.result["agents"]) == 1
assert status.result["agents"][0]["status"], "ok"
server.stop()
io_loop.stop()
async def assert_port_bound():
# Start server
rs = Server()
rs.add_slice(TestSlice("test"))
await rs.start()
async_finalizer(rs.stop)
# Check if server is reachable on loopback interface
client = protocol.Client("client")
result = await client.test_endpoint()
assert result.code == 200
await rs.stop()
async def test_bind_address_ipv4(async_finalizer):
"""This test case check if the Inmanta server doesn't bind on another interface than 127.0.0.1 when bind-address is equal
to 127.0.0.1. Procedure:
1) Get free port on all interfaces.
2) Bind that port on a non-loopback interface, so it's not available for the inmanta server anymore.
3) Start the Inmanta server with bind-address 127.0.0.1. and execute an API call
"""
@protocol.method(path="/test", operation="POST", client_types=[ClientType.api])
async def test_endpoint():
pass
class TestSlice(ServerSlice):
@protocol.handle(test_endpoint)
async def test_endpoint_handle(self):
return 200
# Select a bind address which is not on the loopback interface
non_loopback_interfaces = [i for i in netifaces.interfaces() if i != "lo" and socket.AF_INET in netifaces.ifaddresses(i)]
bind_iface = "eth0" if "eth0" in non_loopback_interfaces else random.choice(non_loopback_interfaces)
bind_addr = netifaces.ifaddresses(bind_iface)[socket.AF_INET][0]["addr"]
# Get free port on all interfaces
sock = netutil.bind_sockets(0, "0.0.0.0", family=socket.AF_INET)[0]
_addr, free_port = sock.getsockname()
sock.close()
# Bind port on non-loopback interface
sock = netutil.bind_sockets(free_port, bind_addr, family=socket.AF_INET)[0]
try:
# Configure server
Config.load_config()
Config.set("server", "bind-port", str(free_port))
Config.set("server", "bind-address", "127.0.0.1")
Config.set("client_rest_transport", "port", str(free_port))
# Start server
rs = Server()
rs.add_slice(TestSlice("test"))
await rs.start()
async_finalizer(rs.stop)
# Check if server is reachable on loopback interface
client = protocol.Client("client")
result = await client.test_endpoint()
assert result.code == 200
finally:
sock.close()
async def test_server_status_database_down(server_config, server_pre_start,
postgres_db,
ensure_running_postgres_db_post,
async_finalizer):
ibl = InmantaBootloader()
await ibl.start()
async_finalizer.add(ibl.stop)
postgres_db.stop()
client = protocol.Client("client")
result = await client.get_server_status()
assert result.code == 200
database_slice = None
for slice in result.result["data"]["slices"]:
if slice["name"] == "core.database":
database_slice = slice
assert database_slice
assert not database_slice["status"]["connected"]
def export(options):
if options.environment is not None:
Config.set("config", "environment", options.environment)
if options.server is not None:
Config.set("compiler_rest_transport", "host", options.server)
if options.server is not None:
Config.set("compiler_rest_transport", "port", options.port)
if options.token is not None:
Config.set("compiler_rest_transport", "token", options.token)
if options.ssl:
Config.set("compiler_rest_transport", "ssl", "true")
if options.ca_cert is not None:
Config.set("compiler_rest_transport", "ssl-ca-cert-file", options.ca_cert)
module.Project.get(options.main_file)
from inmanta.export import Exporter # noqa: H307
exp = None
try:
(types, scopes) = do_compile()
except Exception as e:
exp = e
types, scopes = (None, None)
export = Exporter(options)
version, _ = export.run(types, scopes)
if exp is not None:
if not options.errors:
print(exp, file=sys.stderr)
sys.exit(1)
else:
raise exp
if options.deploy:
conn = protocol.Client("compiler")
LOGGER.info("Triggering deploy for version %d" % version)
tid = cfg_env.get()
IOLoop.current().run_sync(lambda: conn.release_version(tid, version, True), 60)
def __init__(self, host, port, io_loop):
self._client = None
if io_loop is not None:
self._io_loop = io_loop
self._own_loop = False
else:
self._io_loop = IOLoop.current()
self._own_loop = True
if host is None:
self.host = cmdline_rest_transport.host.get()
else:
self.host = host
Config.set("cmdline_rest_transport", "host", host)
if port is None:
self.port = cmdline_rest_transport.port.get()
else:
self.port = port
Config.set("cmdline_rest_transport", "port", str(port))
self._client = protocol.Client("cmdline")
def test_compile_report(server):
from inmanta import protocol
client = protocol.Client("client")
result = yield client.create_project("env-test")
assert (result.code == 200)
project_id = result.result["project"]["id"]
result = yield client.create_environment(project_id=project_id, name="dev")
env_id = result.result["environment"]["id"]
result = yield client.notify_change(id=env_id)
assert (result.code == 200)
while True:
result = yield client.get_reports(environment=env_id)
assert (result.code == 200)
if len(result.result["reports"]) > 0:
break
yield gen.sleep(0.5)
report = result.result["reports"][0]
assert (len(report["reports"]) == 1)
def setup_project(self):
"""
Set up the configured project and environment on the embedded server
"""
self._client = protocol.Client("client")
# get config
project_name = cfg_prj.get()
if project_name is None:
LOGGER.error(
"The name of the project should be configured for an all-in-one deploy"
)
return False
environment_name = cfg_env.get()
if environment_name is None:
LOGGER.error(
"The name of the environment in the project should be configured for an all-in-one deploy"
)
return False
# wait and check to see if the server is up
tries = 0
while tries < MAX_TRIES:
try:
yield self._client.list_projects()
break
except Exception:
tries += 1
# get project id
projects = yield self._client.list_projects()
if projects.code != 200:
LOGGER.error("Unable to retrieve project listing from the server")
return False
project_id = None
for project in projects.result["projects"]:
if project_name == project["name"]:
project_id = project["id"]
break
if project_id is None:
project_id = yield self._create_project(project_name)
if not project_id:
return False
# get or create the environment
environments = yield self._client.list_environments()
if environments.code != 200:
LOGGER.error("Unable to retrieve environments from server")
return False
for env in environments.result["environments"]:
if project_id == env["project"] and environment_name == env["name"]:
self._environment_id = env["id"]
break
if self._environment_id is None:
self._environment_id = yield self._create_environment(
project_id, environment_name)
if not self._environment_id:
return False
return True
def client_multi(server_multi):
client = protocol.Client("client")
yield client
def client(server):
client = protocol.Client("client")
yield client
def client_v2(server):
client = protocol.Client("client",
version_match=VersionMatch.exact,
exact_version=2)
yield client
def commit_resources(self, version, resources):
"""
Commit the entire list of resource to the configurations server.
"""
tid = cfg_env.get()
if tid is None:
LOGGER.error("The environment for this model should be set!")
return
self.deploy_code(tid, version)
conn = protocol.Client("compiler")
LOGGER.info("Uploading %d files" % len(self._file_store))
# collect all hashes and send them at once to the server to check
# if they are already uploaded
hashes = list(self._file_store.keys())
def call():
return conn.stat_files(files=hashes)
res = self.run_sync(call)
if res.code != 200:
raise Exception("Unable to check status of files at server")
to_upload = res.result["files"]
LOGGER.info("Only %d files are new and need to be uploaded" %
len(to_upload))
for hash_id in to_upload:
content = self._file_store[hash_id]
def call():
return conn.upload_file(
id=hash_id,
content=base64.b64encode(content).decode("ascii"))
res = self.run_sync(call)
if res.code != 200:
LOGGER.error("Unable to upload file with hash %s" % hash_id)
else:
LOGGER.debug("Uploaded file with hash %s" % hash_id)
# Collecting version information
version_info = {}
"""
"modules": ModuleTool().freeze(create_file=False),
"project": {"repo": project.get_scm_url(),
"branch": project.get_scm_branch(),
"hash": project.get_scm_version()
}
}
"""
# TODO: start transaction
LOGGER.info("Sending resource updates to server")
for res in resources:
LOGGER.debug(" %s", res["id"])
def put_call():
return conn.put_version(tid=tid,
version=version,
resources=resources,
unknowns=unknown_parameters,
resource_state=self._resource_state,
version_info=version_info)
res = self.run_sync(put_call)
if res.code != 200:
LOGGER.error("Failed to commit resource updates (%s)",
res.result["message"])
def __get_client(cls):
if cls.__client is None:
from inmanta import protocol
cls.__client = protocol.Client("compiler")
return cls.__client
def client(server):
from inmanta import protocol
client = protocol.Client("client")
yield client
async def test_hello_world(server):
client = protocol.Client("client")
result = await client.hello_world()
assert result.code == 200
assert result.result == {"data": "hello-world"}
def __get_client(cls) -> "protocol.Client":
if cls.__client is None:
cls.__client = protocol.Client("compiler")
return cls.__client
