async def start(self,
targets: Sequence[inmanta.protocol.endpoints.CallTarget],
additional_rules: List[routing.Rule] = []) -> None:
"""
Start the server on the current ioloop
"""
global_url_map: Dict[str, Dict[
str, common.UrlMethod]] = self.get_global_url_map(targets)
rules: List[routing.Rule] = []
rules.extend(additional_rules)
for url, handler_config in global_url_map.items():
rules.append(
routing.Rule(routing.PathMatches(url), RESTHandler, {
"transport": self,
"config": handler_config
}))
LOGGER.debug("Registering handler(s) for url %s and methods %s",
url, ", ".join(handler_config.keys()))
application = web.Application(rules, compress_response=True)
crt = inmanta_config.Config.get("server", "ssl_cert_file", None)
key = inmanta_config.Config.get("server", "ssl_key_file", None)
if crt is not None and key is not None:
ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ssl_ctx.load_cert_chain(crt, key)
self._http_server = httpserver.HTTPServer(application,
decompress_request=True,
ssl_options=ssl_ctx)
LOGGER.debug("Created REST transport with SSL")
else:
self._http_server = httpserver.HTTPServer(application,
decompress_request=True)
bind_port = server_config.get_bind_port()
bind_addresses = server_config.server_bind_address.get()
for bind_addr in bind_addresses:
self._http_server.listen(bind_port, bind_addr)
LOGGER.info(f"Server listening on {bind_addr}:{bind_port}")
self.running = True
LOGGER.debug("Start REST transport")
async def test_agents_paging(server, client, env_with_agents: None, environment: str, order_by_column: str, order: str) -> None:
result = await client.get_agents(
environment,
filter={"status": ["paused", "up"]},
)
assert result.code == 200
assert len(result.result["data"]) == 7
all_agents = result.result["data"]
for agent in all_agents:
if not agent["process_name"]:
agent["process_name"] = ""
if not agent["last_failover"]:
agent["last_failover"] = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
else:
agent["last_failover"] = datetime.datetime.strptime(agent["last_failover"], "%Y-%m-%dT%H:%M:%S.%f").replace(
tzinfo=datetime.timezone.utc
)
all_agents_in_expected_order = sorted(all_agents, key=itemgetter(order_by_column, "name"), reverse=order == "DESC")
all_agent_names_in_expected_order = agent_names(all_agents_in_expected_order)
result = await client.get_agents(
environment,
limit=2,
sort=f"{order_by_column}.{order}",
filter={"status": ["paused", "up"]},
)
assert result.code == 200
assert len(result.result["data"]) == 2
assert agent_names(result.result["data"]) == all_agent_names_in_expected_order[:2]
assert result.result["metadata"] == {"total": 7, "before": 0, "after": 5, "page_size": 2}
assert result.result["links"].get("next") is not None
assert result.result["links"].get("prev") is None
port = get_bind_port()
base_url = "http://localhost:%s" % (port,)
http_client = AsyncHTTPClient()
# Test link for next page
url = f"""{base_url}{result.result["links"]["next"]}"""
assert "limit=2" in url
assert "filter.status=" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(environment)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert agent_names(response["data"]) == all_agent_names_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {"total": 7, "before": 2, "after": 3, "page_size": 2}
# Test link for next page
url = f"""{base_url}{response["links"]["next"]}"""
assert "limit=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(environment)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
next_page_agent_names = agent_names(response["data"])
assert next_page_agent_names == all_agent_names_in_expected_order[4:6]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {"total": 7, "before": 4, "after": 1, "page_size": 2}
# Test link for previous page
url = f"""{base_url}{response["links"]["prev"]}"""
assert "limit=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(environment)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
prev_page_agent_names = agent_names(response["data"])
assert prev_page_agent_names == all_agent_names_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {"total": 7, "before": 2, "after": 3, "page_size": 2}
result = await client.get_agents(
environment,
limit=100,
sort=f"{order_by_column}.{order}",
filter={"status": ["paused", "up"]},
)
assert result.code == 200
assert len(result.result["data"]) == 7
assert agent_names(result.result["data"]) == all_agent_names_in_expected_order
assert result.result["metadata"] == {"total": 7, "before": 0, "after": 0, "page_size": 100}
async def test_compile_reports_paging(server, client, env_with_compile_reports,
order_by_column, order):
environment, compile_requested_timestamps = env_with_compile_reports
result = await client.get_compile_reports(
environment,
filter={
"success":
True,
"requested": [
f"lt:{compile_requested_timestamps[-1].astimezone(datetime.timezone.utc)}"
]
},
)
assert result.code == 200
assert len(result.result["data"]) == 6
all_compiles_in_expected_order = sorted(result.result["data"],
key=itemgetter(
order_by_column, "id"),
reverse=order == "DESC")
all_compile_ids_in_expected_order = compile_ids(
all_compiles_in_expected_order)
result = await client.get_compile_reports(
environment,
limit=2,
sort=f"{order_by_column}.{order}",
filter={
"success":
True,
"requested": [
f"lt:{compile_requested_timestamps[-1].astimezone(datetime.timezone.utc)}"
]
},
)
assert result.code == 200
assert len(result.result["data"]) == 2
assert compile_ids(
result.result["data"]) == all_compile_ids_in_expected_order[:2]
assert result.result["metadata"] == {
"total": 6,
"before": 0,
"after": 4,
"page_size": 2
}
assert result.result["links"].get("next") is not None
assert result.result["links"].get("prev") is None
port = get_bind_port()
base_url = "http://localhost:%s" % (port, )
http_client = AsyncHTTPClient()
# Test link for next page
url = f"""{base_url}{result.result["links"]["next"]}"""
assert "limit=2" in url
assert "filter.success=" in url
assert "filter.requested=" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(environment)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert compile_ids(
response["data"]) == all_compile_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 6,
"before": 2,
"after": 2,
"page_size": 2
}
# Test link for next page
url = f"""{base_url}{response["links"]["next"]}"""
assert "limit=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(environment)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
next_page_compile_ids = compile_ids(response["data"])
assert next_page_compile_ids == all_compile_ids_in_expected_order[4:]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is None
assert response["metadata"] == {
"total": 6,
"before": 4,
"after": 0,
"page_size": 2
}
# Test link for previous page
url = f"""{base_url}{response["links"]["prev"]}"""
assert "limit=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(environment)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
prev_page_compile_ids = compile_ids(response["data"])
assert prev_page_compile_ids == all_compile_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 6,
"before": 2,
"after": 2,
"page_size": 2
}
result = await client.get_compile_reports(
environment,
limit=6,
sort=f"{order_by_column}.{order}",
filter={
"success":
True,
"requested": [
f"lt:{compile_requested_timestamps[-1].astimezone(datetime.timezone.utc)}"
]
},
)
assert result.code == 200
assert len(result.result["data"]) == 6
assert compile_ids(
result.result["data"]) == all_compile_ids_in_expected_order
assert result.result["metadata"] == {
"total": 6,
"before": 0,
"after": 0,
"page_size": 6
}
def _collect_server_information(self) -> List[Server]:
bind_port = config.get_bind_port()
server_address = config.server_address.get()
return [
Server(url=AnyUrl(url=f"http://{server_address}:{bind_port}/", scheme="http", host=server_address, port=bind_port))
]
async def run(
self,
force_update: Optional[bool] = False
) -> Tuple[bool, Optional[model.CompileData]]:
"""
Runs this compile run.
:return: Tuple of a boolean representing success and the compile data, if any.
"""
success = False
now = datetime.datetime.now().astimezone()
await self.request.update_fields(started=now)
compile_data_json_file = NamedTemporaryFile()
try:
await self._start_stage("Init", "")
environment_id = self.request.environment
project_dir = self._project_dir
env = await data.Environment.get_by_id(environment_id)
env_string = ", ".join([
f"{k}='{v}'"
for k, v in self.request.environment_variables.items()
])
assert self.stage
await self.stage.update_streams(
out=
f"Using extra environment variables during compile {env_string}\n"
)
if env is None:
await self._error("Environment %s does not exist." %
environment_id)
await self._end_stage(-1)
return False, None
if not os.path.exists(project_dir):
await self._info(
"Creating project directory for environment %s at %s" %
(environment_id, project_dir))
os.mkdir(project_dir)
# Use a separate venv to compile the project to prevent that packages are installed in the
# venv of the Inmanta server.
venv_dir = os.path.join(project_dir, ".env")
async def ensure_venv() -> Optional[data.Report]:
"""
Ensure a venv is present at `venv_dir`.
"""
virtual_env = VirtualEnv(venv_dir)
if virtual_env.exists():
return None
await self._start_stage("Creating venv", command="")
try:
virtual_env.init_env()
except VenvCreationFailedError as e:
await self._error(message=e.msg)
return await self._end_stage(returncode=1)
else:
return await self._end_stage(returncode=0)
async def update_modules() -> data.Report:
return await run_compile_stage_in_venv(
"Updating modules", ["-vvv", "-X", "modules", "update"],
cwd=project_dir)
async def install_modules() -> data.Report:
return await run_compile_stage_in_venv(
"Installing modules", ["-vvv", "-X", "project", "install"],
cwd=project_dir)
async def run_compile_stage_in_venv(
stage_name: str,
inmanta_args: List[str],
cwd: str,
env: Dict[str, str] = {}) -> data.Report:
"""
Run a compile stage by executing the given command in the venv `venv_dir`.
:param stage_name: Name of the compile stage.
:param inmanta_args: The command to be executed in the venv. This command should not include the part
["<python-interpreter>", "-m", "inmanta.app"]
:param cwd: The current working directory to be used for the command invocation.
:param env: Execute the command with these environment variables.
"""
LOGGER.info(stage_name)
python_path = PythonEnvironment.get_python_path_for_env_path(
venv_dir)
assert os.path.exists(python_path)
full_cmd = [python_path, "-m", "inmanta.app"] + inmanta_args
return await self._run_compile_stage(stage_name, full_cmd, cwd,
env)
async def setup(
) -> AsyncIterator[Awaitable[Optional[data.Report]]]:
"""
Returns an iterator over all setup stages. Inspecting stage success state is the responsibility of the caller.
"""
repo_url: str = env.repo_url
repo_branch: str = env.repo_branch
if os.path.exists(os.path.join(project_dir, "project.yml")):
yield self._end_stage(0)
yield ensure_venv()
should_update: bool = force_update or self.request.force_update
# switch branches
if repo_branch:
branch = await self.get_branch()
if branch is not None and repo_branch != branch:
if should_update:
yield self._run_compile_stage(
"Fetching new branch heads",
["git", "fetch"], project_dir)
yield self._run_compile_stage(
f"Switching branch from {branch} to {repo_branch}",
["git", "checkout", repo_branch],
project_dir,
)
if not should_update:
# if we update, update procedure will install modules
yield install_modules()
# update project
if should_update:
# only pull changes if there is an upstream branch
if await self.get_upstream_branch():
yield self._run_compile_stage(
"Pulling updates", ["git", "pull"],
project_dir)
yield update_modules()
else:
if not repo_url:
await self._warning(
f"Failed to compile: no project found in {project_dir} and no repository set."
)
yield self._end_stage(1)
return
if len(os.listdir(project_dir)) > 0:
await self._warning(
f"Failed to compile: no project found in {project_dir} but directory is not empty."
)
yield self._end_stage(1)
return
yield self._end_stage(0)
# clone repo and install project
cmd = ["git", "clone", repo_url, "."]
if repo_branch:
cmd.extend(["-b", repo_branch])
yield self._run_compile_stage("Cloning repository", cmd,
project_dir)
yield ensure_venv()
yield install_modules()
async for stage in setup():
stage_result: Optional[data.Report] = await stage
if stage_result and (stage_result.returncode is None
or stage_result.returncode > 0):
return False, None
server_address = opt.server_address.get()
server_port = opt.get_bind_port()
cmd = [
"-vvv",
"export",
"-X",
"-e",
str(environment_id),
"--server_address",
server_address,
"--server_port",
str(server_port),
"--metadata",
json.dumps(self.request.metadata),
"--export-compile-data",
"--export-compile-data-file",
compile_data_json_file.name,
]
if not self.request.do_export:
f = NamedTemporaryFile()
cmd.append("-j")
cmd.append(f.name)
if config.Config.get("server", "auth", False):
token = encode_token(["compiler", "api"], str(environment_id))
cmd.append("--token")
cmd.append(token)
if opt.server_ssl_cert.get() is not None:
cmd.append("--ssl")
if opt.server_ssl_ca_cert.get() is not None:
cmd.append("--ssl-ca-cert")
cmd.append(opt.server_ssl_ca_cert.get())
self.tail_stdout = ""
env_vars_compile: Dict[str, str] = os.environ.copy()
env_vars_compile.update(self.request.environment_variables)
result: data.Report = await run_compile_stage_in_venv(
"Recompiling configuration model",
cmd,
cwd=project_dir,
env=env_vars_compile)
success = result.returncode == 0
if not success:
if self.request.do_export:
LOGGER.warning("Compile %s failed", self.request.id)
else:
LOGGER.debug("Compile %s failed", self.request.id)
print("---", self.tail_stdout, result.errstream)
match = re.search(r"Committed resources with version (\d+)",
self.tail_stdout)
if match:
self.version = int(match.group(1))
except CancelledError:
# This compile was cancelled. Catch it here otherwise a warning will be printed in the logs because of an
# unhandled exception in a backgrounded coroutine.
pass
except Exception:
LOGGER.exception("An error occurred while recompiling")
finally:
async def warn(message: str) -> None:
if self.stage is not None:
await self._warning(message)
else:
LOGGER.warning(message)
with compile_data_json_file as file:
compile_data_json: str = file.read().decode()
if compile_data_json:
try:
return success, model.CompileData.parse_raw(
compile_data_json)
except json.JSONDecodeError:
await warn(
"Failed to load compile data json for compile %s. Invalid json: '%s'"
% (self.request.id, compile_data_json))
except pydantic.ValidationError:
await warn(
"Failed to parse compile data for compile %s. Json does not match CompileData model: '%s'"
% (self.request.id, compile_data_json))
return success, None
async def test_resource_logs_paging(server, client, order_by_column, order,
env_with_logs):
"""Test querying resource logs with paging, using different sorting parameters."""
environment, msg_timings = env_with_logs
result = await client.resource_logs(
environment,
resource_id_a,
filter={
"minimal_log_level": "INFO",
"timestamp": [f"ge:{msg_timings[2]}"]
},
)
assert result.code == 200
assert len(result.result["data"]) == 7
all_logs_in_expected_order = sorted(result.result["data"],
key=itemgetter(order_by_column),
reverse=order == "DESC")
all_log_messages_in_expected_order = log_messages(
all_logs_in_expected_order)
result = await client.resource_logs(
environment,
resource_id_a,
limit=2,
sort=f"{order_by_column}.{order}",
filter={
"minimal_log_level": "INFO",
"timestamp": [f"ge:{msg_timings[2]}"]
},
)
assert result.code == 200
assert len(result.result["data"]) == 2
assert log_messages(
result.result["data"]) == all_log_messages_in_expected_order[:2]
assert result.result["metadata"] == {
"total": 7,
"before": 0,
"after": 5,
"page_size": 2
}
assert result.result["links"].get("next") is not None
assert result.result["links"].get("prev") is None
port = get_bind_port()
base_url = "http://localhost:%s" % (port, )
http_client = AsyncHTTPClient()
# Test link for next page
url = f"""{base_url}{result.result["links"]["next"]}"""
assert "limit=2" in url
assert "filter.minimal_log_level=INFO" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": environment},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert log_messages(
response["data"]) == all_log_messages_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 7,
"before": 2,
"after": 3,
"page_size": 2
}
# Test link for next page
url = f"""{base_url}{response["links"]["next"]}"""
# The filters should be present for the links as well
assert "limit=2" in url
assert "filter.minimal_log_level=INFO" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": environment},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
next_page_log_messages = log_messages(response["data"])
assert next_page_log_messages == all_log_messages_in_expected_order[4:6]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 7,
"before": 4,
"after": 1,
"page_size": 2
}
# Test link for previous page
url = f"""{base_url}{response["links"]["prev"]}"""
assert "limit=2" in url
assert "filter.minimal_log_level=INFO" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": environment},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
prev_page_log_messages = log_messages(response["data"])
assert prev_page_log_messages == all_log_messages_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 7,
"before": 2,
"after": 3,
"page_size": 2
}
async def test_desired_state_versions_paging(
server, client, order: str,
environments_with_versions: Tuple[Dict[str, uuid.UUID],
List[datetime.datetime]]):
"""Test querying desired state versions with paging, using different sorting parameters."""
environments, timestamps = environments_with_versions
env = environments["multiple_versions"]
order_by_column = "version"
result = await client.list_desired_state_versions(
env,
filter={
"date": [f"gt:{timestamps[1].astimezone(datetime.timezone.utc)}"]
},
)
assert result.code == 200
assert len(result.result["data"]) == 7
all_versions_in_expected_order = sorted(result.result["data"],
key=itemgetter(order_by_column),
reverse=order == "DESC")
all_versions_in_expected_order = version_numbers(
all_versions_in_expected_order)
result = await client.list_desired_state_versions(
env,
limit=2,
sort=f"{order_by_column}.{order}",
filter={
"date": [f"gt:{timestamps[1].astimezone(datetime.timezone.utc)}"]
},
)
assert result.code == 200
assert len(result.result["data"]) == 2
assert version_numbers(
result.result["data"]) == all_versions_in_expected_order[:2]
assert result.result["metadata"] == {
"total": 7,
"before": 0,
"after": 5,
"page_size": 2
}
assert result.result["links"].get("next") is not None
assert result.result["links"].get("prev") is None
port = get_bind_port()
base_url = "http://localhost:%s" % (port, )
http_client = AsyncHTTPClient()
# Test link for next page
url = f"""{base_url}{result.result["links"]["next"]}"""
assert "limit=2" in url
assert "filter.date=" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert version_numbers(
response["data"]) == all_versions_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 7,
"before": 2,
"after": 3,
"page_size": 2
}
# Test link for next page
url = f"""{base_url}{response["links"]["next"]}"""
# The filters should be present for the links as well
assert "limit=2" in url
assert "filter.date=" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
next_page_versions = version_numbers(response["data"])
assert next_page_versions == all_versions_in_expected_order[4:6]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 7,
"before": 4,
"after": 1,
"page_size": 2
}
# Test link for previous page
url = f"""{base_url}{response["links"]["prev"]}"""
assert "limit=2" in url
assert "filter.date=" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
prev_page_versions = version_numbers(response["data"])
assert prev_page_versions == all_versions_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 7,
"before": 2,
"after": 3,
"page_size": 2
}
result = await client.list_desired_state_versions(
env,
limit=100,
sort=f"{order_by_column}.{order}",
filter={
"date": [f"gt:{timestamps[1].astimezone(datetime.timezone.utc)}"]
},
)
assert result.code == 200
assert len(result.result["data"]) == 7
assert version_numbers(
result.result["data"]) == all_versions_in_expected_order
assert result.result["metadata"] == {
"total": 7,
"before": 0,
"after": 0,
"page_size": 100
}
async def test_resource_history_paging(server, client, order_by_column, order,
env_with_resources):
"""Test querying resource history with paging, using different sorting parameters."""
env, cm_times, ids, resources = env_with_resources
resource_with_long_history = ids["long_history"]
result = await client.resource_history(env.id, resource_with_long_history)
assert result.code == 200
assert len(result.result["data"]) == 5
all_resources_in_expected_order = sorted(result.result["data"],
key=itemgetter(
order_by_column,
"attribute_hash"),
reverse=order == "DESC")
all_resource_ids_in_expected_order = attribute_hashes(
all_resources_in_expected_order)
result = await client.resource_history(env.id,
resource_with_long_history,
limit=2,
sort=f"{order_by_column}.{order}")
assert result.code == 200
assert len(result.result["data"]) == 2
assert attribute_hashes(
result.result["data"]) == all_resource_ids_in_expected_order[:2]
assert result.result["metadata"] == {
"total": 5,
"before": 0,
"after": 3,
"page_size": 2
}
assert result.result["links"].get("next") is not None
assert result.result["links"].get("prev") is None
port = get_bind_port()
base_url = "http://localhost:%s" % (port, )
http_client = AsyncHTTPClient()
# Test link for next page
url = f"""{base_url}{result.result["links"]["next"]}"""
assert "limit=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env.id)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert attribute_hashes(
response["data"]) == all_resource_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 5,
"before": 2,
"after": 1,
"page_size": 2
}
# Test link for next page
url = f"""{base_url}{response["links"]["next"]}"""
assert "limit=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env.id)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
next_page_instance_ids = attribute_hashes(response["data"])
assert next_page_instance_ids == all_resource_ids_in_expected_order[4:]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is None
assert response["metadata"] == {
"total": 5,
"before": 4,
"after": 0,
"page_size": 2
}
# Test link for previous page
url = f"""{base_url}{response["links"]["prev"]}"""
assert "limit=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env.id)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
prev_page_instance_ids = attribute_hashes(response["data"])
assert prev_page_instance_ids == all_resource_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 5,
"before": 2,
"after": 1,
"page_size": 2
}
result = await client.resource_history(env.id,
resource_with_long_history,
limit=5,
sort=f"{order_by_column}.{order}")
assert result.code == 200
assert len(result.result["data"]) == 5
assert attribute_hashes(
result.result["data"]) == all_resource_ids_in_expected_order
assert result.result["metadata"] == {
"total": 5,
"before": 0,
"after": 0,
"page_size": 5
}
async def test_resource_action_pagination(postgresql_client, client,
clienthelper, server, agent):
""" Test querying resource actions via the API, including the pagination links."""
project = data.Project(name="test")
await project.insert()
env = data.Environment(name="dev",
project=project.id,
repo_url="",
repo_branch="")
await env.insert()
# Add multiple versions of model
for i in range(0, 11):
cm = data.ConfigurationModel(
environment=env.id,
version=i,
date=datetime.now(),
total=1,
version_info={},
)
await cm.insert()
# Add resource actions for motd
motd_first_start_time = datetime.now()
earliest_action_id = uuid.uuid4()
resource_action = data.ResourceAction(
environment=env.id,
version=0,
resource_version_ids=[f"std::File[agent1,path=/etc/motd],v={0}"],
action_id=earliest_action_id,
action=const.ResourceAction.deploy,
started=motd_first_start_time - timedelta(minutes=1),
)
await resource_action.insert()
resource_action.add_logs([
data.LogLine.log(logging.INFO,
"Successfully stored version %(version)d",
version=0)
])
await resource_action.save()
action_ids_with_the_same_timestamp = []
for i in range(1, 6):
action_id = uuid.uuid4()
action_ids_with_the_same_timestamp.append(action_id)
resource_action = data.ResourceAction(
environment=env.id,
version=i,
resource_version_ids=[f"std::File[agent1,path=/etc/motd],v={i}"],
action_id=action_id,
action=const.ResourceAction.deploy,
started=motd_first_start_time,
)
await resource_action.insert()
resource_action.add_logs([
data.LogLine.log(logging.INFO,
"Successfully stored version %(version)d",
version=i)
])
await resource_action.save()
action_ids_with_the_same_timestamp = sorted(
action_ids_with_the_same_timestamp, reverse=True)
later_action_id = uuid.uuid4()
resource_action = data.ResourceAction(
environment=env.id,
version=6,
resource_version_ids=[f"std::File[agent1,path=/etc/motd],v={6}"],
action_id=later_action_id,
action=const.ResourceAction.deploy,
started=motd_first_start_time + timedelta(minutes=6),
)
await resource_action.insert()
resource_action.add_logs([
data.LogLine.log(logging.INFO,
"Successfully stored version %(version)d",
version=6)
])
await resource_action.save()
for i in range(0, 11):
res1 = data.Resource.new(
environment=env.id,
resource_version_id="std::File[agent1,path=/etc/motd],v=%s" %
str(i),
status=const.ResourceState.deployed,
last_deploy=datetime.now() + timedelta(minutes=i),
attributes={
"attr": [{
"a": 1,
"b": "c"
}],
"path": "/etc/motd"
},
)
await res1.insert()
result = await client.get_resource_actions(
tid=env.id,
resource_type="std::File",
attribute="path",
attribute_value="/etc/motd",
last_timestamp=motd_first_start_time + timedelta(minutes=7),
limit=2,
)
assert result.code == 200
resource_actions = result.result["data"]
expected_action_ids = [later_action_id
] + action_ids_with_the_same_timestamp[:1]
assert [
uuid.UUID(resource_action["action_id"])
for resource_action in resource_actions
] == expected_action_ids
# Use the next link for pagination
next_page = result.result["links"]["next"]
port = opt.get_bind_port()
base_url = "http://localhost:%s" % (port, )
url = f"{base_url}{next_page}"
client = AsyncHTTPClient()
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env.id)},
)
response = await client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
second_page_action_ids = [
uuid.UUID(resource_action["action_id"])
for resource_action in response["data"]
]
assert second_page_action_ids == action_ids_with_the_same_timestamp[1:3]
next_page = response["links"]["next"]
url = f"{base_url}{next_page}"
request.url = url
response = await client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
third_page_action_ids = [
uuid.UUID(resource_action["action_id"])
for resource_action in response["data"]
]
assert third_page_action_ids == action_ids_with_the_same_timestamp[3:5]
# Go back to the previous page
prev_page = response["links"]["prev"]
url = f"{base_url}{prev_page}"
request.url = url
response = await client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
action_ids = [
uuid.UUID(resource_action["action_id"])
for resource_action in response["data"]
]
assert action_ids == second_page_action_ids
# And back to the third
prev_page = response["links"]["next"]
url = f"{base_url}{prev_page}"
request.url = url
response = await client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
action_ids = [
uuid.UUID(resource_action["action_id"])
for resource_action in response["data"]
]
assert action_ids == third_page_action_ids
async def test_resources_paging(server, client, order_by_column, order,
env_with_resources):
"""Test querying resources with paging, using different sorting parameters."""
env = env_with_resources
version = 3
result = await client.get_resources_in_version(
env.id,
version,
filter={"agent": ["1", "2"]},
)
assert result.code == 200
assert len(result.result["data"]) == 5
flattened_resources = [
VersionedResource(**res).all_fields for res in result.result["data"]
]
all_resources_in_expected_order = sorted(flattened_resources,
key=itemgetter(
order_by_column,
"resource_version_id"),
reverse=order == "DESC")
all_resource_ids_in_expected_order = resource_ids(
all_resources_in_expected_order)
result = await client.get_resources_in_version(
env.id,
version,
limit=2,
sort=f"{order_by_column}.{order}",
filter={"agent": ["1", "2"]})
assert result.code == 200
assert len(result.result["data"]) == 2
assert resource_ids(
result.result["data"]) == all_resource_ids_in_expected_order[:2]
assert result.result["metadata"] == {
"total": 5,
"before": 0,
"after": 3,
"page_size": 2
}
assert result.result["links"].get("next") is not None
assert result.result["links"].get("prev") is None
port = get_bind_port()
base_url = "http://localhost:%s" % (port, )
http_client = AsyncHTTPClient()
# Test link for next page
url = f"""{base_url}{result.result["links"]["next"]}"""
assert "limit=2" in url
assert "filter.agent=1" in url
assert "filter.agent=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env.id)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert resource_ids(
response["data"]) == all_resource_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 5,
"before": 2,
"after": 1,
"page_size": 2
}
# Test link for next page
url = f"""{base_url}{response["links"]["next"]}"""
# The filters should be present for the links as well
assert "limit=2" in url
assert "filter.agent=1" in url
assert "filter.agent=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env.id)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
next_page_instance_ids = resource_ids(response["data"])
assert next_page_instance_ids == all_resource_ids_in_expected_order[4:]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is None
assert response["metadata"] == {
"total": 5,
"before": 4,
"after": 0,
"page_size": 2
}
# Test link for previous page
url = f"""{base_url}{response["links"]["prev"]}"""
assert "limit=2" in url
assert "filter.agent=1" in url
assert "filter.agent=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(env.id)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
prev_page_instance_ids = resource_ids(response["data"])
assert prev_page_instance_ids == all_resource_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 5,
"before": 2,
"after": 1,
"page_size": 2
}
async def test_facts_paging(server, client, order_by_column, order,
env_with_facts):
"""Test querying facts with paging, using different sorting parameters."""
env, _, _ = env_with_facts
result = await client.get_all_facts(
env,
filter={"name": "res"},
)
assert result.code == 200
assert len(result.result["data"]) == 6
all_facts_in_expected_order = sorted(result.result["data"],
key=itemgetter(order_by_column, "id"),
reverse=order == "DESC")
all_fact_ids_in_expected_order = fact_ids(all_facts_in_expected_order)
result = await client.get_all_facts(env,
limit=2,
sort=f"{order_by_column}.{order}",
filter={"name": "res"})
assert result.code == 200
assert len(result.result["data"]) == 2
assert fact_ids(
result.result["data"]) == all_fact_ids_in_expected_order[:2]
assert result.result["metadata"] == {
"total": 6,
"before": 0,
"after": 4,
"page_size": 2
}
assert result.result["links"].get("next") is not None
assert result.result["links"].get("prev") is None
port = get_bind_port()
base_url = "http://localhost:%s" % (port, )
http_client = AsyncHTTPClient()
# Test link for next page
url = f"""{base_url}{result.result["links"]["next"]}"""
assert "limit=2" in url
assert "filter.name=res" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": env},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert fact_ids(response["data"]) == all_fact_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 6,
"before": 2,
"after": 2,
"page_size": 2
}
# Test link for next page
url = f"""{base_url}{response["links"]["next"]}"""
# The filters should be present for the links as well
assert "limit=2" in url
assert "filter.name=res" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": env},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
next_page_ids = fact_ids(response["data"])
assert next_page_ids == all_fact_ids_in_expected_order[4:]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is None
assert response["metadata"] == {
"total": 6,
"before": 4,
"after": 0,
"page_size": 2
}
# Test link for previous page
url = f"""{base_url}{response["links"]["prev"]}"""
assert "limit=2" in url
assert "filter.name=res" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": env},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
prev_page_ids = fact_ids(response["data"])
assert prev_page_ids == all_fact_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 6,
"before": 2,
"after": 2,
"page_size": 2
}
async def test_parameters_paging(server, client, order_by_column, order,
env_with_parameters):
"""Test querying parameters with paging, using different sorting parameters."""
env, timestamps = env_with_parameters
result = await client.get_parameters(
env,
filter={"source": "plugin"},
)
assert result.code == 200
assert len(result.result["data"]) == 6
all_parameters = result.result["data"]
for parameter in all_parameters:
if not parameter["updated"]:
parameter["updated"] = datetime.datetime.min.replace(
tzinfo=datetime.timezone.utc)
else:
parameter["updated"] = datetime.datetime.strptime(
parameter["updated"],
"%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=datetime.timezone.utc)
all_parameters_in_expected_order = sorted(all_parameters,
key=itemgetter(
order_by_column, "id"),
reverse=order == "DESC")
all_parameter_ids_in_expected_order = parameter_ids(
all_parameters_in_expected_order)
result = await client.get_parameters(env,
limit=2,
sort=f"{order_by_column}.{order}",
filter={"source": "plugin"})
assert result.code == 200
assert len(result.result["data"]) == 2
assert parameter_ids(
result.result["data"]) == all_parameter_ids_in_expected_order[:2]
assert result.result["metadata"] == {
"total": 6,
"before": 0,
"after": 4,
"page_size": 2
}
assert result.result["links"].get("next") is not None
assert result.result["links"].get("prev") is None
port = get_bind_port()
base_url = "http://localhost:%s" % (port, )
http_client = AsyncHTTPClient()
# Test link for next page
url = f"""{base_url}{result.result["links"]["next"]}"""
assert "limit=2" in url
assert "filter.source=plugin" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": env},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert parameter_ids(
response["data"]) == all_parameter_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 6,
"before": 2,
"after": 2,
"page_size": 2
}
# Test link for next page
url = f"""{base_url}{response["links"]["next"]}"""
# The filters should be present for the links as well
assert "limit=2" in url
assert "filter.source=plugin" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": env},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
next_page_ids = parameter_ids(response["data"])
assert next_page_ids == all_parameter_ids_in_expected_order[4:]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is None
assert response["metadata"] == {
"total": 6,
"before": 4,
"after": 0,
"page_size": 2
}
# Test link for previous page
url = f"""{base_url}{response["links"]["prev"]}"""
assert "limit=2" in url
assert "filter.source=plugin" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": env},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
prev_page_ids = parameter_ids(response["data"])
assert prev_page_ids == all_parameter_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 6,
"before": 2,
"after": 2,
"page_size": 2
}
async def test_notifications_paging(server, client,
environment_with_notifications, order):
environment = environment_with_notifications
order_by_column = "created"
result = await client.list_notifications(
environment,
filter={
"read": False,
},
)
assert result.code == 200
assert len(result.result["data"]) == 6
all_notifications_in_expected_order = sorted(result.result["data"],
key=itemgetter(
order_by_column, "id"),
reverse=order == "DESC")
all_notification_ids_in_expected_order = notification_ids(
all_notifications_in_expected_order)
result = await client.list_notifications(
environment,
limit=2,
sort=f"{order_by_column}.{order}",
filter={
"read": False,
},
)
assert result.code == 200
assert len(result.result["data"]) == 2
assert notification_ids(
result.result["data"]) == all_notification_ids_in_expected_order[:2]
assert result.result["metadata"] == {
"total": 6,
"before": 0,
"after": 4,
"page_size": 2
}
assert result.result["links"].get("next") is not None
assert result.result["links"].get("prev") is None
port = get_bind_port()
base_url = "http://localhost:%s" % (port, )
http_client = AsyncHTTPClient()
# Test link for next page
url = f"""{base_url}{result.result["links"]["next"]}"""
assert "limit=2" in url
assert "filter.read=" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(environment)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert notification_ids(
response["data"]) == all_notification_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 6,
"before": 2,
"after": 2,
"page_size": 2
}
# Test link for next page
url = f"""{base_url}{response["links"]["next"]}"""
assert "limit=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(environment)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert notification_ids(
response["data"]) == all_notification_ids_in_expected_order[4:]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is None
assert response["metadata"] == {
"total": 6,
"before": 4,
"after": 0,
"page_size": 2
}
# Test link for previous page
url = f"""{base_url}{response["links"]["prev"]}"""
assert "limit=2" in url
request = HTTPRequest(
url=url,
headers={"X-Inmanta-tid": str(environment)},
)
response = await http_client.fetch(request, raise_error=False)
assert response.code == 200
response = json.loads(response.body.decode("utf-8"))
assert notification_ids(
response["data"]) == all_notification_ids_in_expected_order[2:4]
assert response["links"].get("prev") is not None
assert response["links"].get("next") is not None
assert response["metadata"] == {
"total": 6,
"before": 2,
"after": 2,
"page_size": 2
}
result = await client.list_notifications(
environment,
limit=6,
sort=f"{order_by_column}.{order}",
filter={
"read": False,
},
)
assert result.code == 200
assert len(result.result["data"]) == 6
assert notification_ids(
result.result["data"]) == all_notification_ids_in_expected_order
assert result.result["metadata"] == {
"total": 6,
"before": 0,
"after": 0,
"page_size": 6
}
async def run(
self,
force_update: Optional[bool] = False
) -> Tuple[bool, Optional[model.CompileData]]:
success = False
now = datetime.datetime.now()
await self.request.update_fields(started=now)
compile_data_json_file = NamedTemporaryFile()
try:
await self._start_stage("Init", "")
environment_id = self.request.environment
project_dir = self._project_dir
env = await data.Environment.get_by_id(environment_id)
env_string = ", ".join([
f"{k}='{v}'"
for k, v in self.request.environment_variables.items()
])
await self.stage.update_streams(
out=
f"Using extra environment variables during compile {env_string}\n"
)
if env is None:
await self._error("Environment %s does not exist." %
environment_id)
await self._end_stage(-1)
return False, None
inmanta_path = [sys.executable, "-m", "inmanta.app"]
if not os.path.exists(project_dir):
await self._info(
"Creating project directory for environment %s at %s" %
(environment_id, project_dir))
os.mkdir(project_dir)
repo_url: str = env.repo_url
repo_branch: str = env.repo_branch
if not repo_url:
if not os.path.exists(os.path.join(project_dir,
"project.yml")):
await self._warning(
f"Failed to compile: no project found in {project_dir} and no repository set"
)
await self._end_stage(0)
else:
await self._end_stage(0)
# checkout repo
if not os.path.exists(os.path.join(project_dir, ".git")):
cmd = ["git", "clone", repo_url, "."]
if repo_branch:
cmd.extend(["-b", repo_branch])
result = await self._run_compile_stage(
"Cloning repository", cmd, project_dir)
if result.returncode is None or result.returncode > 0:
return False, None
elif force_update or self.request.force_update:
result = await self._run_compile_stage(
"Fetching changes", ["git", "fetch", repo_url],
project_dir)
if repo_branch:
branch = await self.get_branch()
if branch is not None and repo_branch != branch:
result = await self._run_compile_stage(
f"switching branch from {branch} to {repo_branch}",
["git", "checkout", repo_branch], project_dir)
if force_update or self.request.force_update:
await self._run_compile_stage("Pulling updates",
["git", "pull"], project_dir)
LOGGER.info("Installing and updating modules")
await self._run_compile_stage(
"Updating modules",
inmanta_path + ["modules", "update"], project_dir)
server_address = opt.server_address.get()
server_port = opt.get_bind_port()
cmd = inmanta_path + [
"-vvv",
"export",
"-X",
"-e",
str(environment_id),
"--server_address",
server_address,
"--server_port",
str(server_port),
"--metadata",
json.dumps(self.request.metadata),
"--export-compile-data",
"--export-compile-data-file",
compile_data_json_file.name,
]
if not self.request.do_export:
f = NamedTemporaryFile()
cmd.append("-j")
cmd.append(f.name)
if config.Config.get("server", "auth", False):
token = encode_token(["compiler", "api"], str(environment_id))
cmd.append("--token")
cmd.append(token)
if opt.server_ssl_cert.get() is not None:
cmd.append("--ssl")
if opt.server_ssl_ca_cert.get() is not None:
cmd.append("--ssl-ca-cert")
cmd.append(opt.server_ssl_ca_cert.get())
self.tail_stdout = ""
env_vars_compile: Dict[str, str] = os.environ.copy()
env_vars_compile.update(self.request.environment_variables)
result = await self._run_compile_stage(
"Recompiling configuration model",
cmd,
project_dir,
env=env_vars_compile)
success = result.returncode == 0
if not success:
if self.request.do_export:
LOGGER.warning("Compile %s failed", self.request.id)
else:
LOGGER.debug("Compile %s failed", self.request.id)
print("---", self.tail_stdout, result.errstream)
match = re.search(r"Committed resources with version (\d+)",
self.tail_stdout)
if match:
self.version = int(match.group(1))
except CancelledError:
# This compile was cancelled. Catch it here otherwise a warning will be printed in the logs because of an
# unhandled exception in a backgrounded coroutine.
pass
except Exception:
LOGGER.exception("An error occured while recompiling")
finally:
with compile_data_json_file as file:
compile_data_json: str = file.read().decode()
if compile_data_json:
try:
return success, model.CompileData.parse_raw(
compile_data_json)
except json.JSONDecodeError:
LOGGER.warning(
"Failed to load compile data json for compile %s. Invalid json: '%s'",
(self.request.id, compile_data_json),
)
except pydantic.ValidationError:
LOGGER.warning(
"Failed to parse compile data for compile %s. Json does not match CompileData model: '%s'",
(self.request.id, compile_data_json),
)
return success, None