async def environment_create_token(self, env: data.Environment,
client_types: List[str],
idempotent: bool) -> str:
"""
Create a new auth token for this environment
"""
return encode_token(client_types, str(env.id), idempotent)
def bootstrap_token(client: Client) -> None:
"""
Generate a bootstrap token that provides access to everything. This token is only valid for 3600 seconds.
"""
click.echo(
"Token: " +
protocol.encode_token(["api", "compiler", "agent"], expire=3600))
def _make_agent_config(self, env: data.Environment, agent_names: list,
agent_map: dict) -> str:
"""
Generate the config file for the process that hosts the autostarted agents
:param env: The environment for which to autostart agents
:param agent_names: The names of the agents
:param agent_map: The agent mapping to use
:return: A string that contains the config file content.
"""
environment_id = str(env.id)
port = Config.get("server_rest_transport", "port", "8888")
privatestatedir = os.path.join(
Config.get("config", "state-dir", "/var/lib/inmanta"),
environment_id)
agent_splay = yield env.get(data.AUTOSTART_SPLAY)
# generate config file
config = """[config]
heartbeat-interval = 60
state-dir=%(statedir)s
agent-names = %(agents)s
environment=%(env_id)s
agent-map=%(agent_map)s
agent_splay=%(agent_splay)d
[agent_rest_transport]
port=%(port)s
host=localhost
""" % {
"agents":
",".join(agent_names),
"env_id":
environment_id,
"port":
port,
"agent_map":
",".join(["%s=%s" % (k, v) for (k, v) in agent_map.items()]),
"statedir":
privatestatedir,
"agent_splay":
agent_splay
}
if server_config.server_enable_auth:
token = protocol.encode_token(["agent"], environment_id)
config += """
token=%s
""" % (token)
ssl_cert = Config.get("server", "ssl_key_file", None)
ssl_ca = Config.get("server", "ssl_cert_file", None)
if ssl_ca is not None and ssl_cert is not None:
config += """
ssl=True
ssl_ca_cert_file=%s
""" % (ssl_ca)
return config
def test_jwt_create(inmanta_config):
"""
Test creating, signing and verifying JWT with HS256 from the configuration
"""
jot = protocol.encode_token(["api"])
payload = protocol.decode_token(jot)
assert "api" in payload["urn:inmanta:ct"]
# test creating an idempotent token
jot1 = protocol.encode_token(["agent"], idempotent=True)
jot3 = protocol.encode_token(["agent"])
time.sleep(1)
jot2 = protocol.encode_token(["agent"], idempotent=True)
jot4 = protocol.encode_token(["agent"])
assert jot1 == jot2
assert jot3 != jot4
assert jot1 != jot3
assert jot2 != jot3
def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request):
from inmanta.server import Server
state_dir = tempfile.mkdtemp()
ssl, auth = request.param
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
if auth:
config.Config.set("server", "auth", "true")
from inmanta import protocol
for x, ct in [("server", None), ("server_rest_transport", None),
("agent_rest_transport", ["agent"]),
("compiler_rest_transport", ["compiler"]),
("client_rest_transport", ["api", "compiler"]),
("cmdline_rest_transport", ["api"])]:
if ssl:
config.Config.set(x, "ssl_cert_file",
os.path.join(path, "server.crt"))
config.Config.set(x, "ssl_key_file",
os.path.join(path, "server.open.key"))
config.Config.set(x, "ssl_ca_cert_file",
os.path.join(path, "server.crt"))
config.Config.set(x, "ssl", "True")
if auth and ct is not None:
token = protocol.encode_token(ct)
config.Config.set(x, "token", token)
port = get_free_tcp_port()
config.Config.get(
"database", "name", "inmanta-" +
''.join(random.choice(string.ascii_letters) for _ in range(10)))
config.Config.set("config", "state-dir", state_dir)
config.Config.set("config", "log-dir", os.path.join(state_dir, "logs"))
config.Config.set("server_rest_transport", "port", port)
config.Config.set("agent_rest_transport", "port", port)
config.Config.set("compiler_rest_transport", "port", port)
config.Config.set("client_rest_transport", "port", port)
config.Config.set("cmdline_rest_transport", "port", port)
config.Config.set(
"config", "executable",
os.path.abspath(os.path.join(__file__, "../../src/inmanta/app.py")))
config.Config.set("server", "agent-timeout", "2")
server = Server(database_host="localhost",
database_port=int(mongo_db.port),
io_loop=io_loop)
server.start()
yield server
server.stop()
shutil.rmtree(state_dir)
async def server_multi(server_pre_start, event_loop, inmanta_config,
postgres_db, database_name, request, clean_reset,
unused_tcp_port_factory):
"""
:param event_loop: explicitly include event_loop to make sure event loop started before and closed after this fixture.
May not be required
"""
state_dir = tempfile.mkdtemp()
ssl, auth, ca = request.param
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
if auth:
config.Config.set("server", "auth", "true")
for x, ct in [
("server", None),
("agent_rest_transport", ["agent"]),
("compiler_rest_transport", ["compiler"]),
("client_rest_transport", ["api", "compiler"]),
("cmdline_rest_transport", ["api"]),
]:
if ssl and not ca:
config.Config.set(x, "ssl_cert_file",
os.path.join(path, "server.crt"))
config.Config.set(x, "ssl_key_file",
os.path.join(path, "server.open.key"))
config.Config.set(x, "ssl_ca_cert_file",
os.path.join(path, "server.crt"))
config.Config.set(x, "ssl", "True")
if ssl and ca:
capath = os.path.join(path, "ca", "enduser-certs")
config.Config.set(x, "ssl_cert_file",
os.path.join(capath, "server.crt"))
config.Config.set(x, "ssl_key_file",
os.path.join(capath, "server.key.open"))
config.Config.set(x, "ssl_ca_cert_file",
os.path.join(capath, "server.chain"))
config.Config.set(x, "ssl", "True")
if auth and ct is not None:
token = protocol.encode_token(ct)
config.Config.set(x, "token", token)
port = str(unused_tcp_port_factory())
config.Config.set("database", "name", database_name)
config.Config.set("database", "host", "localhost")
config.Config.set("database", "port", str(postgres_db.port))
config.Config.set("database", "connection_timeout", str(3))
config.Config.set("config", "state-dir", state_dir)
config.Config.set("config", "log-dir", os.path.join(state_dir, "logs"))
config.Config.set("agent_rest_transport", "port", port)
config.Config.set("compiler_rest_transport", "port", port)
config.Config.set("client_rest_transport", "port", port)
config.Config.set("cmdline_rest_transport", "port", port)
config.Config.set("server", "bind-port", port)
config.Config.set("server", "bind-address", "127.0.0.1")
config.Config.set("config", "executable",
os.path.abspath(inmanta.app.__file__))
config.Config.set("server", "agent-timeout", "2")
config.Config.set("agent", "agent-repair-interval", "0")
config.Config.set("server", "auto-recompile-wait", "0")
ibl = InmantaBootloader()
try:
await ibl.start()
except SliceStartupException as e:
port = config.Config.get("server", "bind-port")
output = subprocess.check_output(["ss", "-antp"])
output = output.decode("utf-8")
logger.debug(f"Port: {port}")
logger.debug(f"Port usage: \n {output}")
raise e
yield ibl.restserver
try:
await asyncio.wait_for(ibl.stop(), 15)
except concurrent.futures.TimeoutError:
logger.exception("Timeout during stop of the server in teardown")
shutil.rmtree(state_dir)
async def run(
self,
force_update: Optional[bool] = False
) -> Tuple[bool, Optional[model.CompileData]]:
"""
Runs this compile run.
:return: Tuple of a boolean representing success and the compile data, if any.
"""
success = False
now = datetime.datetime.now().astimezone()
await self.request.update_fields(started=now)
compile_data_json_file = NamedTemporaryFile()
try:
await self._start_stage("Init", "")
environment_id = self.request.environment
project_dir = self._project_dir
env = await data.Environment.get_by_id(environment_id)
env_string = ", ".join([
f"{k}='{v}'"
for k, v in self.request.environment_variables.items()
])
assert self.stage
await self.stage.update_streams(
out=
f"Using extra environment variables during compile {env_string}\n"
)
if env is None:
await self._error("Environment %s does not exist." %
environment_id)
await self._end_stage(-1)
return False, None
if not os.path.exists(project_dir):
await self._info(
"Creating project directory for environment %s at %s" %
(environment_id, project_dir))
os.mkdir(project_dir)
# Use a separate venv to compile the project to prevent that packages are installed in the
# venv of the Inmanta server.
venv_dir = os.path.join(project_dir, ".env")
async def ensure_venv() -> Optional[data.Report]:
"""
Ensure a venv is present at `venv_dir`.
"""
virtual_env = VirtualEnv(venv_dir)
if virtual_env.exists():
return None
await self._start_stage("Creating venv", command="")
try:
virtual_env.init_env()
except VenvCreationFailedError as e:
await self._error(message=e.msg)
return await self._end_stage(returncode=1)
else:
return await self._end_stage(returncode=0)
async def update_modules() -> data.Report:
return await run_compile_stage_in_venv(
"Updating modules", ["-vvv", "-X", "modules", "update"],
cwd=project_dir)
async def install_modules() -> data.Report:
return await run_compile_stage_in_venv(
"Installing modules", ["-vvv", "-X", "project", "install"],
cwd=project_dir)
async def run_compile_stage_in_venv(
stage_name: str,
inmanta_args: List[str],
cwd: str,
env: Dict[str, str] = {}) -> data.Report:
"""
Run a compile stage by executing the given command in the venv `venv_dir`.
:param stage_name: Name of the compile stage.
:param inmanta_args: The command to be executed in the venv. This command should not include the part
["<python-interpreter>", "-m", "inmanta.app"]
:param cwd: The current working directory to be used for the command invocation.
:param env: Execute the command with these environment variables.
"""
LOGGER.info(stage_name)
python_path = PythonEnvironment.get_python_path_for_env_path(
venv_dir)
assert os.path.exists(python_path)
full_cmd = [python_path, "-m", "inmanta.app"] + inmanta_args
return await self._run_compile_stage(stage_name, full_cmd, cwd,
env)
async def setup(
) -> AsyncIterator[Awaitable[Optional[data.Report]]]:
"""
Returns an iterator over all setup stages. Inspecting stage success state is the responsibility of the caller.
"""
repo_url: str = env.repo_url
repo_branch: str = env.repo_branch
if os.path.exists(os.path.join(project_dir, "project.yml")):
yield self._end_stage(0)
yield ensure_venv()
should_update: bool = force_update or self.request.force_update
# switch branches
if repo_branch:
branch = await self.get_branch()
if branch is not None and repo_branch != branch:
if should_update:
yield self._run_compile_stage(
"Fetching new branch heads",
["git", "fetch"], project_dir)
yield self._run_compile_stage(
f"Switching branch from {branch} to {repo_branch}",
["git", "checkout", repo_branch],
project_dir,
)
if not should_update:
# if we update, update procedure will install modules
yield install_modules()
# update project
if should_update:
# only pull changes if there is an upstream branch
if await self.get_upstream_branch():
yield self._run_compile_stage(
"Pulling updates", ["git", "pull"],
project_dir)
yield update_modules()
else:
if not repo_url:
await self._warning(
f"Failed to compile: no project found in {project_dir} and no repository set."
)
yield self._end_stage(1)
return
if len(os.listdir(project_dir)) > 0:
await self._warning(
f"Failed to compile: no project found in {project_dir} but directory is not empty."
)
yield self._end_stage(1)
return
yield self._end_stage(0)
# clone repo and install project
cmd = ["git", "clone", repo_url, "."]
if repo_branch:
cmd.extend(["-b", repo_branch])
yield self._run_compile_stage("Cloning repository", cmd,
project_dir)
yield ensure_venv()
yield install_modules()
async for stage in setup():
stage_result: Optional[data.Report] = await stage
if stage_result and (stage_result.returncode is None
or stage_result.returncode > 0):
return False, None
server_address = opt.server_address.get()
server_port = opt.get_bind_port()
cmd = [
"-vvv",
"export",
"-X",
"-e",
str(environment_id),
"--server_address",
server_address,
"--server_port",
str(server_port),
"--metadata",
json.dumps(self.request.metadata),
"--export-compile-data",
"--export-compile-data-file",
compile_data_json_file.name,
]
if not self.request.do_export:
f = NamedTemporaryFile()
cmd.append("-j")
cmd.append(f.name)
if config.Config.get("server", "auth", False):
token = encode_token(["compiler", "api"], str(environment_id))
cmd.append("--token")
cmd.append(token)
if opt.server_ssl_cert.get() is not None:
cmd.append("--ssl")
if opt.server_ssl_ca_cert.get() is not None:
cmd.append("--ssl-ca-cert")
cmd.append(opt.server_ssl_ca_cert.get())
self.tail_stdout = ""
env_vars_compile: Dict[str, str] = os.environ.copy()
env_vars_compile.update(self.request.environment_variables)
result: data.Report = await run_compile_stage_in_venv(
"Recompiling configuration model",
cmd,
cwd=project_dir,
env=env_vars_compile)
success = result.returncode == 0
if not success:
if self.request.do_export:
LOGGER.warning("Compile %s failed", self.request.id)
else:
LOGGER.debug("Compile %s failed", self.request.id)
print("---", self.tail_stdout, result.errstream)
match = re.search(r"Committed resources with version (\d+)",
self.tail_stdout)
if match:
self.version = int(match.group(1))
except CancelledError:
# This compile was cancelled. Catch it here otherwise a warning will be printed in the logs because of an
# unhandled exception in a backgrounded coroutine.
pass
except Exception:
LOGGER.exception("An error occurred while recompiling")
finally:
async def warn(message: str) -> None:
if self.stage is not None:
await self._warning(message)
else:
LOGGER.warning(message)
with compile_data_json_file as file:
compile_data_json: str = file.read().decode()
if compile_data_json:
try:
return success, model.CompileData.parse_raw(
compile_data_json)
except json.JSONDecodeError:
await warn(
"Failed to load compile data json for compile %s. Invalid json: '%s'"
% (self.request.id, compile_data_json))
except pydantic.ValidationError:
await warn(
"Failed to parse compile data for compile %s. Json does not match CompileData model: '%s'"
% (self.request.id, compile_data_json))
return success, None
async def run(
self,
force_update: Optional[bool] = False
) -> Tuple[bool, Optional[model.CompileData]]:
success = False
now = datetime.datetime.now()
await self.request.update_fields(started=now)
compile_data_json_file = NamedTemporaryFile()
try:
await self._start_stage("Init", "")
environment_id = self.request.environment
project_dir = self._project_dir
env = await data.Environment.get_by_id(environment_id)
env_string = ", ".join([
f"{k}='{v}'"
for k, v in self.request.environment_variables.items()
])
await self.stage.update_streams(
out=
f"Using extra environment variables during compile {env_string}\n"
)
if env is None:
await self._error("Environment %s does not exist." %
environment_id)
await self._end_stage(-1)
return False, None
inmanta_path = [sys.executable, "-m", "inmanta.app"]
if not os.path.exists(project_dir):
await self._info(
"Creating project directory for environment %s at %s" %
(environment_id, project_dir))
os.mkdir(project_dir)
repo_url: str = env.repo_url
repo_branch: str = env.repo_branch
if not repo_url:
if not os.path.exists(os.path.join(project_dir,
"project.yml")):
await self._warning(
f"Failed to compile: no project found in {project_dir} and no repository set"
)
await self._end_stage(0)
else:
await self._end_stage(0)
# checkout repo
if not os.path.exists(os.path.join(project_dir, ".git")):
cmd = ["git", "clone", repo_url, "."]
if repo_branch:
cmd.extend(["-b", repo_branch])
result = await self._run_compile_stage(
"Cloning repository", cmd, project_dir)
if result.returncode is None or result.returncode > 0:
return False, None
elif force_update or self.request.force_update:
result = await self._run_compile_stage(
"Fetching changes", ["git", "fetch", repo_url],
project_dir)
if repo_branch:
branch = await self.get_branch()
if branch is not None and repo_branch != branch:
result = await self._run_compile_stage(
f"switching branch from {branch} to {repo_branch}",
["git", "checkout", repo_branch], project_dir)
if force_update or self.request.force_update:
await self._run_compile_stage("Pulling updates",
["git", "pull"], project_dir)
LOGGER.info("Installing and updating modules")
await self._run_compile_stage(
"Updating modules",
inmanta_path + ["modules", "update"], project_dir)
server_address = opt.server_address.get()
server_port = opt.get_bind_port()
cmd = inmanta_path + [
"-vvv",
"export",
"-X",
"-e",
str(environment_id),
"--server_address",
server_address,
"--server_port",
str(server_port),
"--metadata",
json.dumps(self.request.metadata),
"--export-compile-data",
"--export-compile-data-file",
compile_data_json_file.name,
]
if not self.request.do_export:
f = NamedTemporaryFile()
cmd.append("-j")
cmd.append(f.name)
if config.Config.get("server", "auth", False):
token = encode_token(["compiler", "api"], str(environment_id))
cmd.append("--token")
cmd.append(token)
if opt.server_ssl_cert.get() is not None:
cmd.append("--ssl")
if opt.server_ssl_ca_cert.get() is not None:
cmd.append("--ssl-ca-cert")
cmd.append(opt.server_ssl_ca_cert.get())
self.tail_stdout = ""
env_vars_compile: Dict[str, str] = os.environ.copy()
env_vars_compile.update(self.request.environment_variables)
result = await self._run_compile_stage(
"Recompiling configuration model",
cmd,
project_dir,
env=env_vars_compile)
success = result.returncode == 0
if not success:
if self.request.do_export:
LOGGER.warning("Compile %s failed", self.request.id)
else:
LOGGER.debug("Compile %s failed", self.request.id)
print("---", self.tail_stdout, result.errstream)
match = re.search(r"Committed resources with version (\d+)",
self.tail_stdout)
if match:
self.version = int(match.group(1))
except CancelledError:
# This compile was cancelled. Catch it here otherwise a warning will be printed in the logs because of an
# unhandled exception in a backgrounded coroutine.
pass
except Exception:
LOGGER.exception("An error occured while recompiling")
finally:
with compile_data_json_file as file:
compile_data_json: str = file.read().decode()
if compile_data_json:
try:
return success, model.CompileData.parse_raw(
compile_data_json)
except json.JSONDecodeError:
LOGGER.warning(
"Failed to load compile data json for compile %s. Invalid json: '%s'",
(self.request.id, compile_data_json),
)
except pydantic.ValidationError:
LOGGER.warning(
"Failed to parse compile data for compile %s. Json does not match CompileData model: '%s'",
(self.request.id, compile_data_json),
)
return success, None
