async def environment_create_token(self, env: data.Environment, client_types: List[str], idempotent: bool) -> str: """ Create a new auth token for this environment """ return encode_token(client_types, str(env.id), idempotent)
def bootstrap_token(client: Client) -> None: """ Generate a bootstrap token that provides access to everything. This token is only valid for 3600 seconds. """ click.echo( "Token: " + protocol.encode_token(["api", "compiler", "agent"], expire=3600))
def _make_agent_config(self, env: data.Environment, agent_names: list, agent_map: dict) -> str: """ Generate the config file for the process that hosts the autostarted agents :param env: The environment for which to autostart agents :param agent_names: The names of the agents :param agent_map: The agent mapping to use :return: A string that contains the config file content. """ environment_id = str(env.id) port = Config.get("server_rest_transport", "port", "8888") privatestatedir = os.path.join( Config.get("config", "state-dir", "/var/lib/inmanta"), environment_id) agent_splay = yield env.get(data.AUTOSTART_SPLAY) # generate config file config = """[config] heartbeat-interval = 60 state-dir=%(statedir)s agent-names = %(agents)s environment=%(env_id)s agent-map=%(agent_map)s agent_splay=%(agent_splay)d [agent_rest_transport] port=%(port)s host=localhost """ % { "agents": ",".join(agent_names), "env_id": environment_id, "port": port, "agent_map": ",".join(["%s=%s" % (k, v) for (k, v) in agent_map.items()]), "statedir": privatestatedir, "agent_splay": agent_splay } if server_config.server_enable_auth: token = protocol.encode_token(["agent"], environment_id) config += """ token=%s """ % (token) ssl_cert = Config.get("server", "ssl_key_file", None) ssl_ca = Config.get("server", "ssl_cert_file", None) if ssl_ca is not None and ssl_cert is not None: config += """ ssl=True ssl_ca_cert_file=%s """ % (ssl_ca) return config
def test_jwt_create(inmanta_config): """ Test creating, signing and verifying JWT with HS256 from the configuration """ jot = protocol.encode_token(["api"]) payload = protocol.decode_token(jot) assert "api" in payload["urn:inmanta:ct"] # test creating an idempotent token jot1 = protocol.encode_token(["agent"], idempotent=True) jot3 = protocol.encode_token(["agent"]) time.sleep(1) jot2 = protocol.encode_token(["agent"], idempotent=True) jot4 = protocol.encode_token(["agent"]) assert jot1 == jot2 assert jot3 != jot4 assert jot1 != jot3 assert jot2 != jot3
def server_multi(inmanta_config, io_loop, mongo_db, mongo_client, request): from inmanta.server import Server state_dir = tempfile.mkdtemp() ssl, auth = request.param path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") if auth: config.Config.set("server", "auth", "true") from inmanta import protocol for x, ct in [("server", None), ("server_rest_transport", None), ("agent_rest_transport", ["agent"]), ("compiler_rest_transport", ["compiler"]), ("client_rest_transport", ["api", "compiler"]), ("cmdline_rest_transport", ["api"])]: if ssl: config.Config.set(x, "ssl_cert_file", os.path.join(path, "server.crt")) config.Config.set(x, "ssl_key_file", os.path.join(path, "server.open.key")) config.Config.set(x, "ssl_ca_cert_file", os.path.join(path, "server.crt")) config.Config.set(x, "ssl", "True") if auth and ct is not None: token = protocol.encode_token(ct) config.Config.set(x, "token", token) port = get_free_tcp_port() config.Config.get( "database", "name", "inmanta-" + ''.join(random.choice(string.ascii_letters) for _ in range(10))) config.Config.set("config", "state-dir", state_dir) config.Config.set("config", "log-dir", os.path.join(state_dir, "logs")) config.Config.set("server_rest_transport", "port", port) config.Config.set("agent_rest_transport", "port", port) config.Config.set("compiler_rest_transport", "port", port) config.Config.set("client_rest_transport", "port", port) config.Config.set("cmdline_rest_transport", "port", port) config.Config.set( "config", "executable", os.path.abspath(os.path.join(__file__, "../../src/inmanta/app.py"))) config.Config.set("server", "agent-timeout", "2") server = Server(database_host="localhost", database_port=int(mongo_db.port), io_loop=io_loop) server.start() yield server server.stop() shutil.rmtree(state_dir)
async def server_multi(server_pre_start, event_loop, inmanta_config, postgres_db, database_name, request, clean_reset, unused_tcp_port_factory): """ :param event_loop: explicitly include event_loop to make sure event loop started before and closed after this fixture. May not be required """ state_dir = tempfile.mkdtemp() ssl, auth, ca = request.param path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") if auth: config.Config.set("server", "auth", "true") for x, ct in [ ("server", None), ("agent_rest_transport", ["agent"]), ("compiler_rest_transport", ["compiler"]), ("client_rest_transport", ["api", "compiler"]), ("cmdline_rest_transport", ["api"]), ]: if ssl and not ca: config.Config.set(x, "ssl_cert_file", os.path.join(path, "server.crt")) config.Config.set(x, "ssl_key_file", os.path.join(path, "server.open.key")) config.Config.set(x, "ssl_ca_cert_file", os.path.join(path, "server.crt")) config.Config.set(x, "ssl", "True") if ssl and ca: capath = os.path.join(path, "ca", "enduser-certs") config.Config.set(x, "ssl_cert_file", os.path.join(capath, "server.crt")) config.Config.set(x, "ssl_key_file", os.path.join(capath, "server.key.open")) config.Config.set(x, "ssl_ca_cert_file", os.path.join(capath, "server.chain")) config.Config.set(x, "ssl", "True") if auth and ct is not None: token = protocol.encode_token(ct) config.Config.set(x, "token", token) port = str(unused_tcp_port_factory()) config.Config.set("database", "name", database_name) config.Config.set("database", "host", "localhost") config.Config.set("database", "port", str(postgres_db.port)) config.Config.set("database", "connection_timeout", str(3)) config.Config.set("config", "state-dir", state_dir) config.Config.set("config", "log-dir", os.path.join(state_dir, "logs")) config.Config.set("agent_rest_transport", "port", port) config.Config.set("compiler_rest_transport", "port", port) config.Config.set("client_rest_transport", "port", port) config.Config.set("cmdline_rest_transport", "port", port) config.Config.set("server", "bind-port", port) config.Config.set("server", "bind-address", "127.0.0.1") config.Config.set("config", "executable", os.path.abspath(inmanta.app.__file__)) config.Config.set("server", "agent-timeout", "2") config.Config.set("agent", "agent-repair-interval", "0") config.Config.set("server", "auto-recompile-wait", "0") ibl = InmantaBootloader() try: await ibl.start() except SliceStartupException as e: port = config.Config.get("server", "bind-port") output = subprocess.check_output(["ss", "-antp"]) output = output.decode("utf-8") logger.debug(f"Port: {port}") logger.debug(f"Port usage: \n {output}") raise e yield ibl.restserver try: await asyncio.wait_for(ibl.stop(), 15) except concurrent.futures.TimeoutError: logger.exception("Timeout during stop of the server in teardown") shutil.rmtree(state_dir)
async def run( self, force_update: Optional[bool] = False ) -> Tuple[bool, Optional[model.CompileData]]: """ Runs this compile run. :return: Tuple of a boolean representing success and the compile data, if any. """ success = False now = datetime.datetime.now().astimezone() await self.request.update_fields(started=now) compile_data_json_file = NamedTemporaryFile() try: await self._start_stage("Init", "") environment_id = self.request.environment project_dir = self._project_dir env = await data.Environment.get_by_id(environment_id) env_string = ", ".join([ f"{k}='{v}'" for k, v in self.request.environment_variables.items() ]) assert self.stage await self.stage.update_streams( out= f"Using extra environment variables during compile {env_string}\n" ) if env is None: await self._error("Environment %s does not exist." % environment_id) await self._end_stage(-1) return False, None if not os.path.exists(project_dir): await self._info( "Creating project directory for environment %s at %s" % (environment_id, project_dir)) os.mkdir(project_dir) # Use a separate venv to compile the project to prevent that packages are installed in the # venv of the Inmanta server. venv_dir = os.path.join(project_dir, ".env") async def ensure_venv() -> Optional[data.Report]: """ Ensure a venv is present at `venv_dir`. """ virtual_env = VirtualEnv(venv_dir) if virtual_env.exists(): return None await self._start_stage("Creating venv", command="") try: virtual_env.init_env() except VenvCreationFailedError as e: await self._error(message=e.msg) return await self._end_stage(returncode=1) else: return await self._end_stage(returncode=0) async def update_modules() -> data.Report: return await run_compile_stage_in_venv( "Updating modules", ["-vvv", "-X", "modules", "update"], cwd=project_dir) async def install_modules() -> data.Report: return await run_compile_stage_in_venv( "Installing modules", ["-vvv", "-X", "project", "install"], cwd=project_dir) async def run_compile_stage_in_venv( stage_name: str, inmanta_args: List[str], cwd: str, env: Dict[str, str] = {}) -> data.Report: """ Run a compile stage by executing the given command in the venv `venv_dir`. :param stage_name: Name of the compile stage. :param inmanta_args: The command to be executed in the venv. This command should not include the part ["<python-interpreter>", "-m", "inmanta.app"] :param cwd: The current working directory to be used for the command invocation. :param env: Execute the command with these environment variables. """ LOGGER.info(stage_name) python_path = PythonEnvironment.get_python_path_for_env_path( venv_dir) assert os.path.exists(python_path) full_cmd = [python_path, "-m", "inmanta.app"] + inmanta_args return await self._run_compile_stage(stage_name, full_cmd, cwd, env) async def setup( ) -> AsyncIterator[Awaitable[Optional[data.Report]]]: """ Returns an iterator over all setup stages. Inspecting stage success state is the responsibility of the caller. """ repo_url: str = env.repo_url repo_branch: str = env.repo_branch if os.path.exists(os.path.join(project_dir, "project.yml")): yield self._end_stage(0) yield ensure_venv() should_update: bool = force_update or self.request.force_update # switch branches if repo_branch: branch = await self.get_branch() if branch is not None and repo_branch != branch: if should_update: yield self._run_compile_stage( "Fetching new branch heads", ["git", "fetch"], project_dir) yield self._run_compile_stage( f"Switching branch from {branch} to {repo_branch}", ["git", "checkout", repo_branch], project_dir, ) if not should_update: # if we update, update procedure will install modules yield install_modules() # update project if should_update: # only pull changes if there is an upstream branch if await self.get_upstream_branch(): yield self._run_compile_stage( "Pulling updates", ["git", "pull"], project_dir) yield update_modules() else: if not repo_url: await self._warning( f"Failed to compile: no project found in {project_dir} and no repository set." ) yield self._end_stage(1) return if len(os.listdir(project_dir)) > 0: await self._warning( f"Failed to compile: no project found in {project_dir} but directory is not empty." ) yield self._end_stage(1) return yield self._end_stage(0) # clone repo and install project cmd = ["git", "clone", repo_url, "."] if repo_branch: cmd.extend(["-b", repo_branch]) yield self._run_compile_stage("Cloning repository", cmd, project_dir) yield ensure_venv() yield install_modules() async for stage in setup(): stage_result: Optional[data.Report] = await stage if stage_result and (stage_result.returncode is None or stage_result.returncode > 0): return False, None server_address = opt.server_address.get() server_port = opt.get_bind_port() cmd = [ "-vvv", "export", "-X", "-e", str(environment_id), "--server_address", server_address, "--server_port", str(server_port), "--metadata", json.dumps(self.request.metadata), "--export-compile-data", "--export-compile-data-file", compile_data_json_file.name, ] if not self.request.do_export: f = NamedTemporaryFile() cmd.append("-j") cmd.append(f.name) if config.Config.get("server", "auth", False): token = encode_token(["compiler", "api"], str(environment_id)) cmd.append("--token") cmd.append(token) if opt.server_ssl_cert.get() is not None: cmd.append("--ssl") if opt.server_ssl_ca_cert.get() is not None: cmd.append("--ssl-ca-cert") cmd.append(opt.server_ssl_ca_cert.get()) self.tail_stdout = "" env_vars_compile: Dict[str, str] = os.environ.copy() env_vars_compile.update(self.request.environment_variables) result: data.Report = await run_compile_stage_in_venv( "Recompiling configuration model", cmd, cwd=project_dir, env=env_vars_compile) success = result.returncode == 0 if not success: if self.request.do_export: LOGGER.warning("Compile %s failed", self.request.id) else: LOGGER.debug("Compile %s failed", self.request.id) print("---", self.tail_stdout, result.errstream) match = re.search(r"Committed resources with version (\d+)", self.tail_stdout) if match: self.version = int(match.group(1)) except CancelledError: # This compile was cancelled. Catch it here otherwise a warning will be printed in the logs because of an # unhandled exception in a backgrounded coroutine. pass except Exception: LOGGER.exception("An error occurred while recompiling") finally: async def warn(message: str) -> None: if self.stage is not None: await self._warning(message) else: LOGGER.warning(message) with compile_data_json_file as file: compile_data_json: str = file.read().decode() if compile_data_json: try: return success, model.CompileData.parse_raw( compile_data_json) except json.JSONDecodeError: await warn( "Failed to load compile data json for compile %s. Invalid json: '%s'" % (self.request.id, compile_data_json)) except pydantic.ValidationError: await warn( "Failed to parse compile data for compile %s. Json does not match CompileData model: '%s'" % (self.request.id, compile_data_json)) return success, None
async def run( self, force_update: Optional[bool] = False ) -> Tuple[bool, Optional[model.CompileData]]: success = False now = datetime.datetime.now() await self.request.update_fields(started=now) compile_data_json_file = NamedTemporaryFile() try: await self._start_stage("Init", "") environment_id = self.request.environment project_dir = self._project_dir env = await data.Environment.get_by_id(environment_id) env_string = ", ".join([ f"{k}='{v}'" for k, v in self.request.environment_variables.items() ]) await self.stage.update_streams( out= f"Using extra environment variables during compile {env_string}\n" ) if env is None: await self._error("Environment %s does not exist." % environment_id) await self._end_stage(-1) return False, None inmanta_path = [sys.executable, "-m", "inmanta.app"] if not os.path.exists(project_dir): await self._info( "Creating project directory for environment %s at %s" % (environment_id, project_dir)) os.mkdir(project_dir) repo_url: str = env.repo_url repo_branch: str = env.repo_branch if not repo_url: if not os.path.exists(os.path.join(project_dir, "project.yml")): await self._warning( f"Failed to compile: no project found in {project_dir} and no repository set" ) await self._end_stage(0) else: await self._end_stage(0) # checkout repo if not os.path.exists(os.path.join(project_dir, ".git")): cmd = ["git", "clone", repo_url, "."] if repo_branch: cmd.extend(["-b", repo_branch]) result = await self._run_compile_stage( "Cloning repository", cmd, project_dir) if result.returncode is None or result.returncode > 0: return False, None elif force_update or self.request.force_update: result = await self._run_compile_stage( "Fetching changes", ["git", "fetch", repo_url], project_dir) if repo_branch: branch = await self.get_branch() if branch is not None and repo_branch != branch: result = await self._run_compile_stage( f"switching branch from {branch} to {repo_branch}", ["git", "checkout", repo_branch], project_dir) if force_update or self.request.force_update: await self._run_compile_stage("Pulling updates", ["git", "pull"], project_dir) LOGGER.info("Installing and updating modules") await self._run_compile_stage( "Updating modules", inmanta_path + ["modules", "update"], project_dir) server_address = opt.server_address.get() server_port = opt.get_bind_port() cmd = inmanta_path + [ "-vvv", "export", "-X", "-e", str(environment_id), "--server_address", server_address, "--server_port", str(server_port), "--metadata", json.dumps(self.request.metadata), "--export-compile-data", "--export-compile-data-file", compile_data_json_file.name, ] if not self.request.do_export: f = NamedTemporaryFile() cmd.append("-j") cmd.append(f.name) if config.Config.get("server", "auth", False): token = encode_token(["compiler", "api"], str(environment_id)) cmd.append("--token") cmd.append(token) if opt.server_ssl_cert.get() is not None: cmd.append("--ssl") if opt.server_ssl_ca_cert.get() is not None: cmd.append("--ssl-ca-cert") cmd.append(opt.server_ssl_ca_cert.get()) self.tail_stdout = "" env_vars_compile: Dict[str, str] = os.environ.copy() env_vars_compile.update(self.request.environment_variables) result = await self._run_compile_stage( "Recompiling configuration model", cmd, project_dir, env=env_vars_compile) success = result.returncode == 0 if not success: if self.request.do_export: LOGGER.warning("Compile %s failed", self.request.id) else: LOGGER.debug("Compile %s failed", self.request.id) print("---", self.tail_stdout, result.errstream) match = re.search(r"Committed resources with version (\d+)", self.tail_stdout) if match: self.version = int(match.group(1)) except CancelledError: # This compile was cancelled. Catch it here otherwise a warning will be printed in the logs because of an # unhandled exception in a backgrounded coroutine. pass except Exception: LOGGER.exception("An error occured while recompiling") finally: with compile_data_json_file as file: compile_data_json: str = file.read().decode() if compile_data_json: try: return success, model.CompileData.parse_raw( compile_data_json) except json.JSONDecodeError: LOGGER.warning( "Failed to load compile data json for compile %s. Invalid json: '%s'", (self.request.id, compile_data_json), ) except pydantic.ValidationError: LOGGER.warning( "Failed to parse compile data for compile %s. Json does not match CompileData model: '%s'", (self.request.id, compile_data_json), ) return success, None