async def test_export_without_environment(tmpdir, server, client, push_method):
server_port = Config.get("client_rest_transport", "port")
server_host = Config.get("client_rest_transport", "host", "localhost")
result = await client.create_project("test")
assert result.code == 200
proj_id = result.result["project"]["id"]
result = await client.create_environment(proj_id, "test", None, None)
assert result.code == 200
env_id = result.result["environment"]["id"]
workspace = tmpdir.mkdir("tmp")
path_main_file = workspace.join("main.cf")
path_project_yml_file = workspace.join("project.yml")
libs_dir = workspace.join("libs")
path_project_yml_file.write(
f"""
name: testproject
modulepath: {libs_dir}
downloadpath: {libs_dir}
repo: https://github.com/inmanta/
"""
)
path_main_file.write(
"""
vm1=std::Host(name="non-existing-machine", os=std::linux)
std::ConfigFile(host=vm1, path="/test", content="")
"""
)
os.chdir(workspace)
args = [
sys.executable,
"-m",
"inmanta.app",
"export",
]
args.extend(["--server_port", str(server_port)])
args.extend(["--server_address", str(server_host)])
args += push_method
process = await subprocess.create_subprocess_exec(*args, stdout=sys.stdout, stderr=sys.stderr)
try:
await asyncio.wait_for(process.communicate(), timeout=30)
except asyncio.TimeoutError as e:
process.kill()
await process.communicate()
raise e
# Make sure exitcode is one
assert process.returncode == 1, f"Process ended with bad return code, got {process.returncode} (expected 1)"
result = await client.list_versions(env_id)
assert result.code == 200
assert len(result.result["versions"]) == 0
shutil.rmtree(workspace)
def _make_agent_config(self, env: data.Environment, agent_names: list,
agent_map: dict) -> str:
"""
Generate the config file for the process that hosts the autostarted agents
:param env: The environment for which to autostart agents
:param agent_names: The names of the agents
:param agent_map: The agent mapping to use
:return: A string that contains the config file content.
"""
environment_id = str(env.id)
port = Config.get("server_rest_transport", "port", "8888")
privatestatedir = os.path.join(
Config.get("config", "state-dir", "/var/lib/inmanta"),
environment_id)
agent_splay = yield env.get(data.AUTOSTART_SPLAY)
# generate config file
config = """[config]
heartbeat-interval = 60
state-dir=%(statedir)s
agent-names = %(agents)s
environment=%(env_id)s
agent-map=%(agent_map)s
agent_splay=%(agent_splay)d
[agent_rest_transport]
port=%(port)s
host=localhost
""" % {
"agents":
",".join(agent_names),
"env_id":
environment_id,
"port":
port,
"agent_map":
",".join(["%s=%s" % (k, v) for (k, v) in agent_map.items()]),
"statedir":
privatestatedir,
"agent_splay":
agent_splay
}
if server_config.server_enable_auth:
token = protocol.encode_token(["agent"], environment_id)
config += """
token=%s
""" % (token)
ssl_cert = Config.get("server", "ssl_key_file", None)
ssl_ca = Config.get("server", "ssl_cert_file", None)
if ssl_ca is not None and ssl_cert is not None:
config += """
ssl=True
ssl_ca_cert_file=%s
""" % (ssl_ca)
return config
def server_token(context: Context,
client_types: "string[]" = ["agent"]) -> "string":
token = Config.get("compiler_rest_transport", "token", "")
if token == "":
return ""
# Request a new token for this agent
token = ""
try:
client = context.get_client()
env = Config.get("config", "environment", None)
if env is None:
raise Exception(
"The environment of this model should be configured in config>environment"
)
def call():
return client.create_token(tid=env,
client_types=list(client_types),
idempotent=True)
result = context.run_sync(call)
if result.code == 200:
token = result.result["token"]
else:
logging.getLogger(__name__).warning("Unable to get a new token")
raise Exception("Unable to get a valid token")
except ConnectionRefusedError:
logging.getLogger(__name__).exception("Unable to get a new token")
raise Exception("Unable to get a valid token")
return token
def get_environment_id(self) -> str:
env = str(Config.get("config", "environment", None))
if env is None:
raise Exception(
"The environment of the model should be configured in config>environment"
)
return env
def environment() -> "string":
"""
Return the environment id
"""
env = str(Config.get("config", "environment", None))
if env is None:
raise Exception("The environment of this model should be configured in config>environment")
return str(env)
def getfact(context: Context, resource: "any", fact_name: "string", default_value: "any"=None) -> "any":
"""
Retrieve a fact of the given resource
"""
resource_id = resources.to_id(resource)
if resource_id is None:
raise Exception("Facts can only be retreived from resources.")
# Special case for unit testing and mocking
if hasattr(context.compiler, "refs") and "facts" in context.compiler.refs:
if resource_id in context.compiler.refs["facts"] and fact_name in context.compiler.refs["facts"][resource_id]:
return context.compiler.refs["facts"][resource_id][fact_name]
fact_value = Unknown(source=resource)
unknown_parameters.append({"resource": resource_id, "parameter": fact_name, "source": "fact"})
if default_value is not None:
return default_value
return fact_value
# End special case
fact_value = None
try:
client = context.get_client()
env = Config.get("config", "environment", None)
if env is None:
raise Exception("The environment of this model should be configured in config>environment")
def call():
return client.get_param(tid=env, id=fact_name, resource_id=resource_id)
result = context.run_sync(call)
if result.code == 200:
fact_value = result.result["parameter"]["value"]
else:
logging.getLogger(__name__).info("Param %s of resource %s is unknown", fact_name, resource_id)
fact_value = Unknown(source=resource)
unknown_parameters.append({"resource": resource_id, "parameter": fact_name, "source": "fact"})
except ConnectionRefusedError:
logging.getLogger(__name__).warning("Param %s of resource %s is unknown because connection to server was refused",
fact_name, resource_id)
fact_value = Unknown(source=resource)
unknown_parameters.append({"resource": resource_id, "parameter": fact_name, "source": "fact"})
if isinstance(fact_value, Unknown) and default_value is not None:
return default_value
return fact_value
def server_ca() -> "string":
filename = Config.get("compiler_rest_transport", "ssl_ca_cert_file", None)
if not filename:
return ""
if not os.path.isfile(filename):
raise Exception("%s isn't a valid file" % filename)
file_fd = open(filename, "r")
if file_fd is None:
raise Exception("Unable to open file %s" % filename)
content = file_fd.read()
return content
def get_bind_port() -> int:
if Config.is_set("server", "bind-port") or Config.is_set(
"server", "bind-address"):
# Use new bind-port option
if Config.is_set("server_rest_transport", "port"):
warnings.warn(
"Ignoring the server_rest_transport.port config option since the new config options "
"server.bind-port/server.bind-address are used.",
category=DeprecationWarning,
)
return server_bind_port.get()
else:
# Fallback to old option
warnings.warn(
"The server_rest_transport.port config option is deprecated in favour of the server.bind-port option.",
category=DeprecationWarning,
)
return Config.get("server_rest_transport", "port", 8888)
def app() -> None:
"""
Run the compiler
"""
# Send logs to stdout
stream_handler = _get_default_stream_handler()
logging.root.handlers = []
logging.root.addHandler(stream_handler)
logging.root.setLevel(0)
# do an initial load of known config files to build the libdir path
Config.load_config()
parser = cmd_parser()
options, other = parser.parse_known_args()
options.other = other
# Log everything to a log_file if logfile is provided
if options.log_file:
watched_file_handler = _get_watched_file_handler(options)
logging.root.addHandler(watched_file_handler)
logging.root.removeHandler(stream_handler)
else:
if options.timed:
formatter = _get_log_formatter_for_stream_handler(timed=True)
stream_handler.setFormatter(formatter)
log_level = _convert_to_log_level(options.verbose)
stream_handler.setLevel(log_level)
logging.captureWarnings(True)
if options.config_file and not os.path.exists(options.config_file):
LOGGER.warning("Config file %s doesn't exist", options.config_file)
# Load the configuration
Config.load_config(options.config_file, options.config_dir)
if options.inmanta_version:
print_versions_installed_components_and_exit()
if options.warnings is not None:
Config.set("warnings", "default", options.warnings)
config = Config.get()
assert isinstance(config, ConfigParser)
WarningsManager.apply_config(config["warnings"] if "warnings" in config else None)
# start the command
if not hasattr(options, "func"):
# show help
parser.print_usage()
return
def report(e: BaseException) -> None:
minus_x_set_top_level_command = options.errors
minus_x_set_subcommand = hasattr(options, "errors_subcommand") and options.errors_subcommand
if not minus_x_set_top_level_command and not minus_x_set_subcommand:
if isinstance(e, CompilerException):
print(e.format_trace(indent=" "), file=sys.stderr)
else:
print(str(e), file=sys.stderr)
else:
sys.excepthook(*sys.exc_info())
if isinstance(e, CompilerException):
from inmanta.compiler.help.explainer import ExplainerFactory
helpmsg = ExplainerFactory().explain_and_format(e, plain=not _is_on_tty())
if helpmsg is not None:
print(helpmsg)
try:
options.func(options)
except ShowUsageException as e:
print(e.args[0], file=sys.stderr)
parser.print_usage()
except CLIException as e:
report(e)
sys.exit(e.exitcode)
except Exception as e:
report(e)
sys.exit(1)
except KeyboardInterrupt as e:
report(e)
sys.exit(1)
sys.exit(0)
async def test_export_with_specific_export_plugin(tmpvenv_active_inherit: env.VirtualEnv, tmpdir, client):
server_port = Config.get("client_rest_transport", "port")
server_host = Config.get("client_rest_transport", "host", "localhost")
result = await client.create_project("test")
assert result.code == 200
proj_id = result.result["project"]["id"]
result = await client.create_environment(proj_id, "test", None, None)
assert result.code == 200
env_id = result.result["environment"]["id"]
workspace = tmpdir.mkdir("tmp")
libs_dir = workspace.join("libs")
# project.yml
path_project_yml_file = workspace.join("project.yml")
path_project_yml_file.write(
f"""
name: testproject
modulepath: {libs_dir}
downloadpath: {libs_dir}
repo: https://github.com/inmanta/
"""
)
# main.cf
path_main_file = workspace.join("main.cf")
path_main_file.write("import test")
# test module
module_dir = libs_dir.join("test")
os.makedirs(module_dir)
# Module.yml
module_yml_file = module_dir.join("module.yml")
module_yml_file.write(
"""
name: test
license: test
version: 1.0.0
"""
)
# .inmanta
dot_inmanta_cfg_file = workspace.join(".inmanta")
dot_inmanta_cfg_file.write(
"""
[config]
export=other_exporter
"""
)
# plugin/__init__.py
plugins_dir = module_dir.join("plugins")
os.makedirs(plugins_dir)
init_file = plugins_dir.join("__init__.py")
init_file.write(
"""
from inmanta.export import export, Exporter
@export("test_exporter")
def test_exporter(exporter: Exporter) -> None:
print("test_exporter ran")
@export("other_exporter")
def other_exporter(exporter: Exporter) -> None:
print("other_exporter ran")
"""
)
# model/_init.cf
model_dir = module_dir.join("model")
os.makedirs(model_dir)
init_cf_file = model_dir.join("_init.cf")
init_cf_file.write("")
await install_project(tmpvenv_active_inherit, workspace)
os.chdir(workspace)
args = [
tmpvenv_active_inherit.python_path,
"-m",
"inmanta.app",
"export",
"--export-plugin",
"test_exporter",
"-e",
str(env_id),
"--server_port",
str(server_port),
"--server_address",
str(server_host),
]
process = await subprocess.create_subprocess_exec(*args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
(stdout, stderr) = await asyncio.wait_for(process.communicate(), timeout=30)
except asyncio.TimeoutError as e:
process.kill()
await process.communicate()
raise e
# Make sure exitcode is zero
assert process.returncode == 0
assert "test_exporter ran" in stdout.decode("utf-8")
assert "other_exporter" not in stdout.decode("utf-8")
# ## Failure
path_main_file.write(
"""import test
vm1=std::Host(name="non-existing-machine", os=std::linux)
vm1.name = "other"
"""
)
process = await subprocess.create_subprocess_exec(*args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
(stdout, stderr) = await asyncio.wait_for(process.communicate(), timeout=30)
except asyncio.TimeoutError as e:
process.kill()
await process.communicate()
raise e
# Make sure exitcode is one
assert process.returncode == 1
assert "test_exporter ran" not in stdout.decode("utf-8")
assert "other_exporter" not in stdout.decode("utf-8")
shutil.rmtree(workspace)
async def test_export(tmpvenv_active_inherit: env.VirtualEnv, tmpdir, server, client, push_method, set_server, set_port):
server_port = Config.get("client_rest_transport", "port")
server_host = Config.get("client_rest_transport", "host", "localhost")
result = await client.create_project("test")
assert result.code == 200
proj_id = result.result["project"]["id"]
result = await client.create_environment(proj_id, "test", None, None)
assert result.code == 200
env_id = result.result["environment"]["id"]
workspace = tmpdir.mkdir("tmp")
path_main_file = workspace.join("main.cf")
path_project_yml_file = workspace.join("project.yml")
path_config_file = workspace.join(".inmanta")
libs_dir = workspace.join("libs")
path_project_yml_file.write(
f"""
name: testproject
modulepath: {libs_dir}
downloadpath: {libs_dir}
repo: https://github.com/inmanta/
"""
)
path_main_file.write(
"""
vm1=std::Host(name="non-existing-machine", os=std::linux)
std::ConfigFile(host=vm1, path="/test", content="")
"""
)
path_config_file.write(
f"""
[compiler_rest_transport]
{'host=' + server_host if not set_server else ''}
{'port=' + str(server_port) if not set_port else ''}
[cmdline_rest_transport]
{'host=' + server_host if not set_server else ''}
{'port=' + str(server_port) if not set_port else ''}
"""
)
await install_project(tmpvenv_active_inherit, workspace)
os.chdir(workspace)
args = [
sys.executable,
"-m",
"inmanta.app",
"export",
"-e",
str(env_id),
]
if set_port:
args.extend(["--server_port", str(server_port)])
if set_server:
args.extend(["--server_address", str(server_host)])
args += push_method
process = await subprocess.create_subprocess_exec(*args, stdout=sys.stdout, stderr=sys.stderr)
try:
await asyncio.wait_for(process.communicate(), timeout=30)
except asyncio.TimeoutError as e:
process.kill()
await process.communicate()
raise e
# Make sure exitcode is zero
assert process.returncode == 0, f"Process ended with bad return code, got {process.returncode} (expected 0)"
result = await client.list_versions(env_id)
assert result.code == 200
assert len(result.result["versions"]) == 1
details_exported_version = result.result["versions"][0]
assert details_exported_version["result"] == VersionState.deploying.name
shutil.rmtree(workspace)
def test_configfile_hierarchy(monkeypatch, tmpdir):
etc_inmanta_dir = os.path.join(tmpdir, "etc", "inmanta")
os.makedirs(etc_inmanta_dir, exist_ok=False)
main_inmanta_cfg_file = os.path.join(etc_inmanta_dir, "inmanta.cfg")
inmanta_d_dir = os.path.join(etc_inmanta_dir, "inmanta.d")
os.mkdir(inmanta_d_dir)
inmanta_d_cfg_file01 = os.path.join(inmanta_d_dir, "01-dbconfig.cfg")
inmanta_d_cfg_file02 = os.path.join(inmanta_d_dir, "02-dbconfig.cfg")
inmanta_d_cfg_file_no_cfg_extension = os.path.join(inmanta_d_dir,
"03-config")
dot_inmanta_file = os.path.join(tmpdir, ".inmanta")
dot_inmanta_cfg_file = os.path.join(tmpdir, ".inmanta.cfg")
min_c_file = os.path.join(tmpdir, "custom.cfg")
monkeypatch.setenv("INMANTA_SERVER_AUTH", "true")
monkeypatch.setenv("INMANTA_SERVER_AGENT_TIMEOUT", "60")
with open(main_inmanta_cfg_file, "w", encoding="utf-8") as f:
f.write("""
[server]
auth=false
[config]
log-dir=/log
[database]
host=host1
name=db1
port=1234
[influxdb]
host=host1
interval=10
tags=tag1=value1
[dashboard]
path=/some/directory
client-id=test
""")
with open(inmanta_d_cfg_file01, "w", encoding="utf-8") as f:
f.write("""
[database]
host=host2
name=db2
[influxdb]
host=host2
""")
with open(inmanta_d_cfg_file02, "w", encoding="utf-8") as f:
f.write("""
[database]
port=5678
[influxdb]
host=host3
interval=20
""")
with open(inmanta_d_cfg_file_no_cfg_extension, "w", encoding="utf-8") as f:
f.write("""
[database]
port=9999
""")
with open(dot_inmanta_file, "w", encoding="utf-8") as f:
f.write("""
[database]
host=host3
[influxdb]
tags=tag2=value2
[dashboard]
path=/some/other/directory
""")
with open(dot_inmanta_cfg_file, "w", encoding="utf-8") as f:
f.write("""
[dashboard]
path=/directory
client-id=test123
""")
with open(min_c_file, "w", encoding="utf-8") as f:
f.write("""
[dashboard]
client-id=test456
""")
os.chdir(tmpdir)
Config.load_config(min_c_config_file=min_c_file,
config_dir=inmanta_d_dir,
main_cfg_file=main_inmanta_cfg_file)
assert Config.get("config", "log-dir") == "/log"
assert Config.get("database", "host") == "host3"
assert Config.get("database", "name") == "db2"
assert Config.get("database", "port") == 5678
assert Config.get("influxdb", "host") == "host3"
assert Config.get("influxdb", "interval") == 20
assert Config.get("influxdb", "tags")["tag2"] == "value2"
assert Config.get("dashboard", "path") == "/directory"
assert Config.get("dashboard", "client-id") == "test456"
assert Config.get("server", "auth")
assert Config.get("server", "agent-timeout") == 60
def server_port() -> "number":
return Config.get("compiler_rest_transport", "port", 8888)
def server_ssl() -> "bool":
return Config.get("compiler_rest_transport", "ssl", False)