def test_module_install_version(
tmpdir: py.path.local,
snippetcompiler_clean,
modules_v2_dir: str,
dev: bool,
) -> None:
"""
Make sure that the module install results in a module instance with the appropriate version information.
:param dev: whether to add a dev tag to the version
"""
module_name: str = "minimalv2module"
module_path: str = os.path.join(str(tmpdir), module_name)
module_version: version.Version = version.Version(
"1.2.3") if not dev else version.Version("1.2.3.dev0")
# set up simple project and activate the snippetcompiler venv
project: module.Project = snippetcompiler_clean.setup_for_snippet("")
# install module
module_from_template(
os.path.join(modules_v2_dir, module_name),
module_path,
new_version=module_version,
)
os.chdir(project.path)
ModuleTool().install(editable=True, path=module_path)
# check version
mod: module.Module = ModuleTool().get_module(module_name)
assert mod.version == module_version
def test_module_add_v2_module_to_v2_module(tmpdir: py.path.local, monkeypatch, modules_v2_dir: str) -> None:
"""
Add a V2 module to a V2 module using the `inmanta module add` command.
"""
# Create module to execute `inmanta module add` command on
module_dir = os.path.join(tmpdir, "test")
module_from_template(source_dir=os.path.join(modules_v2_dir, "elaboratev2module"), dest_dir=module_dir)
monkeypatch.chdir(module_dir)
def _assert_module_requirements(expected_requirements: List[str]) -> None:
module_v2 = ModuleV2(project=None, path=module_dir)
assert sorted(module_v2.metadata.install_requires) == sorted(expected_requirements)
assert not os.path.exists(os.path.join(module_dir, "requirements.txt"))
_assert_module_requirements(expected_requirements=[])
installed_packages = process_env.get_installed_packages()
name_dependent_module = "a_module"
ModuleTool().add(module_req="a_module", v2=True, override=False)
_assert_module_requirements(expected_requirements=[ModuleV2Source.get_package_name_for(name_dependent_module)])
with pytest.raises(CLIException, match="A dependency on the given module was already defined"):
ModuleTool().add(module_req=f"{name_dependent_module}==1.1.1", v2=True, override=False)
_assert_module_requirements(expected_requirements=[ModuleV2Source.get_package_name_for(name_dependent_module)])
ModuleTool().add(module_req=f"{name_dependent_module}==1.1.1", v2=True, override=True)
_assert_module_requirements(expected_requirements=[f"{ModuleV2Source.get_package_name_for(name_dependent_module)}==1.1.1"])
# Ensure no new packages were installed as a side-effect of `inmanta modules add`
assert process_env.get_installed_packages() == installed_packages
def test_3322_module_install_deep_data_files(tmpdir: py.path.local,
snippetcompiler_clean,
modules_v2_dir: str) -> None:
"""
Verify that module installation includes data files regardless of depth in the directory structure.
"""
# set up module directory
module_name: str = "minimalv2module"
module_path: str = str(tmpdir.join(module_name))
module_from_template(
os.path.join(modules_v2_dir, module_name),
module_path,
)
deep_model_file_rel: str = os.path.join(
"model",
*(str(i) for i in range(10)),
"mymod.cf",
)
os.makedirs(os.path.join(module_path,
os.path.dirname(deep_model_file_rel)))
open(os.path.join(module_path, deep_model_file_rel), "w").close()
# set up simple project and activate snippetcompiler venv
snippetcompiler_clean.setup_for_snippet("")
# install module: non-editable mode
ModuleTool().install(editable=False, path=module_path)
assert os.path.exists(
os.path.join(
env.process_env.site_packages_dir,
const.PLUGINS_PACKAGE,
module_name,
deep_model_file_rel,
))
def test_project_install_preinstalled(
local_module_package_index: str,
snippetcompiler_clean,
tmpdir: py.path.local,
modules_v2_dir: str,
editable: bool,
) -> None:
"""
Verify that `inmanta project install` does not override preinstalled modules.
"""
module_name: str = "minimalv2module"
fq_mod_name: str = "inmanta_plugins.minimalv2module"
assert env.process_env.get_module_file(fq_mod_name) is None
# activate snippetcompiler venv
snippetcompiler_clean.setup_for_snippet("")
# preinstall older version of module
module_path: str = os.path.join(str(tmpdir), module_name)
new_version = version.Version("1.2.3.dev0")
module_from_template(os.path.join(modules_v2_dir, module_name),
module_path,
new_version=new_version,
install=True,
editable=editable)
def assert_module_install() -> None:
module_info: Optional[Tuple[Optional[str],
Loader]] = env.process_env.get_module_file(
fq_mod_name)
env_module_file, module_loader = module_info
assert not isinstance(module_loader, loader.PluginModuleLoader)
assert env_module_file is not None
install_path: str = module_path if editable else env.process_env.site_packages_dir
assert env_module_file == os.path.join(install_path,
*fq_mod_name.split("."),
"__init__.py")
assert (env.process_env.get_installed_packages(
only_editable=editable).get(
f"{module.ModuleV2.PKG_NAME_PREFIX}{module_name}",
None) == new_version)
assert_module_install()
# set up project and modules
project: module.Project = snippetcompiler_clean.setup_for_snippet(
f"import {module_name}",
autostd=False,
python_package_sources=[local_module_package_index])
os.chdir(project.path)
# autostd=True reports std as an import for any module, thus requiring it to be v2 because v2 can not depend on v1
module.Project.get().autostd = False
ProjectTool().execute("install", [])
assert_module_install()
def test_non_matching_name_v2_module(self) -> None:
"""
Make sure the warning regarding directory name does not trigger for v2 modules, as it is not relevant there.
"""
template_dir: str = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "data", "modules_v2",
"minimalv2module")
with tempfile.TemporaryDirectory() as tmpdir:
mod_dir: str = os.path.join(tmpdir, "not-the-module-name")
module_from_template(template_dir, mod_dir)
module.ModuleV2(project=module.DummyProject(), path=mod_dir)
self.handler.flush()
assert self.stream.getvalue().strip() == ""
def test_module_add_v2_module_to_project(
tmpdir: py.path.local, snippetcompiler_clean, local_module_package_index: str, modules_v2_dir: str
) -> None:
"""
Add a V2 module to an inmanta project using the `inmanta module add` command.
"""
# Ensure that versions 1.1.1 and 1.2.0 exist in the pip index.
# Version 1.2.3 is present in the pip index exported by local_module_package_index.
pip_index = PipIndex(artifact_dir=os.path.join(str(tmpdir), "pip-index"))
for version in ["1.1.1", "1.2.0"]:
module_from_template(
source_dir=os.path.join(modules_v2_dir, "elaboratev2module"),
dest_dir=os.path.join(tmpdir, f"elaboratev2module-v{version}"),
new_version=Version(version),
publish_index=pip_index,
)
# Create project
project: Project = snippetcompiler_clean.setup_for_snippet(
snippet="", autostd=False, python_package_sources=[local_module_package_index, pip_index.url]
)
requirements_txt_file = os.path.join(project.path, "requirements.txt")
def _assert_project_state(pkg_name: str, expected_version: Version, project_requires_constraint: str) -> None:
installed_packages = process_env.get_installed_packages()
assert pkg_name in installed_packages
assert installed_packages[pkg_name] == expected_version
with open(project.get_metadata_file_path(), "r", encoding="utf-8") as fd:
project_metadata = ProjectMetadata.parse(fd)
assert not project_metadata.requires
with open(requirements_txt_file, "r", encoding="utf-8") as fd:
assert fd.read().strip() == ModuleV2Source.get_package_name_for(project_requires_constraint)
module_name = "elaboratev2module"
pkg_name = "inmanta-module-elaboratev2module"
assert pkg_name not in process_env.get_installed_packages().keys()
version_constraint = f"{module_name}==1.2.3"
ModuleTool().add(module_req=version_constraint, v2=True, override=False)
_assert_project_state(pkg_name=pkg_name, expected_version=Version("1.2.3"), project_requires_constraint=version_constraint)
new_version_constraint = f"{module_name}==1.2.0"
with pytest.raises(CLIException, match="A dependency on the given module was already defined"):
ModuleTool().add(module_req=new_version_constraint, v2=True, override=False)
_assert_project_state(pkg_name=pkg_name, expected_version=Version("1.2.3"), project_requires_constraint=version_constraint)
ModuleTool().add(module_req=new_version_constraint, v2=True, override=True)
_assert_project_state(
pkg_name=pkg_name, expected_version=Version("1.2.0"), project_requires_constraint=new_version_constraint
)
def test_module_add_v2_wrong_name_error(tmpdir: py.path.local, monkeypatch, modules_v2_dir: str) -> None:
"""
Test the error messages of v2 modules when adding with a wrong name. (issue #3556)
"""
# Create module to execute `inmanta module add` command on
module_dir = os.path.join(tmpdir, "test")
module_from_template(source_dir=os.path.join(modules_v2_dir, "elaboratev2module"), dest_dir=module_dir)
monkeypatch.chdir(module_dir)
with pytest.raises(ValueError, match="Invalid Inmanta module requirement: Inmanta module names use '_', not '-'."):
ModuleTool().add(module_req="a-module", v2=True, override=False)
with pytest.raises(
ValueError, match="Invalid Inmanta module requirement: Use the Inmanta module name instead of the Python package name"
):
ModuleTool().add(module_req="inmanta-module-a-module", v2=True, override=False)
def test_3322_module_install_preinstall_cleanup(tmpdir: py.path.local,
snippetcompiler_clean,
modules_v2_dir: str) -> None:
"""
Verify that installing a module from source cleans up any old installation's data files.
"""
# set up module directory
module_name: str = "minimalv2module"
module_path: str = str(tmpdir.join(module_name))
module_from_template(
os.path.join(modules_v2_dir, module_name),
module_path,
new_version=version.Version("1.0.0"),
)
model_file_rel: str = os.path.join("model", "mymod.cf")
model_file_source_path: str = os.path.join(module_path, model_file_rel)
assert not os.path.exists(model_file_source_path)
open(model_file_source_path, "w").close()
def model_file_installed() -> bool:
return os.path.exists(
os.path.join(
env.process_env.site_packages_dir,
const.PLUGINS_PACKAGE,
module_name,
model_file_rel,
))
# set up simple project and activate snippetcompiler venv
snippetcompiler_clean.setup_for_snippet("")
# install module: non-editable mode
ModuleTool().install(editable=False, path=module_path)
assert model_file_installed()
# remove model file and reinstall
os.remove(model_file_source_path)
module_from_template(
module_path,
new_version=version.Version("2.0.0"),
in_place=True,
)
ModuleTool().install(editable=False, path=module_path)
assert not model_file_installed()
def test_module_install_reinstall(
tmpdir: py.path.local,
snippetcompiler_clean,
modules_v2_dir,
) -> None:
"""
Verify that reinstalling a module from source installs any changes to model and Python files if the version is bumped.
"""
module_name: str = "minimalv2module"
module_path: str = str(tmpdir.join(module_name))
module_from_template(
os.path.join(modules_v2_dir, module_name),
dest_dir=module_path,
new_version=version.Version("1.0.0"),
)
# set up simple project and activate snippetcompiler venv
snippetcompiler_clean.setup_for_snippet("")
def new_files_exist() -> Iterator[bool]:
return (
os.path.exists(
os.path.join(env.process_env.site_packages_dir,
const.PLUGINS_PACKAGE, module_name, rel_path))
for rel_path in [os.path.join("model", "newmod.cf"), "newmod.py"])
# install module
ModuleTool().install(editable=False, path=module_path)
assert not any(new_files_exist())
# make some changes to the source and install again
model_dir: str = os.path.join(module_path, "model")
os.makedirs(model_dir, exist_ok=True)
open(os.path.join(model_dir, "newmod.cf"), "w").close()
open(
os.path.join(module_path, const.PLUGINS_PACKAGE, module_name,
"newmod.py"), "w").close()
module_from_template(module_path,
new_version=version.Version("2.0.0"),
in_place=True)
ModuleTool().install(editable=False, path=module_path)
assert all(new_files_exist())
def test_project_install_with_install_mode(
tmpdir: py.path.local, modules_v2_dir: str, snippetcompiler_clean,
install_mode: Optional[str]) -> None:
"""
Test whether the `inmanta module install` command takes into account the `install_mode` configured on the inmanta project.
"""
index: PipIndex = PipIndex(
artifact_dir=os.path.join(str(tmpdir), ".custom-index"))
module_template_path: str = os.path.join(modules_v2_dir,
"elaboratev2module")
module_name: str = "mod"
package_name: str = module.ModuleV2Source.get_package_name_for(module_name)
for module_version in ["1.0.0", "1.0.1.dev0"]:
module_from_template(
module_template_path,
os.path.join(str(tmpdir), f"mod-{module_version}"),
new_name=module_name,
new_version=version.Version(module_version),
publish_index=index,
)
# set up project
snippetcompiler_clean.setup_for_snippet(
f"import {module_name}",
autostd=False,
python_package_sources=[index.url],
python_requires=[Requirement.parse(package_name)],
install_mode=install_mode,
)
os.chdir(module.Project.get().path)
ProjectTool().execute("install", [])
if install_mode is None or install_mode == InstallMode.release:
expected_version = version.Version("1.0.0")
else:
expected_version = version.Version("1.0.1.dev0")
installed_packages: Dict[
str, version.Version] = env.process_env.get_installed_packages()
assert package_name in installed_packages
assert installed_packages[package_name] == expected_version
def test_module_add_preinstalled(tmpdir: py.path.local, modules_v2_dir: str, snippetcompiler_clean, caplog) -> None:
"""
Verify that `inmanta module add` respects preinstalled modules when they're compatible and logs a warning when they're
not.
"""
module_name: str = "mymodule"
pip_index = PipIndex(artifact_dir=str(tmpdir.join("pip-index")))
snippetcompiler_clean.setup_for_snippet(snippet="", autostd=False, python_package_sources=[pip_index.url])
# preinstall 1.0.0, don't publish to index
module_from_template(
os.path.join(modules_v2_dir, "minimalv2module"),
str(tmpdir.join(module_name, "1.0.0")),
new_name=module_name,
new_version=Version("1.0.0"),
install=True,
)
# publish 1.1.0 and 2.0.0 to index
module_from_template(
os.path.join(modules_v2_dir, "minimalv2module"),
str(tmpdir.join(module_name, "1.1.0")),
new_name=module_name,
new_version=Version("1.1.0"),
install=False,
publish_index=pip_index,
)
module_from_template(
os.path.join(modules_v2_dir, "minimalv2module"),
str(tmpdir.join(module_name, "2.0.0")),
new_name=module_name,
new_version=Version("2.0.0"),
install=False,
publish_index=pip_index,
)
# verify that compatible constraint does not reinstall or update
ModuleTool().add(module_req=f"{module_name}~=1.0", v2=True, override=True)
caplog.clear()
with caplog.at_level(logging.WARNING):
assert ModuleTool().get_module(module_name).version == Version("1.0.0")
assert "does not match constraint" not in caplog.text
# verify that incompatible constraint does reinstall and logs a warning
with caplog.at_level(logging.WARNING):
ModuleTool().add(module_req=f"{module_name}~=2.0", v2=True, override=True)
assert (
f"Currently installed {module_name}-1.0.0 does not match constraint ~=2.0: updating to compatible version."
in caplog.messages
)
assert ModuleTool().get_module(module_name).version == Version("2.0.0")
def test_moduletool_list(capsys, tmpdir: py.path.local,
local_module_package_index: str,
snippetcompiler_clean, modules_v2_dir: str) -> None:
"""
Verify that `inmanta module list` correctly lists all installed modules, both v1 and v2.
"""
# set up venv
snippetcompiler_clean.setup_for_snippet("", autostd=False)
module_template_path: str = os.path.join(modules_v2_dir, "minimalv2module")
module_from_template(
module_template_path,
str(tmpdir.join("custom_mod_one")),
new_name="custom_mod_one",
new_version=version.Version("1.0.0"),
install=True,
editable=False,
)
module_from_template(
module_template_path,
str(tmpdir.join("custom_mod_two")),
new_name="custom_mod_two",
new_version=version.Version("1.0.0"),
new_content_init_cf="import custom_mod_one",
new_requirements=[
module.InmantaModuleRequirement.parse("custom_mod_one~=1.0")
],
install=True,
editable=True,
)
# set up project with a v1 and a v2 module
project: module.Project = snippetcompiler_clean.setup_for_snippet(
"""
import std
import custom_mod_one
import custom_mod_two
""".strip(),
python_package_sources=[local_module_package_index],
project_requires=[
module.InmantaModuleRequirement.parse("std~=2.0,<2.1.11"),
module.InmantaModuleRequirement.parse("custom_mod_one>0"),
],
python_requires=[
module.ModuleV2Source.get_python_package_requirement(
module.InmantaModuleRequirement.parse("custom_mod_one<999")),
],
install_mode=InstallMode.release,
autostd=False,
)
capsys.readouterr()
ModuleTool().list()
out, err = capsys.readouterr()
assert (out.strip() == """
+----------------+------+----------+----------------+----------------+---------+
| Name | Type | Editable | Installed | Expected in | Matches |
| | | | version | project | |
+================+======+==========+================+================+=========+
| custom_mod_one | v2 | no | 1.0.0 | >0,<999,~=1.0 | yes |
| custom_mod_two | v2 | yes | 1.0.0 | * | yes |
| std | v1 | yes | 2.1.10 | 2.1.10 | yes |
+----------------+------+----------+----------------+----------------+---------+
""".strip())
# install incompatible version for custom_mod_one
module_from_template(
str(tmpdir.join("custom_mod_one")),
new_version=version.Version("2.0.0"),
install=True,
editable=False,
in_place=True,
)
project.invalidate_state("custom_mod_one")
capsys.readouterr()
ModuleTool().list()
out, err = capsys.readouterr()
assert (out.strip() == """
+----------------+------+----------+----------------+----------------+---------+
| Name | Type | Editable | Installed | Expected in | Matches |
| | | | version | project | |
+================+======+==========+================+================+=========+
| custom_mod_one | v2 | no | 2.0.0 | >0,<999,~=1.0 | no |
| custom_mod_two | v2 | yes | 1.0.0 | * | yes |
| std | v1 | yes | 2.1.10 | 2.1.10 | yes |
+----------------+------+----------+----------------+----------------+---------+
""".strip())
def test_project_install_incompatible_dependencies(
caplog,
snippetcompiler_clean,
tmpdir: py.path.local,
modules_dir: str,
modules_v2_dir: str,
) -> None:
"""
Verify that introducing version incompatibilities in the Python environment results in the appropriate exception and
warnings.
"""
index: PipIndex = PipIndex(
artifact_dir=os.path.join(str(tmpdir), ".custom-index"))
# prepare v2 modules
v2_template_path: str = os.path.join(modules_v2_dir, "minimalv2module")
v2mod1: module.ModuleV2Metadata = module_from_template(
v2_template_path,
os.path.join(str(tmpdir), "v2mod1"),
new_name="v2mod1",
new_requirements=[Requirement.parse("more-itertools~=7.0")],
publish_index=index,
)
v2mod2: module.ModuleV2Metadata = module_from_template(
v2_template_path,
os.path.join(str(tmpdir), "v2mod2"),
new_name="v2mod2",
new_requirements=[Requirement.parse("more-itertools~=8.0")],
publish_index=index,
)
# set up project
snippetcompiler_clean.setup_for_snippet(
f"""
import {module.ModuleV2.get_name_from_metadata(v2mod1)}
import {module.ModuleV2.get_name_from_metadata(v2mod2)}
""",
autostd=False,
install_project=False,
python_package_sources=[index.url, "https://pypi.org/simple"],
python_requires=[
Requirement.parse(
module.ModuleV2Source.get_package_name_for(
module.ModuleV2.get_name_from_metadata(metadata)))
for metadata in [v2mod1, v2mod2]
],
)
# install project
os.chdir(module.Project.get().path)
with pytest.raises(
CompilerException,
match=
("Not all installed modules are compatible: requirements conflicts were found. Please resolve any conflicts before"
" attempting another compile. Run `pip check` to check for any incompatibilities."
),
):
ProjectTool().execute("install", [])
assert any(
re.match(
"Incompatibility between constraint more-itertools~=[78].0 and installed version [78]\\..*",
rec.message) is not None for rec in caplog.records)
def test_project_install_incompatible_versions(
caplog,
snippetcompiler_clean,
tmpdir: py.path.local,
modules_dir: str,
modules_v2_dir: str,
autostd: bool,
) -> None:
"""
Verify that introducing module version incompatibilities results in the appropriate exception and warnings.
Make sure this works both for autostd (no explicit import) and for standard modules.
"""
v2_mod_name: str = "std" if autostd else "v2mod"
# declare conflicting module parameters
current_version: version.Version = version.Version("1.0.0")
req_v1_on_v1: module.InmantaModuleRequirement = module.InmantaModuleRequirement.parse(
"v1mod1>42")
req_v1_on_v2: module.InmantaModuleRequirement = module.InmantaModuleRequirement.parse(
f"{v2_mod_name}>10000")
# prepare v1 modules
v1_modules_path: str = os.path.join(str(tmpdir), "libs")
v1mod1_path: str = os.path.join(v1_modules_path, "v1mod1")
shutil.copytree(os.path.join(modules_dir, "minimalv1module"), v1mod1_path)
with open(os.path.join(v1mod1_path, module.ModuleV1.MODULE_FILE),
"r+") as fh:
config: Dict[str, object] = yaml.safe_load(fh)
config["name"] = "v1mod1"
config["version"] = str(current_version)
fh.seek(0)
yaml.dump(config, fh)
v1mod2_path: str = os.path.join(v1_modules_path, "v1mod2")
shutil.copytree(os.path.join(modules_dir, "minimalv1module"), v1mod2_path)
with open(os.path.join(v1mod2_path, module.ModuleV1.MODULE_FILE),
"r+") as fh:
config: Dict[str, object] = yaml.safe_load(fh)
config["name"] = "v1mod2"
config["requires"] = [str(req_v1_on_v2), str(req_v1_on_v1)]
fh.seek(0)
yaml.dump(config, fh)
# prepare v2 module
index: PipIndex = PipIndex(
artifact_dir=os.path.join(str(tmpdir), ".custom-index"))
module_from_template(
os.path.join(modules_v2_dir, "minimalv2module"),
os.path.join(str(tmpdir), v2_mod_name),
new_version=current_version,
new_name=v2_mod_name,
publish_index=index,
)
# set up project
snippetcompiler_clean.setup_for_snippet(
"""
import v1mod2
import v1mod1
%s
""" % (f"import {v2_mod_name}" if not autostd else ""),
autostd=autostd,
install_project=False,
add_to_module_path=[v1_modules_path],
python_package_sources=[index.url],
python_requires=[
Requirement.parse(
module.ModuleV2Source.get_package_name_for(v2_mod_name))
],
)
# install project
os.chdir(module.Project.get().path)
with pytest.raises(
CompilerException,
match=
"Not all module dependencies have been met. Run `inmanta modules update` to resolve this."
):
ProjectTool().execute("install", [])
log_messages: Set[str] = {rec.message for rec in caplog.records}
expected: Set[str] = {
f"requirement {req_v1_on_v1} on module v1mod1 not fulfilled, now at version {current_version}",
f"requirement {req_v1_on_v2} on module {v2_mod_name} not fulfilled, now at version {current_version}",
}
assert expected.issubset(log_messages)
def test_modules_v2_compile(tmpdir: str, snippetcompiler_clean,
modules_dir: str, modules_v2_dir: str) -> None:
# activate compiler venv and install std module (#3297)
snippetcompiler_clean.setup_for_snippet("", install_project=True)
v1_template_path: str = os.path.join(modules_dir, "minimalv1module")
v2_template_path: str = os.path.join(modules_v2_dir, "minimalv2module")
libs_dir: str = os.path.join(str(tmpdir), "libs")
# main module, used as proxy to load test module's model and plugins
main_module: str = "main"
# module under test
test_module: str = "test_module"
def test_module_plugin_contents(value: int) -> str:
"""
Returns the contents of the plugin file for the test module.
"""
return f"""
from inmanta.plugins import plugin
VALUE: str = {value}
@plugin
def get_value() -> "int":
return VALUE
""".strip()
# The model for the test module, regardless of value
test_module_model: str = f"value = {test_module}::get_value()"
# install main module (make it v1 so there are no restrictions on what it can depend on)
v1_module_from_template(
v1_template_path,
os.path.join(libs_dir, f"{main_module}"),
new_name=main_module,
new_content_init_cf=f"""
import {test_module}
# test variable import
test_module_value = {test_module}::value
# test plugin call
test_module_value = {test_module}::get_value()
# test intermodule Python imports
test_module_value = {main_module}::get_test_module_value()
""".strip(),
new_content_init_py=f"""
from inmanta.plugins import plugin
from {const.PLUGINS_PACKAGE}.{test_module} import VALUE
@plugin
def get_test_module_value() -> "int":
return VALUE
""".strip(),
)
def verify_compile(expected_value: int) -> None:
"""
Verify compilation by importing main module and checking its variable's value.
"""
snippetcompiler_clean.setup_for_snippet(
f"""
import {main_module}
# make sure imported variable has expected value (these statements will produce compiler error if it doesn't)
value = {main_module}::test_module_value
value = {expected_value}
""".strip(),
add_to_module_path=[libs_dir],
# set autostd=False because no v2 std exists at the time of writing
autostd=False,
install_project=False,
)
compiler.do_compile()
# install test module as v1 and verify compile
v1_module_from_template(
v1_template_path,
os.path.join(libs_dir, test_module),
new_name=test_module,
new_content_init_cf=test_module_model,
new_content_init_py=test_module_plugin_contents(1),
)
verify_compile(1)
# install module as v2 (on top of v1) and verify compile
v2_module_path: str = os.path.join(str(tmpdir), test_module)
module_from_template(
v2_template_path,
v2_module_path,
new_name=test_module,
new_content_init_cf=test_module_model,
new_content_init_py=test_module_plugin_contents(2),
install=True,
editable=True, # ! this is editable for the next test step
)
verify_compile(2)
# verify editable mode for plugins
with open(
os.path.join(v2_module_path, const.PLUGINS_PACKAGE, test_module,
"__init__.py"), "w") as fh:
fh.write(test_module_plugin_contents(3))
verify_compile(3)
# verify editable mode for model
with open(os.path.join(v2_module_path, "model", "_init.cf"), "w") as fh:
fh.write("value = 4")
# can't just verify_compile(4) because the plugin is still at 3 and promoting it to 4 would not test changes in the model
with pytest.raises(DoubleSetException) as excinfo:
verify_compile(3)
assert excinfo.value.newvalue == 4
def test_load_import_based_v2_module(
local_module_package_index: str,
snippetcompiler_clean,
modules_dir: str,
modules_v2_dir: str,
tmpdir: py.path.local,
v1: bool,
explicit_dependency: bool,
) -> None:
"""
A module needs to explicitly list its v2 dependencies in order to be able to load them. Import-based loading is not
allowed.
"""
main_module_name: str = "mymodule"
dependency_module_name: str = "minimalv2module"
index: PipIndex = PipIndex(os.path.join(str(tmpdir), ".my-index"))
libs_dir: str = os.path.join(str(tmpdir), "libs")
os.makedirs(libs_dir)
model: str = f"import {dependency_module_name}"
requirements: List[InmantaModuleRequirement] = ([
InmantaModuleRequirement.parse(dependency_module_name)
] if explicit_dependency else [])
if v1:
v1_module_from_template(
os.path.join(modules_dir, "minimalv1module"),
os.path.join(libs_dir, main_module_name),
new_name=main_module_name,
new_content_init_cf=model,
new_requirements=requirements,
)
else:
module_from_template(
os.path.join(modules_v2_dir, "minimalv2module"),
os.path.join(str(tmpdir), main_module_name),
new_name=main_module_name,
new_content_init_cf=model,
new_requirements=requirements,
install=False,
publish_index=index,
)
project: Project = snippetcompiler_clean.setup_for_snippet(
f"import {main_module_name}",
autostd=False,
install_project=False,
add_to_module_path=[libs_dir],
python_package_sources=[local_module_package_index, index.url],
# make sure that even listing the requirement in project.yml does not suffice
project_requires=[
InmantaModuleRequirement.parse(dependency_module_name)
],
python_requires=[] if v1 else [
Requirement.parse(
ModuleV2Source.get_package_name_for(main_module_name))
],
)
if explicit_dependency:
# assert that it doesn't raise an error with explicit requirements set
project.load_module_recursive(install=True)
else:
with pytest.raises(
ModuleLoadingException,
match=f"Failed to load module {dependency_module_name}"):
project.load_module_recursive(install=True)
def test_project_install_modules_cache_invalid(
caplog,
local_module_package_index: str,
snippetcompiler_clean,
tmpdir: py.path.local,
modules_dir: str,
modules_v2_dir: str,
) -> None:
"""
Verify that introducing invalidities in the modules cache results in the appropriate exception and warnings.
- preinstall old (v1 or v2) version of {dependency_module}
- install project with {main_module} that depends on {dependency_module}>={v2_version}
"""
main_module: str = "main_module"
dependency_module: str = "minimalv1module"
fq_mod_name: str = "inmanta_plugins.minimalv1module"
index: PipIndex = PipIndex(
artifact_dir=os.path.join(str(tmpdir), ".custom-index"))
libs_dir: str = os.path.join(str(tmpdir), "libs")
os.makedirs(libs_dir)
assert env.process_env.get_module_file(fq_mod_name) is None
# prepare most recent v2 module
v2_template_path: str = os.path.join(str(tmpdir), dependency_module)
v1: module.ModuleV1 = module.ModuleV1(project=DummyProject(autostd=False),
path=os.path.join(
modules_dir, dependency_module))
v2_version: version.Version = version.Version(
str(v1.version.major + 1) + ".0.0")
ModuleConverter(v1).convert(output_directory=v2_template_path)
module_from_template(v2_template_path,
os.path.join(str(tmpdir), dependency_module,
"stable"),
new_version=v2_version,
publish_index=index)
# prepare main module that depends on stable v2 version of first module
module_from_template(
v2_template_path,
os.path.join(str(tmpdir), main_module),
new_name=main_module,
new_content_init_cf=f"import {dependency_module}",
# requires stable version, not currently installed dev version
new_requirements=[
module.InmantaModuleRequirement.parse(
f"{dependency_module}>={v2_version}")
],
install=False,
publish_index=index,
)
# preinstall module
# set up project, including activation of venv before installing the module
snippetcompiler_clean.setup_for_snippet("", install_project=False)
# install older v2 module
module_from_template(
v2_template_path,
os.path.join(str(tmpdir), dependency_module, "dev"),
new_version=version.Version(f"{v2_version}.dev0"),
install=True,
)
# set up project for installation
project: module.Project = snippetcompiler_clean.setup_for_snippet(
f"import {main_module}",
autostd=False,
install_project=False,
add_to_module_path=[libs_dir],
python_package_sources=[index.url, local_module_package_index],
# make sure main module gets installed, pulling in newest version of dependency module
python_requires=[
Requirement.parse(
module.ModuleV2Source.get_package_name_for(main_module))
],
)
# populate project.modules[dependency_module] to force the error conditions in this simplified example
project.get_module(dependency_module, allow_v1=True)
os.chdir(project.path)
with pytest.raises(
CompilerException,
match=
("Not all modules were loaded correctly as a result of transitive dependencies."
" A recompile should load them correctly."),
):
ProjectTool().execute("install", [])
message: str = (
f"Compiler has loaded module {dependency_module}=={v2_version}.dev0 but {dependency_module}=={v2_version} has"
" later been installed as a side effect.")
assert message in (rec.message for rec in caplog.records)
def test_module_update_with_v2_module(
tmpdir: py.path.local,
modules_v2_dir: str,
snippetcompiler_clean,
modules_repo: str,
corrupt_module: bool,
install_mode: InstallMode,
) -> None:
"""
Assert that the `inmanta module update` command works correctly when executed on a project with a V2 module.
:param corrupt_module: Whether the module to be updated contains a syntax error or not.
Dependency graph:
-> Inmanta project
-> module1 (v2) -> module2 (v2)
-> mod11 (v1)
"""
template_module_name = "elaboratev2module"
template_module_dir = os.path.join(modules_v2_dir, template_module_name)
pip_index = PipIndex(artifact_dir=os.path.join(str(tmpdir), "pip-index"))
def assert_version_installed(module_name: str, version: str) -> None:
package_name = ModuleV2Source.get_package_name_for(module_name)
installed_packages: Dict[
str, Version] = process_env.get_installed_packages()
assert package_name in installed_packages
assert str(installed_packages[package_name]) == version
for module_name, versions in {
"module2": ["2.0.1", "2.1.0", "2.2.0", "2.2.1.dev0", "3.0.1"],
"module1": ["1.2.4", "1.2.5"],
}.items():
# Module1 only has patch updates and Module2 has minor and major updates.
# This ensure that the test covers the different types of version bumps.
for current_version in versions:
module_dir = os.path.join(tmpdir,
f"{module_name}-v{current_version}")
module_from_template(
source_dir=template_module_dir,
dest_dir=module_dir,
new_version=Version(current_version),
new_name=module_name,
new_requirements=[
InmantaModuleRequirement(
Requirement.parse("module2<3.0.0"))
] if module_name == "module1" else None,
install=False,
publish_index=pip_index,
new_content_init_cf="import module2"
if module_name == "module1" else None,
)
patched_module_dir = os.path.join(tmpdir, "module1-v1.2.3")
module_from_template(
source_dir=template_module_dir,
dest_dir=patched_module_dir,
new_version=Version("1.2.3"),
new_name="module1",
# Add a dependency on module2, without setting an explicit version constraint. Later version of module1
# do set a version constraint on the dependency on module2. This way it is verified whether the module update
# command takes into account the version constraints set in a new version of a module.
new_requirements=[
InmantaModuleRequirement(Requirement.parse("module2"))
],
install=False,
publish_index=pip_index,
new_content_init_cf="entity"
if corrupt_module else None, # Introduce syntax error in the module
)
module_path = os.path.join(tmpdir, "modulepath")
os.mkdir(module_path)
mod11_dir = clone_repo(modules_repo, "mod11", module_path, tag="3.2.1")
snippetcompiler_clean.setup_for_snippet(
# Don't import module2 in the inmanta project. An import for module2 is present in module1 instead.
# This tests whether the module update command takes into account all transitive dependencies.
snippet="""
import module1
import mod11
""",
autostd=False,
install_v2_modules=[
LocalPackagePath(path=os.path.join(tmpdir, "module2-v2.0.1")),
LocalPackagePath(path=patched_module_dir),
],
add_to_module_path=[module_path],
python_package_sources=[pip_index.url],
project_requires=[
InmantaModuleRequirement.parse("module1<1.2.5"),
InmantaModuleRequirement.parse("mod11<4.2.0"),
],
install_mode=install_mode,
install_project=False,
)
assert_version_installed(module_name="module1", version="1.2.3")
assert_version_installed(module_name="module2", version="2.0.1")
assert ModuleV1(project=None, path=mod11_dir).version == Version("3.2.1")
ModuleTool().update()
assert_version_installed(module_name="module1", version="1.2.4")
assert_version_installed(module_name="module2",
version="2.2.0" if install_mode
== InstallMode.release else "2.2.1.dev0")
assert ModuleV1(project=None, path=mod11_dir).version == Version("4.1.2")