def do_update(specs: "Dict[str, List[InmantaModuleRequirement]]",
modules: List[str]) -> None:
v2_modules = {
module
for module in modules
if my_project.module_source.path_for(module) is not None
}
v2_python_specs: List[Requirement] = [
ModuleV2Source.get_python_package_requirement(module_spec)
for module, module_specs in specs.items()
for module_spec in module_specs if module in v2_modules
]
if v2_python_specs:
env.process_env.install_from_index(
v2_python_specs,
my_project.module_source.urls,
upgrade=True,
allow_pre_releases=my_project.install_mode !=
InstallMode.release,
)
for v1_module in set(modules).difference(v2_modules):
spec = specs.get(v1_module, [])
try:
ModuleV1.update(my_project,
v1_module,
spec,
install_mode=my_project.install_mode)
except Exception:
LOGGER.exception("Failed to update module %s", v1_module)
# Load the newly installed modules into the modules cache
my_project.install_modules(bypass_module_cache=True,
update_dependencies=True)
def test_module_conversion_in_place_minimal(tmpdir):
module_name = "minimalv1module"
tmpdir = os.path.join(tmpdir, module_name)
path = os.path.normpath(
os.path.join(__file__, os.pardir, os.pardir, "data", "modules",
module_name))
shutil.copytree(path, tmpdir)
dummyproject = DummyProject()
module_in = ModuleV1(dummyproject, tmpdir)
ModuleConverter(module_in).convert_in_place()
assert_v2_module(module_name, tmpdir, minimal=True)
def construct_module(self, project: Optional[Project],
path: str) -> Module:
"""Construct a V1 or V2 module from a folder"""
try:
return ModuleV2(project, path)
except (ModuleMetadataFileNotFound, InvalidMetadata,
InvalidModuleException):
try:
return ModuleV1(project, path)
except (ModuleMetadataFileNotFound, InvalidMetadata,
InvalidModuleException):
raise InvalidModuleException(
f"No module can be found at {path}")
def test_module_conversion(tmpdir):
module_name = "elaboratev1module"
path = os.path.normpath(
os.path.join(__file__, os.pardir, os.pardir, "data", "modules",
module_name))
dummyproject = DummyProject()
module_in = ModuleV1(dummyproject, path)
assert sorted([str(r) for r in module_in.get_all_requires()
]) == ["mod1==1.0", "mod2", "std"]
ModuleConverter(module_in).convert(tmpdir)
assert_v2_module(module_name, tmpdir)
def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir):
"""
Ensure that the conversion of the std module from a V1 to a V2 module,
doesn't include the std module as a requirement in the setup.cfg file.
"""
clone_dir = os.path.join(tmpdir, "std")
subprocess.check_call(
["git", "clone", "https://github.com/inmanta/std.git", clone_dir])
dummyproject = DummyProject()
module_in = ModuleV1(dummyproject, clone_dir)
ModuleConverter(module_in).convert_in_place()
setup_cfg_file = os.path.join(clone_dir, "setup.cfg")
parser = configparser.ConfigParser()
parser.read(setup_cfg_file)
assert parser.has_option("options", "install_requires")
install_requires = [
Requirement.parse(r)
for r in parser.get("options", "install_requires").split("\n") if r
]
pkg_names = [r.name for r in install_requires]
assert "inmanta-module-std" not in pkg_names
def test_load_module_v1_and_v2_installed(
tmpdir: py.path.local,
snippetcompiler_clean,
modules_dir: str,
caplog,
) -> None:
"""
Test whether the Project.load_module() method works correctly when the v1 and v2 version of a module are both installed.
The V2 module should be loaded and a warning should be logged.
"""
module_name = "minimalv1module"
module_dir = os.path.join(modules_dir, module_name)
# Convert v1 to v2 module
module = ModuleV1(project=DummyProject(autostd=False), path=module_dir)
module_dir_v2 = os.path.join(tmpdir, f"{module_name}-v2")
ModuleConverter(module).convert(output_directory=module_dir_v2)
# The V1 module is installed implicitly. The snippetcompiler_clean fixture adds
# the `modules_dir` to the modulepath of the newly created project.
project: Project = snippetcompiler_clean.setup_for_snippet(
snippet=f"import {module_name}",
install_v2_modules=[LocalPackagePath(module_dir_v2, editable=False)],
install_project=False,
)
assert module_name not in project.modules
caplog.clear()
with caplog.at_level(logging.WARNING):
project.load_module(module_name=module_name,
install_v1=False,
install_v2=False,
allow_v1=True)
assert f"Module {module_name} is installed as a V1 module and a V2 module: V1 will be ignored." in caplog.text
assert module_name in project.modules
assert isinstance(project.modules[module_name], ModuleV2)
def _assert_module_requirements(expected_requirements: List[str]) -> None:
module_v1 = ModuleV1(project=None, path=module_dir)
assert sorted(module_v1.metadata.requires) == sorted(expected_requirements)
assert not os.path.exists(os.path.join(module_dir, "requirements.txt"))
def _assert_module_requirements(expected_requirement: str) -> None:
module_v1 = ModuleV1(project=None, path=module_dir)
assert module_v1.metadata.requires == []
with open(requirements_txt_file, "r", encoding="utf-8") as fd:
assert fd.read().strip() == expected_requirement
def test_v1_and_v2_module_installed_simultaneously(tmpdir: py.path.local,
snippetcompiler_clean,
capsys, caplog,
modules_dir: str) -> None:
"""
When a module is installed both in V1 and V2 format, ensure that:
* A warning is logged
* The V2 module is loaded and not the V1 module.
"""
# Work around caching problem in venv
feature_compiler_cache.set("False")
module_name = "v1_print_plugin"
def compile_and_verify(
expected_message: str,
expect_warning: bool,
install_v2_modules: List[LocalPackagePath] = []) -> None:
caplog.clear()
snippetcompiler_clean.setup_for_snippet(
f"import {module_name}",
install_v2_modules=install_v2_modules,
autostd=False)
snippetcompiler_clean.do_export()
assert expected_message in capsys.readouterr().out
got_warning = f"Module {module_name} is installed as a V1 module and a V2 module" in caplog.text
assert got_warning == expect_warning
# Run compile. Only a V1 module is installed in the module path
expected_message_v1 = "Hello world"
compile_and_verify(expected_message=expected_message_v1,
expect_warning=False)
assert isinstance(Project.get().modules[module_name], ModuleV1)
# Convert V1 module to V2 module and install it as well
module_dir = os.path.join(modules_dir, module_name)
v1_module_dir = os.path.join(tmpdir, "v1_module")
shutil.copytree(module_dir, v1_module_dir)
assert os.path.isdir(v1_module_dir)
v2_module_dir = os.path.join(tmpdir, "v2_module")
module = ModuleV1(project=DummyProject(autostd=False), path=v1_module_dir)
ModuleConverter(module).convert(output_directory=v2_module_dir)
# Print a different message in the V2 module, to detect which of both gets loaded
expected_message_v2 = "Other message"
with open(os.path.join(v2_module_dir, "model", "_init.cf"), "r+") as fd:
content = fd.read()
assert expected_message_v1 in content
content = content.replace(expected_message_v1, expected_message_v2)
assert expected_message_v2 in content
fd.seek(0)
fd.write(content)
# Run compile again. V1 version and V2 version are installed simultaneously
compile_and_verify(
expected_message=expected_message_v2,
expect_warning=True,
install_v2_modules=[
LocalPackagePath(path=v2_module_dir, editable=False)
],
)
assert isinstance(Project.get().modules[module_name], ModuleV2)
def list(self, requires: bool = False) -> None:
"""
List all modules in a table
"""
def show_bool(b: bool) -> str:
return "yes" if b else "no"
table = []
project = Project.get()
project.get_complete_ast()
names: Sequence[str] = sorted(project.modules.keys())
specs: Dict[str, List[
InmantaModuleRequirement]] = project.collect_imported_requirements(
)
for name in names:
mod: Module = Project.get().modules[name]
version = str(mod.version)
if name not in specs:
specs[name] = []
generation: str = str(mod.GENERATION.name).lower()
reqv: str
matches: bool
editable: bool
if isinstance(mod, ModuleV1):
try:
if project.install_mode == InstallMode.master:
reqv = "master"
else:
release_only = project.install_mode == InstallMode.release
versions = ModuleV1.get_suitable_version_for(
name,
specs[name],
mod._path,
release_only=release_only)
if versions is None:
reqv = "None"
else:
reqv = str(versions)
except Exception:
LOGGER.exception("Problem getting version for module %s" %
name)
reqv = "ERROR"
matches = version == reqv
editable = True
else:
reqv = ",".join(req.version_spec_str()
for req in specs[name] if req.specs) or "*"
matches = all(version in req for req in specs[name])
editable = mod.is_editable()
table.append((name, generation, editable, version, reqv, matches))
if requires:
LOGGER.warning(
"The `inmanta module list -r` command has been deprecated.")
for name, _, _, version, _, _ in table:
print(" - %s==%s" % (name, version))
else:
t = texttable.Texttable()
t.set_deco(texttable.Texttable.HEADER | texttable.Texttable.BORDER
| texttable.Texttable.VLINES)
t.header(("Name", "Type", "Editable", "Installed version",
"Expected in project", "Matches"))
t.set_cols_dtype(("t", "t", show_bool, "t", "t", show_bool))
for row in table:
t.add_row(row)
print(t.draw())
def test_module_update_with_v2_module(
tmpdir: py.path.local,
modules_v2_dir: str,
snippetcompiler_clean,
modules_repo: str,
corrupt_module: bool,
install_mode: InstallMode,
) -> None:
"""
Assert that the `inmanta module update` command works correctly when executed on a project with a V2 module.
:param corrupt_module: Whether the module to be updated contains a syntax error or not.
Dependency graph:
-> Inmanta project
-> module1 (v2) -> module2 (v2)
-> mod11 (v1)
"""
template_module_name = "elaboratev2module"
template_module_dir = os.path.join(modules_v2_dir, template_module_name)
pip_index = PipIndex(artifact_dir=os.path.join(str(tmpdir), "pip-index"))
def assert_version_installed(module_name: str, version: str) -> None:
package_name = ModuleV2Source.get_package_name_for(module_name)
installed_packages: Dict[
str, Version] = process_env.get_installed_packages()
assert package_name in installed_packages
assert str(installed_packages[package_name]) == version
for module_name, versions in {
"module2": ["2.0.1", "2.1.0", "2.2.0", "2.2.1.dev0", "3.0.1"],
"module1": ["1.2.4", "1.2.5"],
}.items():
# Module1 only has patch updates and Module2 has minor and major updates.
# This ensure that the test covers the different types of version bumps.
for current_version in versions:
module_dir = os.path.join(tmpdir,
f"{module_name}-v{current_version}")
module_from_template(
source_dir=template_module_dir,
dest_dir=module_dir,
new_version=Version(current_version),
new_name=module_name,
new_requirements=[
InmantaModuleRequirement(
Requirement.parse("module2<3.0.0"))
] if module_name == "module1" else None,
install=False,
publish_index=pip_index,
new_content_init_cf="import module2"
if module_name == "module1" else None,
)
patched_module_dir = os.path.join(tmpdir, "module1-v1.2.3")
module_from_template(
source_dir=template_module_dir,
dest_dir=patched_module_dir,
new_version=Version("1.2.3"),
new_name="module1",
# Add a dependency on module2, without setting an explicit version constraint. Later version of module1
# do set a version constraint on the dependency on module2. This way it is verified whether the module update
# command takes into account the version constraints set in a new version of a module.
new_requirements=[
InmantaModuleRequirement(Requirement.parse("module2"))
],
install=False,
publish_index=pip_index,
new_content_init_cf="entity"
if corrupt_module else None, # Introduce syntax error in the module
)
module_path = os.path.join(tmpdir, "modulepath")
os.mkdir(module_path)
mod11_dir = clone_repo(modules_repo, "mod11", module_path, tag="3.2.1")
snippetcompiler_clean.setup_for_snippet(
# Don't import module2 in the inmanta project. An import for module2 is present in module1 instead.
# This tests whether the module update command takes into account all transitive dependencies.
snippet="""
import module1
import mod11
""",
autostd=False,
install_v2_modules=[
LocalPackagePath(path=os.path.join(tmpdir, "module2-v2.0.1")),
LocalPackagePath(path=patched_module_dir),
],
add_to_module_path=[module_path],
python_package_sources=[pip_index.url],
project_requires=[
InmantaModuleRequirement.parse("module1<1.2.5"),
InmantaModuleRequirement.parse("mod11<4.2.0"),
],
install_mode=install_mode,
install_project=False,
)
assert_version_installed(module_name="module1", version="1.2.3")
assert_version_installed(module_name="module2", version="2.0.1")
assert ModuleV1(project=None, path=mod11_dir).version == Version("3.2.1")
ModuleTool().update()
assert_version_installed(module_name="module1", version="1.2.4")
assert_version_installed(module_name="module2",
version="2.2.0" if install_mode
== InstallMode.release else "2.2.1.dev0")
assert ModuleV1(project=None, path=mod11_dir).version == Version("4.1.2")