def do_update(specs: "Dict[str, List[InmantaModuleRequirement]]", modules: List[str]) -> None: v2_modules = { module for module in modules if my_project.module_source.path_for(module) is not None } v2_python_specs: List[Requirement] = [ ModuleV2Source.get_python_package_requirement(module_spec) for module, module_specs in specs.items() for module_spec in module_specs if module in v2_modules ] if v2_python_specs: env.process_env.install_from_index( v2_python_specs, my_project.module_source.urls, upgrade=True, allow_pre_releases=my_project.install_mode != InstallMode.release, ) for v1_module in set(modules).difference(v2_modules): spec = specs.get(v1_module, []) try: ModuleV1.update(my_project, v1_module, spec, install_mode=my_project.install_mode) except Exception: LOGGER.exception("Failed to update module %s", v1_module) # Load the newly installed modules into the modules cache my_project.install_modules(bypass_module_cache=True, update_dependencies=True)
def test_module_conversion_in_place_minimal(tmpdir): module_name = "minimalv1module" tmpdir = os.path.join(tmpdir, module_name) path = os.path.normpath( os.path.join(__file__, os.pardir, os.pardir, "data", "modules", module_name)) shutil.copytree(path, tmpdir) dummyproject = DummyProject() module_in = ModuleV1(dummyproject, tmpdir) ModuleConverter(module_in).convert_in_place() assert_v2_module(module_name, tmpdir, minimal=True)
def construct_module(self, project: Optional[Project], path: str) -> Module: """Construct a V1 or V2 module from a folder""" try: return ModuleV2(project, path) except (ModuleMetadataFileNotFound, InvalidMetadata, InvalidModuleException): try: return ModuleV1(project, path) except (ModuleMetadataFileNotFound, InvalidMetadata, InvalidModuleException): raise InvalidModuleException( f"No module can be found at {path}")
def test_module_conversion(tmpdir): module_name = "elaboratev1module" path = os.path.normpath( os.path.join(__file__, os.pardir, os.pardir, "data", "modules", module_name)) dummyproject = DummyProject() module_in = ModuleV1(dummyproject, path) assert sorted([str(r) for r in module_in.get_all_requires() ]) == ["mod1==1.0", "mod2", "std"] ModuleConverter(module_in).convert(tmpdir) assert_v2_module(module_name, tmpdir)
def test_issue_3159_conversion_std_module_add_self_to_dependencies(tmpdir): """ Ensure that the conversion of the std module from a V1 to a V2 module, doesn't include the std module as a requirement in the setup.cfg file. """ clone_dir = os.path.join(tmpdir, "std") subprocess.check_call( ["git", "clone", "https://github.com/inmanta/std.git", clone_dir]) dummyproject = DummyProject() module_in = ModuleV1(dummyproject, clone_dir) ModuleConverter(module_in).convert_in_place() setup_cfg_file = os.path.join(clone_dir, "setup.cfg") parser = configparser.ConfigParser() parser.read(setup_cfg_file) assert parser.has_option("options", "install_requires") install_requires = [ Requirement.parse(r) for r in parser.get("options", "install_requires").split("\n") if r ] pkg_names = [r.name for r in install_requires] assert "inmanta-module-std" not in pkg_names
def test_load_module_v1_and_v2_installed( tmpdir: py.path.local, snippetcompiler_clean, modules_dir: str, caplog, ) -> None: """ Test whether the Project.load_module() method works correctly when the v1 and v2 version of a module are both installed. The V2 module should be loaded and a warning should be logged. """ module_name = "minimalv1module" module_dir = os.path.join(modules_dir, module_name) # Convert v1 to v2 module module = ModuleV1(project=DummyProject(autostd=False), path=module_dir) module_dir_v2 = os.path.join(tmpdir, f"{module_name}-v2") ModuleConverter(module).convert(output_directory=module_dir_v2) # The V1 module is installed implicitly. The snippetcompiler_clean fixture adds # the `modules_dir` to the modulepath of the newly created project. project: Project = snippetcompiler_clean.setup_for_snippet( snippet=f"import {module_name}", install_v2_modules=[LocalPackagePath(module_dir_v2, editable=False)], install_project=False, ) assert module_name not in project.modules caplog.clear() with caplog.at_level(logging.WARNING): project.load_module(module_name=module_name, install_v1=False, install_v2=False, allow_v1=True) assert f"Module {module_name} is installed as a V1 module and a V2 module: V1 will be ignored." in caplog.text assert module_name in project.modules assert isinstance(project.modules[module_name], ModuleV2)
def _assert_module_requirements(expected_requirements: List[str]) -> None: module_v1 = ModuleV1(project=None, path=module_dir) assert sorted(module_v1.metadata.requires) == sorted(expected_requirements) assert not os.path.exists(os.path.join(module_dir, "requirements.txt"))
def _assert_module_requirements(expected_requirement: str) -> None: module_v1 = ModuleV1(project=None, path=module_dir) assert module_v1.metadata.requires == [] with open(requirements_txt_file, "r", encoding="utf-8") as fd: assert fd.read().strip() == expected_requirement
def test_v1_and_v2_module_installed_simultaneously(tmpdir: py.path.local, snippetcompiler_clean, capsys, caplog, modules_dir: str) -> None: """ When a module is installed both in V1 and V2 format, ensure that: * A warning is logged * The V2 module is loaded and not the V1 module. """ # Work around caching problem in venv feature_compiler_cache.set("False") module_name = "v1_print_plugin" def compile_and_verify( expected_message: str, expect_warning: bool, install_v2_modules: List[LocalPackagePath] = []) -> None: caplog.clear() snippetcompiler_clean.setup_for_snippet( f"import {module_name}", install_v2_modules=install_v2_modules, autostd=False) snippetcompiler_clean.do_export() assert expected_message in capsys.readouterr().out got_warning = f"Module {module_name} is installed as a V1 module and a V2 module" in caplog.text assert got_warning == expect_warning # Run compile. Only a V1 module is installed in the module path expected_message_v1 = "Hello world" compile_and_verify(expected_message=expected_message_v1, expect_warning=False) assert isinstance(Project.get().modules[module_name], ModuleV1) # Convert V1 module to V2 module and install it as well module_dir = os.path.join(modules_dir, module_name) v1_module_dir = os.path.join(tmpdir, "v1_module") shutil.copytree(module_dir, v1_module_dir) assert os.path.isdir(v1_module_dir) v2_module_dir = os.path.join(tmpdir, "v2_module") module = ModuleV1(project=DummyProject(autostd=False), path=v1_module_dir) ModuleConverter(module).convert(output_directory=v2_module_dir) # Print a different message in the V2 module, to detect which of both gets loaded expected_message_v2 = "Other message" with open(os.path.join(v2_module_dir, "model", "_init.cf"), "r+") as fd: content = fd.read() assert expected_message_v1 in content content = content.replace(expected_message_v1, expected_message_v2) assert expected_message_v2 in content fd.seek(0) fd.write(content) # Run compile again. V1 version and V2 version are installed simultaneously compile_and_verify( expected_message=expected_message_v2, expect_warning=True, install_v2_modules=[ LocalPackagePath(path=v2_module_dir, editable=False) ], ) assert isinstance(Project.get().modules[module_name], ModuleV2)
def list(self, requires: bool = False) -> None: """ List all modules in a table """ def show_bool(b: bool) -> str: return "yes" if b else "no" table = [] project = Project.get() project.get_complete_ast() names: Sequence[str] = sorted(project.modules.keys()) specs: Dict[str, List[ InmantaModuleRequirement]] = project.collect_imported_requirements( ) for name in names: mod: Module = Project.get().modules[name] version = str(mod.version) if name not in specs: specs[name] = [] generation: str = str(mod.GENERATION.name).lower() reqv: str matches: bool editable: bool if isinstance(mod, ModuleV1): try: if project.install_mode == InstallMode.master: reqv = "master" else: release_only = project.install_mode == InstallMode.release versions = ModuleV1.get_suitable_version_for( name, specs[name], mod._path, release_only=release_only) if versions is None: reqv = "None" else: reqv = str(versions) except Exception: LOGGER.exception("Problem getting version for module %s" % name) reqv = "ERROR" matches = version == reqv editable = True else: reqv = ",".join(req.version_spec_str() for req in specs[name] if req.specs) or "*" matches = all(version in req for req in specs[name]) editable = mod.is_editable() table.append((name, generation, editable, version, reqv, matches)) if requires: LOGGER.warning( "The `inmanta module list -r` command has been deprecated.") for name, _, _, version, _, _ in table: print(" - %s==%s" % (name, version)) else: t = texttable.Texttable() t.set_deco(texttable.Texttable.HEADER | texttable.Texttable.BORDER | texttable.Texttable.VLINES) t.header(("Name", "Type", "Editable", "Installed version", "Expected in project", "Matches")) t.set_cols_dtype(("t", "t", show_bool, "t", "t", show_bool)) for row in table: t.add_row(row) print(t.draw())
def test_module_update_with_v2_module( tmpdir: py.path.local, modules_v2_dir: str, snippetcompiler_clean, modules_repo: str, corrupt_module: bool, install_mode: InstallMode, ) -> None: """ Assert that the `inmanta module update` command works correctly when executed on a project with a V2 module. :param corrupt_module: Whether the module to be updated contains a syntax error or not. Dependency graph: -> Inmanta project -> module1 (v2) -> module2 (v2) -> mod11 (v1) """ template_module_name = "elaboratev2module" template_module_dir = os.path.join(modules_v2_dir, template_module_name) pip_index = PipIndex(artifact_dir=os.path.join(str(tmpdir), "pip-index")) def assert_version_installed(module_name: str, version: str) -> None: package_name = ModuleV2Source.get_package_name_for(module_name) installed_packages: Dict[ str, Version] = process_env.get_installed_packages() assert package_name in installed_packages assert str(installed_packages[package_name]) == version for module_name, versions in { "module2": ["2.0.1", "2.1.0", "2.2.0", "2.2.1.dev0", "3.0.1"], "module1": ["1.2.4", "1.2.5"], }.items(): # Module1 only has patch updates and Module2 has minor and major updates. # This ensure that the test covers the different types of version bumps. for current_version in versions: module_dir = os.path.join(tmpdir, f"{module_name}-v{current_version}") module_from_template( source_dir=template_module_dir, dest_dir=module_dir, new_version=Version(current_version), new_name=module_name, new_requirements=[ InmantaModuleRequirement( Requirement.parse("module2<3.0.0")) ] if module_name == "module1" else None, install=False, publish_index=pip_index, new_content_init_cf="import module2" if module_name == "module1" else None, ) patched_module_dir = os.path.join(tmpdir, "module1-v1.2.3") module_from_template( source_dir=template_module_dir, dest_dir=patched_module_dir, new_version=Version("1.2.3"), new_name="module1", # Add a dependency on module2, without setting an explicit version constraint. Later version of module1 # do set a version constraint on the dependency on module2. This way it is verified whether the module update # command takes into account the version constraints set in a new version of a module. new_requirements=[ InmantaModuleRequirement(Requirement.parse("module2")) ], install=False, publish_index=pip_index, new_content_init_cf="entity" if corrupt_module else None, # Introduce syntax error in the module ) module_path = os.path.join(tmpdir, "modulepath") os.mkdir(module_path) mod11_dir = clone_repo(modules_repo, "mod11", module_path, tag="3.2.1") snippetcompiler_clean.setup_for_snippet( # Don't import module2 in the inmanta project. An import for module2 is present in module1 instead. # This tests whether the module update command takes into account all transitive dependencies. snippet=""" import module1 import mod11 """, autostd=False, install_v2_modules=[ LocalPackagePath(path=os.path.join(tmpdir, "module2-v2.0.1")), LocalPackagePath(path=patched_module_dir), ], add_to_module_path=[module_path], python_package_sources=[pip_index.url], project_requires=[ InmantaModuleRequirement.parse("module1<1.2.5"), InmantaModuleRequirement.parse("mod11<4.2.0"), ], install_mode=install_mode, install_project=False, ) assert_version_installed(module_name="module1", version="1.2.3") assert_version_installed(module_name="module2", version="2.0.1") assert ModuleV1(project=None, path=mod11_dir).version == Version("3.2.1") ModuleTool().update() assert_version_installed(module_name="module1", version="1.2.4") assert_version_installed(module_name="module2", version="2.2.0" if install_mode == InstallMode.release else "2.2.1.dev0") assert ModuleV1(project=None, path=mod11_dir).version == Version("4.1.2")