def test_for_git_failures(git_modules_dir, modules_repo):
coroot = os.path.join(git_modules_dir, "testproject2")
subprocess.check_output(
[
"git", "clone",
os.path.join(git_modules_dir, "repos", "testproject"),
"testproject2"
],
cwd=git_modules_dir,
stderr=subprocess.STDOUT,
)
os.chdir(coroot)
Config.load_config()
ProjectTool().execute("install", [])
gp = module.gitprovider
module.gitprovider = BadModProvider(gp,
os.path.join(coroot, "libs", "mod6"))
try:
# test all tools, perhaps isolate to other test case
ProjectTool().execute("install", [])
ModuleTool().execute("list", [])
ModuleTool().execute("update", [])
ModuleTool().execute("status", [])
ModuleTool().execute("push", [])
finally:
module.gitprovider = gp
def test_complex_checkout(git_modules_dir, modules_repo):
coroot = os.path.join(git_modules_dir, "testproject")
subprocess.check_output([
"git", "clone",
os.path.join(git_modules_dir, "repos", "testproject")
],
cwd=git_modules_dir,
stderr=subprocess.STDOUT)
os.chdir(coroot)
Config.load_config()
ProjectTool().execute("install", [])
expected = ["mod1", "mod2", "mod3", "mod6", "mod7"]
for i in expected:
dirname = os.path.join(coroot, "libs", i)
assert os.path.exists(os.path.join(dirname, "signal"))
assert not os.path.exists(os.path.join(dirname, "badsignal"))
assert not os.path.exists(os.path.join(coroot, "libs", "mod5"))
# test all tools, perhaps isolate to other test case
ModuleTool().execute("list", [])
ModuleTool().execute("update", [])
ModuleTool().execute("status", [])
ModuleTool().execute("push", [])
def test_module_update_dependencies(
tmpdir: py.path.local,
monkeypatch,
snippetcompiler_clean,
modules_dir: str,
) -> None:
"""
Verify that `inmanta project update` correctly handles module's Python dependencies:
- update should include install
- update should update Python dependencies within module's constraints
- update should update transitive Python dependencies
"""
snippetcompiler_clean.setup_for_snippet(
snippet="import my_mod",
autostd=False,
install_project=False,
add_to_module_path=[str(tmpdir.join("modules"))],
)
# create index with multiple versions for packages a, b and c
index: PipIndex = PipIndex(str(tmpdir.join("index")))
create_python_package("a",
Version("1.0.0"),
str(tmpdir.join("a-1.0.0")),
publish_index=index)
for v in ("1.0.0", "1.0.1", "2.0.0"):
create_python_package("b",
Version(v),
str(tmpdir.join(f"b-{v}")),
requirements=[Requirement.parse("c")],
publish_index=index)
for v in ("1.0.0", "2.0.0"):
create_python_package("c",
Version(v),
str(tmpdir.join(f"c-{v}")),
publish_index=index)
# install b-1.0.0 and c-1.0.0
process_env.install_from_index(
[Requirement.parse(req) for req in ("b==1.0.0", "c==1.0.0")],
index_urls=[index.url])
# create my_mod
v1_module_from_template(
source_dir=os.path.join(modules_dir, "minimalv1module"),
dest_dir=str(tmpdir.join("modules", "my_mod")),
new_name="my_mod",
new_requirements=[Requirement.parse(req) for req in ("a", "b~=1.0.0")],
)
# run `inmanta project update` without running install first
monkeypatch.setenv("PIP_INDEX_URL", index.url)
ProjectTool().update()
# Verify that:
# - direct dependency a has been installed
# - direct dependency b has been updated but not past the allowed constraint
# - transitive dependency c has been updated
assert process_env.are_installed(("a==1.0.0", "b==1.0.1", "c==2.0.0"))
def test_master_checkout(git_modules_dir, modules_repo):
coroot = install_project(git_modules_dir, "masterproject")
ProjectTool().execute("install", [])
dirname = os.path.join(coroot, "libs", "mod8")
assert os.path.exists(os.path.join(dirname, "devsignal"))
assert os.path.exists(os.path.join(dirname, "mastersignal"))
def test_project_install_preinstalled(
local_module_package_index: str,
snippetcompiler_clean,
tmpdir: py.path.local,
modules_v2_dir: str,
editable: bool,
) -> None:
"""
Verify that `inmanta project install` does not override preinstalled modules.
"""
module_name: str = "minimalv2module"
fq_mod_name: str = "inmanta_plugins.minimalv2module"
assert env.process_env.get_module_file(fq_mod_name) is None
# activate snippetcompiler venv
snippetcompiler_clean.setup_for_snippet("")
# preinstall older version of module
module_path: str = os.path.join(str(tmpdir), module_name)
new_version = version.Version("1.2.3.dev0")
module_from_template(os.path.join(modules_v2_dir, module_name),
module_path,
new_version=new_version,
install=True,
editable=editable)
def assert_module_install() -> None:
module_info: Optional[Tuple[Optional[str],
Loader]] = env.process_env.get_module_file(
fq_mod_name)
env_module_file, module_loader = module_info
assert not isinstance(module_loader, loader.PluginModuleLoader)
assert env_module_file is not None
install_path: str = module_path if editable else env.process_env.site_packages_dir
assert env_module_file == os.path.join(install_path,
*fq_mod_name.split("."),
"__init__.py")
assert (env.process_env.get_installed_packages(
only_editable=editable).get(
f"{module.ModuleV2.PKG_NAME_PREFIX}{module_name}",
None) == new_version)
assert_module_install()
# set up project and modules
project: module.Project = snippetcompiler_clean.setup_for_snippet(
f"import {module_name}",
autostd=False,
python_package_sources=[local_module_package_index])
os.chdir(project.path)
# autostd=True reports std as an import for any module, thus requiring it to be v2 because v2 can not depend on v1
module.Project.get().autostd = False
ProjectTool().execute("install", [])
assert_module_install()
def test_bad_checkout(git_modules_dir, modules_repo):
coroot = os.path.join(git_modules_dir, "badproject")
subprocess.check_output(
["git", "clone",
os.path.join(git_modules_dir, "repos", "badproject")],
cwd=git_modules_dir,
stderr=subprocess.STDOUT)
os.chdir(coroot)
Config.load_config()
with pytest.raises(ModuleLoadingException):
ProjectTool().execute("install", [])
def test_for_repo_without_versions(git_modules_dir, modules_repo):
coroot = os.path.join(git_modules_dir, "noverproject")
subprocess.check_output([
"git", "clone",
os.path.join(git_modules_dir, "repos", "noverproject")
],
cwd=git_modules_dir,
stderr=subprocess.STDOUT)
os.chdir(coroot)
Config.load_config()
ProjectTool().execute("install", [])
def test_bad_dep_checkout(git_modules_dir, modules_repo):
coroot = os.path.join(git_modules_dir, "baddep")
subprocess.check_output(
["git", "clone",
os.path.join(git_modules_dir, "repos", "baddep")],
cwd=git_modules_dir,
stderr=subprocess.STDOUT)
os.chdir(coroot)
Config.load_config()
with pytest.raises(CompilerException,
match="Not all module dependencies have been met"):
ProjectTool().execute("install", [])
def test_dev_checkout(git_modules_dir, modules_repo):
coroot = os.path.join(git_modules_dir, "devproject")
subprocess.check_output(
["git", "clone",
os.path.join(git_modules_dir, "repos", "devproject")],
cwd=git_modules_dir,
stderr=subprocess.STDOUT)
os.chdir(coroot)
Config.load_config()
ProjectTool().execute("install", [])
dirname = os.path.join(coroot, "libs", "mod8")
assert os.path.exists(os.path.join(dirname, "devsignal"))
assert not os.path.exists(os.path.join(dirname, "mastersignal"))
def test_project_install(
local_module_package_index: str,
snippetcompiler_clean,
install_module_names: List[str],
module_dependencies: List[str],
) -> None:
"""
Install a simple inmanta project with `inmanta project install`. Make sure both v1 and v2 modules are installed
as expected.
"""
fq_mod_names: List[str] = [
f"inmanta_plugins.{mod}"
for mod in chain(install_module_names, module_dependencies)
]
# set up project and modules
project: module.Project = snippetcompiler_clean.setup_for_snippet(
"\n".join(f"import {mod}" for mod in ["std", *install_module_names]),
autostd=False,
python_package_sources=[local_module_package_index],
python_requires=[
Requirement.parse(module.ModuleV2Source.get_package_name_for(mod))
for mod in install_module_names
],
install_project=False,
)
os.chdir(project.path)
for fq_mod_name in fq_mod_names:
assert env.process_env.get_module_file(fq_mod_name) is None
# autostd=True reports std as an import for any module, thus requiring it to be v2 because v2 can not depend on v1
module.Project.get().autostd = False
ProjectTool().execute("install", [])
for fq_mod_name in fq_mod_names:
module_info: Optional[Tuple[Optional[str],
Loader]] = env.process_env.get_module_file(
fq_mod_name)
env_module_file, module_loader = module_info
assert not isinstance(module_loader, loader.PluginModuleLoader)
assert env_module_file is not None
assert env_module_file == os.path.join(
env.process_env.site_packages_dir, *fq_mod_name.split("."),
"__init__.py")
v1_mod_dir: str = os.path.join(project.path, project.downloadpath)
assert os.path.exists(v1_mod_dir)
assert os.listdir(v1_mod_dir) == ["std"]
def test_project_install_with_install_mode(
tmpdir: py.path.local, modules_v2_dir: str, snippetcompiler_clean,
install_mode: Optional[str]) -> None:
"""
Test whether the `inmanta module install` command takes into account the `install_mode` configured on the inmanta project.
"""
index: PipIndex = PipIndex(
artifact_dir=os.path.join(str(tmpdir), ".custom-index"))
module_template_path: str = os.path.join(modules_v2_dir,
"elaboratev2module")
module_name: str = "mod"
package_name: str = module.ModuleV2Source.get_package_name_for(module_name)
for module_version in ["1.0.0", "1.0.1.dev0"]:
module_from_template(
module_template_path,
os.path.join(str(tmpdir), f"mod-{module_version}"),
new_name=module_name,
new_version=version.Version(module_version),
publish_index=index,
)
# set up project
snippetcompiler_clean.setup_for_snippet(
f"import {module_name}",
autostd=False,
python_package_sources=[index.url],
python_requires=[Requirement.parse(package_name)],
install_mode=install_mode,
)
os.chdir(module.Project.get().path)
ProjectTool().execute("install", [])
if install_mode is None or install_mode == InstallMode.release:
expected_version = version.Version("1.0.0")
else:
expected_version = version.Version("1.0.1.dev0")
installed_packages: Dict[
str, version.Version] = env.process_env.get_installed_packages()
assert package_name in installed_packages
assert installed_packages[package_name] == expected_version
def test_install_for_git_failures(git_modules_dir, modules_repo):
coroot = os.path.join(git_modules_dir, "testproject3")
subprocess.check_output(
[
"git", "clone",
os.path.join(git_modules_dir, "repos", "testproject"),
"testproject3"
],
cwd=git_modules_dir,
stderr=subprocess.STDOUT,
)
os.chdir(coroot)
Config.load_config()
gp = module.gitprovider
module.gitprovider = BadModProvider(gp,
os.path.join(coroot, "libs", "mod6"))
try:
with pytest.raises(ModuleLoadingException):
ProjectTool().execute("install", [])
finally:
module.gitprovider = gp
def test_project_install_incompatible_dependencies(
caplog,
snippetcompiler_clean,
tmpdir: py.path.local,
modules_dir: str,
modules_v2_dir: str,
) -> None:
"""
Verify that introducing version incompatibilities in the Python environment results in the appropriate exception and
warnings.
"""
index: PipIndex = PipIndex(
artifact_dir=os.path.join(str(tmpdir), ".custom-index"))
# prepare v2 modules
v2_template_path: str = os.path.join(modules_v2_dir, "minimalv2module")
v2mod1: module.ModuleV2Metadata = module_from_template(
v2_template_path,
os.path.join(str(tmpdir), "v2mod1"),
new_name="v2mod1",
new_requirements=[Requirement.parse("more-itertools~=7.0")],
publish_index=index,
)
v2mod2: module.ModuleV2Metadata = module_from_template(
v2_template_path,
os.path.join(str(tmpdir), "v2mod2"),
new_name="v2mod2",
new_requirements=[Requirement.parse("more-itertools~=8.0")],
publish_index=index,
)
# set up project
snippetcompiler_clean.setup_for_snippet(
f"""
import {module.ModuleV2.get_name_from_metadata(v2mod1)}
import {module.ModuleV2.get_name_from_metadata(v2mod2)}
""",
autostd=False,
install_project=False,
python_package_sources=[index.url, "https://pypi.org/simple"],
python_requires=[
Requirement.parse(
module.ModuleV2Source.get_package_name_for(
module.ModuleV2.get_name_from_metadata(metadata)))
for metadata in [v2mod1, v2mod2]
],
)
# install project
os.chdir(module.Project.get().path)
with pytest.raises(
CompilerException,
match=
("Not all installed modules are compatible: requirements conflicts were found. Please resolve any conflicts before"
" attempting another compile. Run `pip check` to check for any incompatibilities."
),
):
ProjectTool().execute("install", [])
assert any(
re.match(
"Incompatibility between constraint more-itertools~=[78].0 and installed version [78]\\..*",
rec.message) is not None for rec in caplog.records)
def test_project_install_incompatible_versions(
caplog,
snippetcompiler_clean,
tmpdir: py.path.local,
modules_dir: str,
modules_v2_dir: str,
autostd: bool,
) -> None:
"""
Verify that introducing module version incompatibilities results in the appropriate exception and warnings.
Make sure this works both for autostd (no explicit import) and for standard modules.
"""
v2_mod_name: str = "std" if autostd else "v2mod"
# declare conflicting module parameters
current_version: version.Version = version.Version("1.0.0")
req_v1_on_v1: module.InmantaModuleRequirement = module.InmantaModuleRequirement.parse(
"v1mod1>42")
req_v1_on_v2: module.InmantaModuleRequirement = module.InmantaModuleRequirement.parse(
f"{v2_mod_name}>10000")
# prepare v1 modules
v1_modules_path: str = os.path.join(str(tmpdir), "libs")
v1mod1_path: str = os.path.join(v1_modules_path, "v1mod1")
shutil.copytree(os.path.join(modules_dir, "minimalv1module"), v1mod1_path)
with open(os.path.join(v1mod1_path, module.ModuleV1.MODULE_FILE),
"r+") as fh:
config: Dict[str, object] = yaml.safe_load(fh)
config["name"] = "v1mod1"
config["version"] = str(current_version)
fh.seek(0)
yaml.dump(config, fh)
v1mod2_path: str = os.path.join(v1_modules_path, "v1mod2")
shutil.copytree(os.path.join(modules_dir, "minimalv1module"), v1mod2_path)
with open(os.path.join(v1mod2_path, module.ModuleV1.MODULE_FILE),
"r+") as fh:
config: Dict[str, object] = yaml.safe_load(fh)
config["name"] = "v1mod2"
config["requires"] = [str(req_v1_on_v2), str(req_v1_on_v1)]
fh.seek(0)
yaml.dump(config, fh)
# prepare v2 module
index: PipIndex = PipIndex(
artifact_dir=os.path.join(str(tmpdir), ".custom-index"))
module_from_template(
os.path.join(modules_v2_dir, "minimalv2module"),
os.path.join(str(tmpdir), v2_mod_name),
new_version=current_version,
new_name=v2_mod_name,
publish_index=index,
)
# set up project
snippetcompiler_clean.setup_for_snippet(
"""
import v1mod2
import v1mod1
%s
""" % (f"import {v2_mod_name}" if not autostd else ""),
autostd=autostd,
install_project=False,
add_to_module_path=[v1_modules_path],
python_package_sources=[index.url],
python_requires=[
Requirement.parse(
module.ModuleV2Source.get_package_name_for(v2_mod_name))
],
)
# install project
os.chdir(module.Project.get().path)
with pytest.raises(
CompilerException,
match=
"Not all module dependencies have been met. Run `inmanta modules update` to resolve this."
):
ProjectTool().execute("install", [])
log_messages: Set[str] = {rec.message for rec in caplog.records}
expected: Set[str] = {
f"requirement {req_v1_on_v1} on module v1mod1 not fulfilled, now at version {current_version}",
f"requirement {req_v1_on_v2} on module {v2_mod_name} not fulfilled, now at version {current_version}",
}
assert expected.issubset(log_messages)
def test_project_install_modules_cache_invalid(
caplog,
local_module_package_index: str,
snippetcompiler_clean,
tmpdir: py.path.local,
modules_dir: str,
modules_v2_dir: str,
) -> None:
"""
Verify that introducing invalidities in the modules cache results in the appropriate exception and warnings.
- preinstall old (v1 or v2) version of {dependency_module}
- install project with {main_module} that depends on {dependency_module}>={v2_version}
"""
main_module: str = "main_module"
dependency_module: str = "minimalv1module"
fq_mod_name: str = "inmanta_plugins.minimalv1module"
index: PipIndex = PipIndex(
artifact_dir=os.path.join(str(tmpdir), ".custom-index"))
libs_dir: str = os.path.join(str(tmpdir), "libs")
os.makedirs(libs_dir)
assert env.process_env.get_module_file(fq_mod_name) is None
# prepare most recent v2 module
v2_template_path: str = os.path.join(str(tmpdir), dependency_module)
v1: module.ModuleV1 = module.ModuleV1(project=DummyProject(autostd=False),
path=os.path.join(
modules_dir, dependency_module))
v2_version: version.Version = version.Version(
str(v1.version.major + 1) + ".0.0")
ModuleConverter(v1).convert(output_directory=v2_template_path)
module_from_template(v2_template_path,
os.path.join(str(tmpdir), dependency_module,
"stable"),
new_version=v2_version,
publish_index=index)
# prepare main module that depends on stable v2 version of first module
module_from_template(
v2_template_path,
os.path.join(str(tmpdir), main_module),
new_name=main_module,
new_content_init_cf=f"import {dependency_module}",
# requires stable version, not currently installed dev version
new_requirements=[
module.InmantaModuleRequirement.parse(
f"{dependency_module}>={v2_version}")
],
install=False,
publish_index=index,
)
# preinstall module
# set up project, including activation of venv before installing the module
snippetcompiler_clean.setup_for_snippet("", install_project=False)
# install older v2 module
module_from_template(
v2_template_path,
os.path.join(str(tmpdir), dependency_module, "dev"),
new_version=version.Version(f"{v2_version}.dev0"),
install=True,
)
# set up project for installation
project: module.Project = snippetcompiler_clean.setup_for_snippet(
f"import {main_module}",
autostd=False,
install_project=False,
add_to_module_path=[libs_dir],
python_package_sources=[index.url, local_module_package_index],
# make sure main module gets installed, pulling in newest version of dependency module
python_requires=[
Requirement.parse(
module.ModuleV2Source.get_package_name_for(main_module))
],
)
# populate project.modules[dependency_module] to force the error conditions in this simplified example
project.get_module(dependency_module, allow_v1=True)
os.chdir(project.path)
with pytest.raises(
CompilerException,
match=
("Not all modules were loaded correctly as a result of transitive dependencies."
" A recompile should load them correctly."),
):
ProjectTool().execute("install", [])
message: str = (
f"Compiler has loaded module {dependency_module}=={v2_version}.dev0 but {dependency_module}=={v2_version} has"
" later been installed as a side effect.")
assert message in (rec.message for rec in caplog.records)