def test_can_understand_wheel_dependecies(
current_platform: TargetPlatform,
requirement_parser: RequirementParser):
runtime_dependencies = RequirementSet(current_platform)
runtime_dependency = requirement_parser.parse("runtime_dependency")
runtime_dependencies.add(runtime_dependency)
build_dependencies = RequirementSet(current_platform)
build_dependency = requirement_parser.parse("build_dependency")
build_dependencies.add(build_dependency)
wheel = Wheel(
name="testpackage",
version="",
deps=runtime_dependencies,
target_platform=current_platform,
license="",
homepage="",
description="",
build_dependencies=build_dependencies,
)
requirement = requirement_parser.parse("testpackage")
dependency_graph = DependencyGraph()
dependency_graph.import_wheel(wheel, requirement_parser)
assert dependency_graph.is_buildtime_dependency(requirement,
build_dependency)
assert dependency_graph.is_runtime_dependency(requirement,
runtime_dependency)
def test_versions_add_if_same_requirement_is_added_twice(current_platform):
requirement_set = RequirementSet(current_platform)
requirement_set.add(requirement_parser.parse("pypi2nix <= 2.0"))
requirement_set.add(requirement_parser.parse("pypi2nix >= 1.9"))
requirement = requirement_set.requirements["pypi2nix"]
assert isinstance(requirement, VersionRequirement)
assert len(requirement.version()) == 2
def test_length_is_one_after_adding_same_requirement_twice(
current_platform, requirement_parser
):
requirement_set = RequirementSet(current_platform)
requirement_set.add(requirement_parser.parse("pypi2nix"))
requirement_set.add(requirement_parser.parse("pypi2nix"))
assert len(requirement_set) == 1
def test_sources_contains_a_source_per_git_requirement(current_platform,
requirement_parser):
requirement_set = RequirementSet(current_platform)
requirement_set.add(requirement_parser.parse("no-git-source"))
requirement_set.add(
requirement_parser.parse("git+https://url.test/path#egg=test-egg"))
assert len(requirement_set.sources()) == 1
def test_elements_from_both_sets_can_be_found_in_sum_of_sets(current_platform):
left = RequirementSet(current_platform)
left.add(requirement_parser.parse("test1"))
right = RequirementSet(current_platform)
right.add(requirement_parser.parse("test2"))
sum = left + right
assert "test1" in sum
assert "test2" in sum
def test_pip_downloads_sources_to_target_directory(pip, project_dir,
current_platform):
download_path = os.path.join(project_dir, "download")
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip.download_sources(requirements=requirements,
target_directory=download_path)
assert list_files(download_path)
def test_install_six_yields_non_empty_freeze_output(pip, project_dir,
download_dir,
current_platform):
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip.download_sources(requirements, download_dir)
pip.install(requirements, source_directories=[download_dir])
assert pip.freeze()
def wrapper(requirement_lines: List[str]) -> List[Wheel]:
requirements = RequirementSet(current_platform)
for line in requirement_lines:
requirements.add(requirement_parser.parse(line))
wheel_paths = wheel_builder.build(requirements)
metadata_fetcher = MetadataFetcher(generated_sources, logger,
requirement_parser, pypi)
return metadata_fetcher.main(wheel_paths, current_platform,
wheel_builder.source_distributions)
def test_pip_with_data_directory_index_can_download_six(
pip_from_data_directory: VirtualenvPip,
download_dir: str,
requirement_parser: RequirementParser,
current_platform: TargetPlatform,
) -> None:
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip_from_data_directory.download_sources(requirements, download_dir)
def wrapper(requirement_lines: List[str]) -> List[Wheel]:
requirements = RequirementSet(current_platform)
for line in requirement_lines:
requirements.add(requirement_parser.parse(line))
wheel_paths = wheel_builder.build(requirements)
stage2 = Stage2(sources_for_test_packages, logger, requirement_parser, pypi)
return stage2.main(
wheel_paths, current_platform, wheel_builder.additional_build_dependencies
)
def test_pip_without_index_cannot_download_six(
pip_without_index: VirtualenvPip,
download_dir: str,
requirement_parser: RequirementParser,
current_platform: TargetPlatform,
) -> None:
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
with pytest.raises(PipFailed):
pip_without_index.download_sources(requirements, download_dir)
def test_can_add_build_dependencies_to_wheel(
wheel: Wheel,
current_platform: TargetPlatform,
requirement_parser: RequirementParser,
):
build_dependencies = RequirementSet(current_platform)
build_dependencies.add(requirement_parser.parse("dep1"))
build_dependencies.add(requirement_parser.parse("dep2"))
wheel.add_build_dependencies(build_dependencies)
assert "dep1" in wheel.build_dependencies(current_platform)
assert "dep2" in wheel.build_dependencies(current_platform)
def test_install_six_yields_non_empty_freeze_output(
pip: Pip, project_dir, download_dir, current_platform, requirement_parser
):
lib_dir = os.path.join(project_dir, "lib")
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip.download_sources(requirements, download_dir)
pip.install(
requirements, source_directories=[download_dir], target_directory=lib_dir
)
assert pip.freeze([lib_dir])
def test_freeze_respects_additional_python_path(pip, project_dir, current_platform):
prefix = os.path.join(project_dir, "custom-prefix")
download_dir = os.path.join(project_dir, "download")
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip.download_sources(requirements, download_dir)
pip.install(
requirements, target_directory=prefix, source_directories=[download_dir]
)
freeze_without_six = pip.freeze()
freeze_with_six = pip.freeze(python_path=[prefix])
assert len(freeze_without_six) < len(freeze_with_six)
def test_pip_downloads_sources_to_target_directory(
pip: Pip,
project_dir: str,
current_platform: TargetPlatform,
requirement_parser: RequirementParser,
):
download_path = Path(project_dir) / "download"
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip.download_sources(requirements=requirements,
target_directory=download_path)
assert download_path.list_files()
def source_distribution_archive(pip, requirement, download_dir,
current_platform):
requirement_set = RequirementSet(current_platform)
requirement_set.add(requirement)
pip.download_sources(requirement_set, download_dir)
for file_name in os.listdir(download_dir):
if file_name.startswith(requirement.name()):
return Archive(path=os.path.join(download_dir, file_name))
else:
raise Exception(
"Could not download source distribution for `{}`".format(
requirement.name()))
def test_pip_can_install_wheels_previously_downloaded(pip, project_dir,
current_platform):
download_directory = os.path.join(project_dir, "download")
target_directory = os.path.join(project_dir, "wheels")
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip.download_sources(requirements, download_directory)
pip.build_wheels(
requirements=requirements,
source_directories=[download_directory],
target_directory=target_directory,
)
assert list_files(target_directory)
assert any(map(lambda x: x.endswith(".whl"), list_files(target_directory)))
def test_that_extras_are_preserved_when_converting_to_and_from_a_file(
requirement_parser: RequirementParser,
requirement_set: RequirementSet,
current_platform: TargetPlatform,
project_dir: str,
logger: Logger,
):
requirement_set.add(requirement_parser.parse("req[extra1]"))
requirements_file = requirement_set.to_file(project_dir, current_platform,
requirement_parser, logger)
new_requirements_set = RequirementSet.from_file(requirements_file,
current_platform,
requirement_parser, logger)
requirement = new_requirements_set["req"]
assert requirement.extras() == {"extra1"}
class RequirementsCollector:
def __init__(
self,
platform: TargetPlatform,
requirement_parser: RequirementParser,
logger: Logger,
project_directory: str,
):
self.platform = platform
self.requirement_set = RequirementSet(platform)
self.requirement_parser = requirement_parser
self.logger = logger
self._project_directory = project_directory
self._sources = Sources()
def requirements(self) -> RequirementSet:
return self.requirement_set
def add_line(self, line: str) -> None:
requirement = self.requirement_parser.parse(line)
if isinstance(requirement, PathRequirement):
requirement = requirement.change_path(
lambda path: self._handle_requirements_path(
name=requirement.name(), path=path))
self.requirement_set.add(requirement)
def add_file(self, file_path: str) -> None:
requirements_file = RequirementsFile(file_path,
self._project_directory,
self.requirement_parser,
self.logger)
requirements_file.process()
self._sources.update(requirements_file.sources())
added_requirements = RequirementSet.from_file(requirements_file,
self.platform,
self.requirement_parser,
self.logger)
self.requirement_set += added_requirements
def sources(self) -> Sources:
sources = Sources()
sources.update(self.requirement_set.sources())
sources.update(self._sources)
return sources
def _handle_requirements_path(self, name: str, path: str) -> str:
self._sources.add(name, PathSource(path))
return os.path.abspath(path)
def test_install_to_target_directory_does_not_install_to_default_directory(
pip, project_dir, download_dir, current_platform):
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
target_directory = os.path.join(project_dir, "target-directory")
os.makedirs(target_directory)
pip.download_sources(requirements, download_dir)
assert not os.listdir(target_directory)
pip.install(
requirements,
source_directories=[download_dir],
target_directory=target_directory,
)
assert os.listdir(target_directory)
def test_can_install_generated_packages(
pip,
current_platform: TargetPlatform,
requirement_parser: RequirementParser,
target_directory: Path,
install_target: Path,
package_generator: PackageGenerator,
):
package_generator.generate_setuptools_package(name="testpackage")
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("testpackage"))
pip.install(
requirements,
source_directories=[str(target_directory)],
target_directory=str(install_target),
)
assert "testpackage" in pip.freeze(python_path=[str(install_target)])
def test_pip_can_install_wheels_previously_downloaded(
pip: Pip,
project_dir: str,
current_platform: TargetPlatform,
requirement_parser: RequirementParser,
download_dir: Path,
wheels_dir: Path,
):
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip.download_sources(requirements, download_dir)
pip.build_wheels(
requirements=requirements,
source_directories=[download_dir],
target_directory=wheels_dir,
)
assert wheels_dir.list_files()
assert any(map(lambda x: x.endswith(".whl"), wheels_dir.list_files()))
def build_dependencies(self, target_platform: TargetPlatform) -> RequirementSet:
setup_requires = self.setup_cfg.get("options", {}).get("setup_requires")
requirements = RequirementSet(target_platform)
if isinstance(setup_requires, str):
requirements.add(self.requirement_parser.parse(setup_requires))
elif isinstance(setup_requires, list):
for requirement_string in setup_requires:
try:
requirement = self.requirement_parser.parse(requirement_string)
except ParsingFailed as e:
self.logger.warning(
f"Failed to parse build dependency of `{self.setup_cfg_path}`"
)
self.logger.warning(f"Possible reason: `{e.reason}`")
else:
if requirement.applies_to_target(target_platform):
requirements.add(requirement)
return requirements
def test_can_generate_packages_with_requirements(
package_generator: PackageGenerator,
requirement_parser: RequirementParser,
pip: Pip,
target_directory: Path,
install_target: Path,
current_platform: TargetPlatform,
):
package_generator.generate_setuptools_package(
name="testpackage", install_requires=["other-package"])
package_generator.generate_setuptools_package(name="other-package")
requirements = RequirementSet(target_platform=current_platform)
requirements.add(requirement_parser.parse("testpackage"))
pip.install(
requirements,
source_directories=[target_directory],
target_directory=install_target,
)
assert "other-package" in pip.freeze([install_target])
def build_dependencies_from_pyproject_toml(
self, target_platform: TargetPlatform) -> RequirementSet:
requirement_set = RequirementSet(target_platform)
if self.pyproject_toml is None:
pass
else:
for build_input in self.pyproject_toml.get("build-system",
{}).get("requires", []):
try:
requirement = requirement_parser.parse(build_input)
except ParsingFailed as e:
self.logger.warning(
"Failed to parse build dependency of `{name}`".format(
name=self.name))
self.logger.warning(
"Possible reason: `{reason}`".format(reason=e.reason))
else:
if requirement.applies_to_target(target_platform):
requirement_set.add(requirement)
return requirement_set
def test_can_generate_valid_packages_with_extras_require(
package_generator: PackageGenerator,
requirement_parser: RequirementParser,
pip: Pip,
target_directory: Path,
install_target: Path,
current_platform: TargetPlatform,
):
package_generator.generate_setuptools_package(
name="testpackage", extras_require={"extra": ["extrapackage"]})
package_generator.generate_setuptools_package(name="extrapackage")
requirements = RequirementSet(target_platform=current_platform)
requirements.add(requirement_parser.parse("testpackage[extra]"))
pip.install(
requirements,
source_directories=[target_directory],
target_directory=install_target,
)
installed_packages = pip.freeze([install_target])
assert "extrapackage" in installed_packages
def _extract_deps(
constructor,
deps: Iterable[str],
target_platform: TargetPlatform,
requirement_parser: RequirementParser,
current_wheel_name: str,
) -> RequirementSet:
"""Get dependent packages from metadata.
Note that this is currently very rough stuff. I consider only the
first 'requires' dataset in 'run_requires'. Other requirement sets
like 'test_requires' are completely ignored.
"""
extracted_deps = RequirementSet(target_platform)
for dep_string in deps:
dependency = requirement_parser.parse(dep_string)
if not constructor._valid_dependency(current_wheel_name,
dependency.name()):
continue
extracted_deps.add(dependency)
return extracted_deps
def test_install_to_target_directory_does_not_install_to_default_directory(
pip: Pip,
project_dir: str,
download_dir: Path,
current_platform: TargetPlatform,
requirement_parser: RequirementParser,
):
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
target_directory = Path(project_dir) / "target-directory"
target_directory.ensure_directory()
pip.download_sources(requirements, download_dir)
assert not target_directory.list_files()
pip.install(
requirements,
source_directories=[download_dir],
target_directory=target_directory,
)
assert target_directory.list_files()
def build_dependencies_from_setup_cfg(
self, target_platform: TargetPlatform,
requirement_parser: RequirementParser) -> RequirementSet:
setup_requires = self.setup_cfg.get("options",
{}).get("setup_requires")
requirements = RequirementSet(target_platform)
if isinstance(setup_requires, str):
requirements.add(requirement_parser.parse(setup_requires))
elif isinstance(setup_requires, list):
for requirement_string in setup_requires:
try:
requirement = requirement_parser.parse(requirement_string)
except ParsingFailed as e:
self.logger.warning(
"Failed to parse build dependency of `{name}`".format(
name=self.name))
self.logger.warning(
"Possible reason: `{reason}`".format(reason=e.reason))
else:
if requirement.applies_to_target(target_platform):
requirements.add(requirement)
return requirements
def test_can_generate_valid_packages_with_two_runtime_dependencies(
package_generator: PackageGenerator,
requirement_parser: RequirementParser,
pip: Pip,
target_directory: Path,
install_target: Path,
current_platform: TargetPlatform,
):
package_generator.generate_setuptools_package(
name="testpackage", install_requires=["dependency1", "dependency2"])
package_generator.generate_setuptools_package(name="dependency1")
package_generator.generate_setuptools_package(name="dependency2")
requirements = RequirementSet(target_platform=current_platform)
requirements.add(requirement_parser.parse("testpackage"))
pip.install(
requirements,
source_directories=[target_directory],
target_directory=install_target,
)
installed_packages = pip.freeze([install_target])
assert "dependency1" in installed_packages
assert "dependency2" in installed_packages
