def test_can_set_buildtime_dependency(package_a: Requirement,
package_b: Requirement,
dependency_graph: DependencyGraph):
dependency_graph.set_buildtime_dependency(dependent=package_a,
dependency=package_b)
assert dependency_graph.is_buildtime_dependency(dependent=package_a,
dependency=package_b)
def test_cyclic_runtime_dependencies_not_allowed(
package_a: Requirement, package_b: Requirement,
dependency_graph: DependencyGraph):
dependency_graph.set_runtime_dependency(dependent=package_a,
dependency=package_b)
with pytest.raises(CyclicDependencyOccured):
dependency_graph.set_runtime_dependency(dependent=package_b,
dependency=package_a)
def test_can_detect_external_dependencies_for_packages(
package_a: Requirement,
external_dependency_a: ExternalDependency,
dependency_graph: DependencyGraph,
):
dependency_graph.set_external_dependency(dependent=package_a,
dependency=external_dependency_a)
assert dependency_graph.get_all_external_dependencies(package_a) == {
external_dependency_a,
}
def test_cannot_add_circular_buildtime_dependencies(
package_a: Requirement,
package_b: Requirement,
dependency_graph: DependencyGraph,
):
dependency_graph.set_buildtime_dependency(dependent=package_a,
dependency=package_b)
with pytest.raises(CyclicDependencyOccured):
dependency_graph.set_buildtime_dependency(dependent=package_b,
dependency=package_a)
def test_external_dependencies_from_graph_are_retrieved(
requirement_parser: RequirementParser, ) -> None:
dependency_graph = DependencyGraph()
requirement = requirement_parser.parse("testpackage")
external_dependency = ExternalDependency("external")
dependency_graph.set_external_dependency(dependent=requirement,
dependency=external_dependency)
retriever = RequirementDependencyRetriever(dependency_graph)
assert external_dependency in retriever.get_external_dependency_for_requirement(
requirement)
def test_build_time_dependencies_dont_show_up_as_runtime_dependencies(
package_a: Requirement,
package_b: Requirement,
package_c: Requirement,
dependency_graph: DependencyGraph,
):
dependency_graph.set_runtime_dependency(dependent=package_a,
dependency=package_b)
dependency_graph.set_buildtime_dependency(dependent=package_b,
dependency=package_c)
assert not dependency_graph.is_runtime_dependency(dependent=package_a,
dependency=package_c)
def test_can_detect_indirect_runtime_dependencies(
package_a: Requirement,
package_b: Requirement,
package_c: Requirement,
dependency_graph: DependencyGraph,
) -> None:
dependency_graph.set_runtime_dependency(dependent=package_a,
dependency=package_b)
dependency_graph.set_runtime_dependency(dependent=package_b,
dependency=package_c)
assert dependency_graph.is_runtime_dependency(dependent=package_a,
dependency=package_c)
def test_can_retrieve_external_dependencies_from_runtime_dependencies(
package_a: Requirement,
package_b: Requirement,
external_dependency_a: ExternalDependency,
dependency_graph: DependencyGraph,
):
dependency_graph.set_runtime_dependency(dependent=package_a,
dependency=package_b)
dependency_graph.set_external_dependency(dependent=package_b,
dependency=external_dependency_a)
assert dependency_graph.get_all_external_dependencies(
package=package_a) == {
external_dependency_a,
}
def dependency_graph(
draw,
python_packages=lists(requirement(), unique_by=lambda x: x.name()),
external_dependencies=lists(external_dependency()),
is_runtime_dependency=booleans(),
selections=integers(),
):
graph = DependencyGraph()
packages = draw(python_packages)
if not packages:
return graph
for package in packages:
index = draw(selections) % len(packages)
try:
if draw(is_runtime_dependency):
graph.set_runtime_dependency(dependent=package,
dependency=packages[index])
else:
graph.set_buildtime_dependency(dependent=package,
dependency=packages[index])
except CyclicDependencyOccured:
continue
for dependency in draw(external_dependencies):
graph.set_external_dependency(dependent=packages[draw(selections) %
len(packages)],
dependency=dependency)
return graph
def test_can_retriev_all_runtime_dependency_names(
package_a: Requirement,
package_b: Requirement,
package_c: Requirement,
dependency_graph: DependencyGraph,
) -> None:
dependency_graph.set_runtime_dependency(dependent=package_a,
dependency=package_b)
dependency_graph.set_runtime_dependency(dependent=package_b,
dependency=package_c)
assert dependency_graph.get_all_runtime_dependency_names(package_a) == {
package_a.name(),
package_b.name(),
package_c.name(),
}
def check_dependency_graph(self, dependency_graph: DependencyGraph,
requirement_parser: RequirementParser):
self.assertTrue(
dependency_graph.is_runtime_dependency(
requirement_parser.parse("django"),
requirement_parser.parse("pytz"),
))
def test_that_we_can_generate_an_empty_requirement_set_from_freshly_constructed_collector(
current_platform, requirement_parser, logger: Logger,
project_dir: str):
collector = RequirementsCollector(current_platform, requirement_parser,
logger, project_dir, DependencyGraph())
requirements = collector.requirements()
assert len(requirements) == 0
def collector(
current_platform: TargetPlatform,
requirement_parser: RequirementParser,
logger: Logger,
project_dir: str,
) -> RequirementsCollector:
return RequirementsCollector(current_platform, requirement_parser, logger,
project_dir, DependencyGraph())
def test_can_add_two_dependencies_graphs_and_buildtime_dependencies_are_also_added(
package_a: Requirement,
package_b: Requirement,
package_c: Requirement,
dependency_graph: DependencyGraph,
):
other_dependency_graph = copy(dependency_graph)
dependency_graph.set_buildtime_dependency(dependent=package_a,
dependency=package_b)
other_dependency_graph.set_buildtime_dependency(dependent=package_b,
dependency=package_c)
sum_graph = dependency_graph + other_dependency_graph
assert not dependency_graph.is_buildtime_dependency(dependent=package_a,
dependency=package_c)
assert not other_dependency_graph.is_buildtime_dependency(
dependent=package_a, dependency=package_c)
assert sum_graph.is_buildtime_dependency(dependent=package_a,
dependency=package_c)
def test_can_retrieve_external_dependencies_after_adding_graphs(
package_a: Requirement,
package_b: Requirement,
external_dependency_a: ExternalDependency,
external_dependency_b: ExternalDependency,
dependency_graph: DependencyGraph,
):
other_dependency_graph = copy(dependency_graph)
dependency_graph.set_external_dependency(dependent=package_a,
dependency=external_dependency_a)
dependency_graph.set_runtime_dependency(dependent=package_a,
dependency=package_b)
other_dependency_graph.set_external_dependency(
dependent=package_b, dependency=external_dependency_b)
sum_graph = dependency_graph + other_dependency_graph
assert sum_graph.get_all_external_dependencies(package=package_a) == {
external_dependency_a,
external_dependency_b,
}
def setup_requirements_collector(self) -> RequirementsCollector:
setup_requirement_collector = RequirementsCollector(
self.target_platform(),
self.requirement_parser(),
self.logger(),
str(self.configuration.project_directory),
DependencyGraph(),
)
for build_input in self.configuration.setup_requirements:
setup_requirement_collector.add_line(build_input)
return setup_requirement_collector
def test_can_understand_wheel_dependecies(
current_platform: TargetPlatform,
requirement_parser: RequirementParser):
runtime_dependencies = RequirementSet(current_platform)
runtime_dependency = requirement_parser.parse("runtime_dependency")
runtime_dependencies.add(runtime_dependency)
build_dependencies = RequirementSet(current_platform)
build_dependency = requirement_parser.parse("build_dependency")
build_dependencies.add(build_dependency)
wheel = Wheel(
name="testpackage",
version="",
deps=runtime_dependencies,
target_platform=current_platform,
license="",
homepage="",
description="",
build_dependencies=build_dependencies,
)
requirement = requirement_parser.parse("testpackage")
dependency_graph = DependencyGraph()
dependency_graph.import_wheel(wheel, requirement_parser)
assert dependency_graph.is_buildtime_dependency(requirement,
build_dependency)
assert dependency_graph.is_runtime_dependency(requirement,
runtime_dependency)
def run(self) -> None:
requirements = self.requirements_collector().requirements()
self.logger().info("pypi2nix v{} running ...".format(pypi2nix_version))
if not requirements:
self.logger().info(
"No requirements were specified. Ending program.")
return
setup_requirements = self.setup_requirements_collector().requirements()
requirements_name = os.path.join(self.configuration.target_directory,
self.configuration.output_basename)
sources = Sources()
sources.update(setup_requirements.sources())
sources.update(requirements.sources())
sources.update(self.setup_requirements_collector().sources())
sources.update(self.requirements_collector().sources())
self.logger().info("Downloading wheels and creating wheelhouse ...")
pip = NixPip(
nix=self.nix(),
project_directory=self.configuration.project_directory,
extra_env=self.configuration.extra_environment,
extra_build_inputs=self._extra_build_inputs(),
wheels_cache=self.configuration.wheels_caches,
target_platform=self.target_platform(),
logger=self.logger(),
requirement_parser=self.requirement_parser(),
)
wheel_builder = WheelBuilder(
pip=pip,
download_directory=self.configuration.project_directory /
"downloads",
lib_directory=self.configuration.project_directory / "lib",
extracted_wheel_directory=self.configuration.project_directory /
"extracted-wheels",
wheel_directory=self.configuration.project_directory / "wheels",
logger=self.logger(),
requirement_parser=self.requirement_parser(),
target_platform=self.target_platform(),
base_dependency_graph=self.base_dependency_graph(),
)
wheels = wheel_builder.build(requirements=requirements,
setup_requirements=setup_requirements)
requirements_frozen = wheel_builder.get_frozen_requirements()
source_distributions = wheel_builder.source_distributions
self.logger().info("Extracting metadata from pypi.python.org ...")
metadata_fetcher = MetadataFetcher(
sources=sources,
logger=self.logger(),
requirement_parser=self.requirement_parser(),
pypi=Pypi(logger=self.logger()),
)
packages_metadata = metadata_fetcher.main(
wheel_paths=wheels,
target_platform=self.target_platform(),
source_distributions=source_distributions,
)
self.logger().info("Generating Nix expressions ...")
renderers: List[ExpressionRenderer] = []
renderers.append(
RequirementsRenderer(
requirements_name=requirements_name,
extra_build_inputs=self.extra_build_inputs(),
python_version=self.configuration.python_version,
target_directory=self.configuration.target_directory,
logger=self.logger(),
common_overrides=self.configuration.overrides,
target_platform=self.target_platform(),
requirements_frozen=requirements_frozen,
code_formatter=self.code_formatter(),
))
renderers.append(
FlakeRenderer(
target_path=Path(self.configuration.target_directory) /
"flake.nix",
target_platform=self.target_platform(),
logger=self.logger(),
overrides=self.configuration.overrides,
extra_build_inputs=self.extra_build_inputs(),
code_formatter=self.code_formatter(),
))
for renderer in renderers:
renderer.render_expression(
packages_metadata=packages_metadata,
sources=sources,
)
if self.configuration.dependency_graph_output_location:
dependency_graph = DependencyGraph()
for wheel in packages_metadata:
dependency_graph.import_wheel(wheel, self.requirement_parser())
with open(str(self.configuration.dependency_graph_output_location),
"w") as output_file:
output_file.write(dependency_graph.serialize())
self.print_user_information()
def _read_dependency_graph(self) -> DependencyGraph:
with open(self._dependency_graph_output_path()) as f:
return DependencyGraph.deserialize(f.read())
def base_dependency_graph() -> DependencyGraph:
return DependencyGraph()
def collector(current_platform, requirement_parser, logger: Logger,
project_dir: str):
return RequirementsCollector(current_platform, requirement_parser, logger,
project_dir, DependencyGraph())
def test_no_external_dependency_for_empty_dependency_graph(
requirement_parser: RequirementParser, ) -> None:
dependency_graph = DependencyGraph()
retriever = RequirementDependencyRetriever(dependency_graph)
requirement = requirement_parser.parse("testpackage")
assert not retriever.get_external_dependency_for_requirement(requirement)
def test_serialization_and_deserialization_leads_to_identity(
dependency_graph: DependencyGraph, ):
assert DependencyGraph.deserialize(
dependency_graph.serialize()) == dependency_graph
def test_equality_of_empty_graphs():
assert DependencyGraph() == DependencyGraph()
def main(
version: str,
verbose: int,
quiet: int,
nix_shell: Optional[str],
nix_path: List[str],
basename: str,
extra_build_inputs: List[str],
emit_extra_build_inputs: bool,
extra_env: str,
enable_tests: bool,
python_version_argument: str,
requirements: List[str],
editable: List[str],
setup_requires: List[str],
overrides: List[NetworkFile],
default_overrides: bool,
wheels_cache: List[str],
build_directory: Optional[str],
dependency_graph_output: Optional[str],
dependency_graph_input: Optional[NetworkFile],
) -> None:
overrides_list: List[Overrides] = []
if version:
click.echo(pypi2nix_version)
exit(0)
verbosity = verbosity_from_int(verbose - quiet + DEFAULT_VERBOSITY)
nix_executable_directory: Optional[str]
if nix_shell is None:
nix_executable_directory = None
else:
if not os.path.isfile(nix_shell):
raise click.exceptions.UsageError(
f"Specified `nix-shell` executable `{nix_shell}` does not exist."
)
else:
nix_executable_directory = os.path.dirname(
os.path.abspath(nix_shell))
overrides_list += [
OverridesNetworkFile(network_file) for network_file in overrides
]
if default_overrides:
overrides_list += tuple([
OverridesGithub(
owner="nix-community",
repo="pypi2nix-overrides",
path="overrides.nix",
)
])
python_version = getattr(PythonVersion, python_version_argument, None)
if python_version is None:
raise click.exceptions.UsageError(
f"Python version `{python_version_argument}` not available")
project_directory_context: ProjectDirectory = (
TemporaryProjectDirectory() if build_directory is None else
PersistentProjectDirectory(path=build_directory))
if dependency_graph_input:
dependency_graph = DependencyGraph.deserialize(
dependency_graph_input.fetch())
else:
dependency_graph = DependencyGraph()
with project_directory_context as _project_directory:
configuration = ApplicationConfiguration(
emit_extra_build_inputs=emit_extra_build_inputs,
enable_tests=enable_tests,
extra_build_inputs=args_as_list(extra_build_inputs),
extra_environment=extra_env,
nix_executable_directory=nix_executable_directory,
nix_path=nix_path,
output_basename=basename,
overrides=overrides_list,
python_version=python_version,
requirement_files=requirements,
requirements=editable,
setup_requirements=setup_requires,
verbosity=verbosity,
wheels_caches=wheels_cache,
project_directory=_project_directory,
target_directory=os.getcwd(),
dependency_graph_output_location=Path(dependency_graph_output)
if dependency_graph_output else None,
dependency_graph_input=dependency_graph,
)
Pypi2nix(configuration).run()
def dependency_graph() -> DependencyGraph:
return DependencyGraph()
def __init__(self, dependency_graph: DependencyGraph = DependencyGraph()):
self._dependency_graph = dependency_graph
