def test_sha256_sum_gives_correct_hash_for_empty_file( tmpdir_factory, text, expected_hash ): file_path = Path(str(tmpdir_factory.mktemp("sha256_example"))) / "empty.txt" file_path.write_text(text) assert file_path.sha256_sum() == expected_hash
def test_pip_downloads_sources_to_target_directory( pip: Pip, project_dir: str, current_platform: TargetPlatform, requirement_parser: RequirementParser, ): download_path = Path(project_dir) / "download" requirements = RequirementSet(current_platform) requirements.add(requirement_parser.parse("six")) pip.download_sources(requirements=requirements, target_directory=download_path) assert download_path.list_files()
def generate_setuptools_package( self, name: str, version: str = "1.0", install_requires: List[str] = []) -> SourceDistribution: with TemporaryDirectory() as directory_path_string: build_directory: Path = Path(directory_path_string) self._generate_setup_py(build_directory, name=name, version=version) self._generate_setup_cfg( build_directory, name=name, version=version, install_requires=install_requires, ) built_distribution_archive = self._build_package( build_directory=build_directory, name=name, version=version) source_distribution = SourceDistribution.from_archive( built_distribution_archive, logger=self._logger, requirement_parser=self._requirement_parser, ) self._move_package_target_directory(built_distribution_archive) return source_distribution
class Index: UrlEntry = namedtuple("UrlEntry", ["url", "sha256"]) GitEntry = namedtuple("GitEntry", ["url", "sha256", "rev"]) Entry = Union[UrlEntry, GitEntry] logger: Logger = attrib() path: Path = attrib(default=Path(os.path.dirname(__file__)) / "index.json", ) def __getitem__(self, key: str) -> "Index.Entry": with self._index_json() as index: entry = index[key] if self._is_schema_valid(entry, URL_SCHEMA): return Index.UrlEntry(url=entry["url"], sha256=entry["sha256"]) elif self._is_schema_valid(entry, GIT_SCHEMA): return Index.GitEntry(url=entry["url"], sha256=entry["sha256"], rev=entry["rev"]) else: raise Exception() def __setitem__(self, key: str, value: "Index.Entry") -> None: with self._index_json(write=True) as index: if isinstance(value, self.UrlEntry): index[key] = { "url": value.url, "sha256": value.sha256, "__type__": "fetchurl", } if isinstance(value, self.GitEntry): index[key] = { "url": value.url, "sha256": value.sha256, "rev": value.rev, "__type__": "fetchgit", } def is_valid(self) -> bool: with self._index_json() as index: return self._is_schema_valid(index, INDEX_SCHEMA) @contextmanager def _index_json(self, write: bool = False ) -> Iterator[Dict[str, Dict[str, str]]]: with open(str(self.path)) as f: index = json.load(f) yield index if write: with open(str(self.path), "w") as f: json.dump(index, f, sort_keys=True, indent=4) def _is_schema_valid(self, json_value: Any, schema: Any) -> bool: try: validate(json_value, schema) except ValidationError as e: self.logger.error(str(e)) return False else: return True
def pip( request, nix: Nix, project_dir: str, current_platform: TargetPlatform, logger: Logger, requirement_parser: RequirementParser, ): if request.param == "nix": return NixPip( nix=nix, project_directory=Path(project_dir), extra_build_inputs=[], extra_env="", wheels_cache=[], target_platform=current_platform, logger=logger, requirement_parser=requirement_parser, ) else: pip = VirtualenvPip( logger=logger, target_platform=current_platform, target_directory=os.path.join(project_dir, "venv-pip"), env_builder=venv.EnvBuilder(with_pip=True), requirement_parser=requirement_parser, ) pip.prepare_virtualenv() return pip
def test_flake_renderer_creates_flake_nix_file(flake_renderer: FlakeRenderer, flake_path: Path): flake_renderer.render_expression( packages_metadata=[], sources=Sources(), ) assert flake_path.is_file()
def _generate_setup_py(self, target_directory: Path, name: str, version: str) -> None: content = render_template( Path("setup.py"), context={}, ) (target_directory / "setup.py").write_text(content)
def test_pip_can_install_wheels_previously_downloaded( pip: Pip, project_dir: str, current_platform: TargetPlatform, requirement_parser: RequirementParser, download_dir: Path, wheels_dir: Path, ): requirements = RequirementSet(current_platform) requirements.add(requirement_parser.parse("six")) pip.download_sources(requirements, download_dir) pip.build_wheels( requirements=requirements, source_directories=[download_dir], target_directory=wheels_dir, ) assert wheels_dir.list_files() assert any(map(lambda x: x.endswith(".whl"), wheels_dir.list_files()))
def test_pip_wheel_does_not_build_wheels_if_requirements_are_empty( pip: Pip, wheels_dir: Path, download_dir: Path, current_platform: TargetPlatform ): pip.build_wheels( requirements=RequirementSet(current_platform), target_directory=wheels_dir, source_directories=[download_dir], ) assert not wheels_dir.list_files()
def test_that_path_paths_from_requirement_files_are_preserved_in_sources( collector: RequirementsCollector, tmpdir: Any) -> None: with current_working_directory(str(tmpdir)): requirements_file_path = tmpdir.join("requirements.txt") with open(requirements_file_path, "w") as f: print("path/to/egg#egg=testegg", file=f) collector.add_file(str(requirements_file_path)) testegg_source = collector.sources()["testegg"] assert isinstance(testegg_source, PathSource) assert testegg_source.path == Path("path/to/egg")
def test_install_six_yields_non_empty_freeze_output( pip: Pip, project_dir: str, download_dir: Path, current_platform: TargetPlatform, requirement_parser, ): lib_dir = Path(os.path.join(project_dir, "lib")) requirements = RequirementSet(current_platform) requirements.add(requirement_parser.parse("six")) pip.download_sources(requirements, download_dir) pip.install(requirements, source_directories=[download_dir], target_directory=lib_dir) assert pip.freeze([lib_dir])
def test_can_build_a_flake(tmpdir, nix: Nix): build_dir: Path = Path(str(tmpdir)) out_link_path = build_dir / "result" flake_path = build_dir / "flake.nix" flake_path.write_text( """{ description = "A flake for building Hello World"; inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03; outputs = { self, nixpkgs }: { defaultPackage.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.hello; }; }""" ) nix.build_flake(flake_path / "..", out_link=out_link_path) completed_process = subprocess.run(str(out_link_path / "bin" / "hello")) assert completed_process.returncode == 0
def _generate_setup_cfg( self, target_directory: Path, name: str, version: str, install_requires: List[str], ) -> None: content = render_template( Path("setup.cfg"), context={ "name": name, "version": version, "install_requires": install_requires, }, ) (target_directory / "setup.cfg").write_text(content)
class PathSource: def __init__(self, path: str) -> None: self.path = Path(path) @property def _normalized_path(self) -> Path: return self.path.resolve() def sha256(self) -> str: return self._normalized_path.sha256_sum() def nix_expression(self) -> str: return " ".join([ "builtins.fetchurl {", f'url = "file://{self._normalized_path}";', f'sha256 = "{self.sha256()}";', "}", ])
def wheel_builder( pip: Pip, project_dir: str, logger: Logger, requirement_parser: RequirementParser, current_platform: TargetPlatform, base_dependency_graph: DependencyGraph, ) -> WheelBuilder: base_dir = Path(project_dir) return WheelBuilder( pip=pip, download_directory=base_dir / "downloads", lib_directory=base_dir / "lib", wheel_directory=base_dir / "wheels", extracted_wheel_directory=base_dir / "extracted-wheels", logger=logger, requirement_parser=requirement_parser, target_platform=current_platform, base_dependency_graph=base_dependency_graph, )
def generate_setuptools_package( self, name: str, version: str = "1.0", install_requires: List[str] = [], extras_require: Dict[str, List[str]] = {}, ) -> SourceDistribution: with TemporaryDirectory() as directory_path_string: build_directory: Path = Path(directory_path_string) self._generate_setup_py(build_directory, name=name, version=version) self._generate_setup_cfg( build_directory, name=name, version=version, install_requires=install_requires, extras_require=extras_require, ) self._generate_python_module( build_directory, name=name, ) built_distribution_archive = self._build_package( build_directory=build_directory, name=name, version=version) source_distribution = SourceDistribution.from_archive( built_distribution_archive, logger=self._logger, requirement_parser=self._requirement_parser, ) self._move_package_target_directory(built_distribution_archive) self._sources.add( name=name, source=PathSource( path=str(self._get_distribution_path(name, version))), ) return source_distribution
def build_wheel(target_directory: Path, requirement: str) -> str: logger = StreamLogger(sys.stdout) requirement_parser = RequirementParser(logger=logger) package_directory: str = str(ROOT / "unittests" / "data") escaped_requirement = shlex.quote(requirement) target_directory = target_directory.resolve() with tempfile.TemporaryDirectory() as build_directory: os.chdir(str(build_directory)) nix = Nix(logger=logger) nix.shell( command= f"pip wheel {escaped_requirement} --find-links {str(package_directory)} --no-deps", derivation_path=DERIVATION_PATH, nix_arguments=dict(), ) try: parsed_requirement = requirement_parser.parse(requirement) except ParsingFailed: for path in os.listdir("."): if path.endswith(".whl"): wheel_path = path break else: raise Exception("Build process did not produce .whl file") else: for path in os.listdir("."): if path.endswith(".whl") and parsed_requirement.name() in path: wheel_path = path break else: raise Exception("Build process did not produce .whl file") target_file_name = os.path.basename(wheel_path) target_path = target_directory / target_file_name shutil.move(wheel_path, str(target_path)) return target_file_name
def test_install_to_target_directory_does_not_install_to_default_directory( pip: Pip, project_dir: str, download_dir: Path, current_platform: TargetPlatform, requirement_parser: RequirementParser, ): requirements = RequirementSet(current_platform) requirements.add(requirement_parser.parse("six")) target_directory = Path(project_dir) / "target-directory" target_directory.ensure_directory() pip.download_sources(requirements, download_dir) assert not target_directory.list_files() pip.install( requirements, source_directories=[download_dir], target_directory=target_directory, ) assert target_directory.list_files()
def test_install_does_not_install_anything_with_empty_requirements( pip: Pip, project_dir: str, current_platform: TargetPlatform): target_directory = Path(project_dir) / "target_dir" target_directory.ensure_directory() pip.install(RequirementSet(current_platform), [], target_directory) assert not target_directory.list_files()
def wheels_dir(project_dir: str) -> Path: path = os.path.join(project_dir, "wheels") os.makedirs(path) return Path(path)
def _output_files(self) -> Set[Path]: return { Path(self.example_directory()) / "requirements.nix", Path(self.flake_path()), }
def package_source_directory(tmpdir_factory: Any) -> Path: path_as_str: str = str(tmpdir_factory.mktemp("package_source_directory")) return Path(path_as_str)
def download_dir(project_dir: str) -> Path: path: str = os.path.join(project_dir, "download") os.makedirs(path) return Path(path)
def render_expression( self, packages_metadata: Iterable[Wheel], sources: Sources, ) -> None: """Create Nix expressions. """ default_file = os.path.join(self._target_directory, f"{self._requirements_name}.nix") overrides_file = os.path.join( self._target_directory, f"{self._requirements_name}_override.nix") metadata_by_name: Dict[str, Wheel] = {x.name: x for x in packages_metadata} generated_packages_metadata = [] for item in sorted(packages_metadata, key=lambda x: x.name): if item.build_dependencies: buildInputs = "\n".join( sorted([ ' self."{}"'.format(dependency.name()) for dependency in item.build_dependencies( self._target_platform) ])) buildInputs = "[\n" + buildInputs + "\n ]" else: buildInputs = "[ ]" propagatedBuildInputs = "[ ]" dependencies = item.dependencies(extras=[]) if dependencies: deps = [ x.name() for x in dependencies if x.name() in metadata_by_name.keys() ] if deps: propagatedBuildInputs = "[\n%s\n ]" % ("\n".join( sorted([ ' self."%s"' % (metadata_by_name[x].name) for x in deps if x != item.name ]))) source = sources[item.name] fetch_expression = source.nix_expression() package_format = item.package_format generated_packages_metadata.append( dict( name=item.name, version=item.version, fetch_expression=fetch_expression, buildInputs=buildInputs, propagatedBuildInputs=propagatedBuildInputs, homepage=item.homepage, license=item.license, description=escape_string(item.description), package_format=package_format, )) generated_template = TEMPLATES.get_template("generated.nix.j2") generated = "\n\n".join( generated_template.render(**x) for x in generated_packages_metadata) overrides = TEMPLATES.get_template("overrides.nix.j2").render() common_overrides_expressions = [ " (" + override.nix_expression(self._logger) + ")" for override in self._common_overrides ] default_template = TEMPLATES.get_template("requirements.nix.j2") overrides_file_nix_path = os.path.join( ".", os.path.split(overrides_file)[1]) default = default_template.render( version=pypi2nix_version, command_arguments=" ".join(map(shlex.quote, sys.argv[1:])), python_version=self._python_version.derivation_name(), extra_build_inputs=(self._extra_build_inputs and "with pkgs; [ %s ]" % (" ".join(self._extra_build_inputs)) or "[]"), overrides_file=overrides_file_nix_path, enable_tests="false", generated_package_nix=generated, common_overrides="\n".join(common_overrides_expressions), python_major_version=self._python_version.major_version(), ) if not os.path.exists(overrides_file): with open(overrides_file, "w+") as f: f.write(overrides.strip()) self._logger.info("|-> writing %s" % overrides_file) with open(default_file, "w+") as f: f.write(default.strip()) with open(self._frozen_file, "w+") as f: f.write(self._requirements_frozen) self._code_formatter.format_file(Path(default_file)) self._code_formatter.format_file(Path(overrides_file))
def path_source(tmpdir_factory): path = Path(str(tmpdir_factory.mktemp("path_source") / "test.txt")) path.write_text("") return PathSource(str(path))
import os from typing import Dict import jinja2 from pypi2nix.path import Path HERE = Path(os.path.dirname(__file__)) _templates = jinja2.Environment(loader=jinja2.FileSystemLoader(str(HERE / "templates"))) def render_template(template_path: Path, context=Dict[str, str]) -> str: template = _templates.get_template(str(template_path)) return template.render(**context)
def install_target(tmpdir_factory) -> Path: return Path(str(tmpdir_factory.mktemp("install-target")))
def target_directory(tmpdir_factory) -> Path: return Path(str(tmpdir_factory.mktemp("target-directory")))
def find_root(start: Path = Path(".")) -> Path: absolute_location = start.resolve() if (absolute_location / ".git").is_directory(): return absolute_location else: return find_root(absolute_location / "..")
def run(self) -> None: requirements = self.requirements_collector().requirements() self.logger().info("pypi2nix v{} running ...".format(pypi2nix_version)) if not requirements: self.logger().info( "No requirements were specified. Ending program.") return setup_requirements = self.setup_requirements_collector().requirements() requirements_name = os.path.join(self.configuration.target_directory, self.configuration.output_basename) sources = Sources() sources.update(setup_requirements.sources()) sources.update(requirements.sources()) sources.update(self.setup_requirements_collector().sources()) sources.update(self.requirements_collector().sources()) self.logger().info("Downloading wheels and creating wheelhouse ...") pip = NixPip( nix=self.nix(), project_directory=self.configuration.project_directory, extra_env=self.configuration.extra_environment, extra_build_inputs=self._extra_build_inputs(), wheels_cache=self.configuration.wheels_caches, target_platform=self.target_platform(), logger=self.logger(), requirement_parser=self.requirement_parser(), ) wheel_builder = WheelBuilder( pip=pip, download_directory=self.configuration.project_directory / "downloads", lib_directory=self.configuration.project_directory / "lib", extracted_wheel_directory=self.configuration.project_directory / "extracted-wheels", wheel_directory=self.configuration.project_directory / "wheels", logger=self.logger(), requirement_parser=self.requirement_parser(), target_platform=self.target_platform(), base_dependency_graph=self.base_dependency_graph(), ) wheels = wheel_builder.build(requirements=requirements, setup_requirements=setup_requirements) requirements_frozen = wheel_builder.get_frozen_requirements() source_distributions = wheel_builder.source_distributions self.logger().info("Extracting metadata from pypi.python.org ...") metadata_fetcher = MetadataFetcher( sources=sources, logger=self.logger(), requirement_parser=self.requirement_parser(), pypi=Pypi(logger=self.logger()), ) packages_metadata = metadata_fetcher.main( wheel_paths=wheels, target_platform=self.target_platform(), source_distributions=source_distributions, ) self.logger().info("Generating Nix expressions ...") renderers: List[ExpressionRenderer] = [] renderers.append( RequirementsRenderer( requirements_name=requirements_name, extra_build_inputs=self.extra_build_inputs(), python_version=self.configuration.python_version, target_directory=self.configuration.target_directory, logger=self.logger(), common_overrides=self.configuration.overrides, target_platform=self.target_platform(), requirements_frozen=requirements_frozen, code_formatter=self.code_formatter(), )) renderers.append( FlakeRenderer( target_path=Path(self.configuration.target_directory) / "flake.nix", target_platform=self.target_platform(), logger=self.logger(), overrides=self.configuration.overrides, extra_build_inputs=self.extra_build_inputs(), code_formatter=self.code_formatter(), )) for renderer in renderers: renderer.render_expression( packages_metadata=packages_metadata, sources=sources, ) if self.configuration.dependency_graph_output_location: dependency_graph = DependencyGraph() for wheel in packages_metadata: dependency_graph.import_wheel(wheel, self.requirement_parser()) with open(str(self.configuration.dependency_graph_output_location), "w") as output_file: output_file.write(dependency_graph.serialize()) self.print_user_information()