def test_sha256_sum_gives_correct_hash_for_empty_file(
tmpdir_factory, text, expected_hash
):
file_path = Path(str(tmpdir_factory.mktemp("sha256_example"))) / "empty.txt"
file_path.write_text(text)
assert file_path.sha256_sum() == expected_hash
def test_pip_downloads_sources_to_target_directory(
pip: Pip,
project_dir: str,
current_platform: TargetPlatform,
requirement_parser: RequirementParser,
):
download_path = Path(project_dir) / "download"
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip.download_sources(requirements=requirements,
target_directory=download_path)
assert download_path.list_files()
def generate_setuptools_package(
self,
name: str,
version: str = "1.0",
install_requires: List[str] = []) -> SourceDistribution:
with TemporaryDirectory() as directory_path_string:
build_directory: Path = Path(directory_path_string)
self._generate_setup_py(build_directory,
name=name,
version=version)
self._generate_setup_cfg(
build_directory,
name=name,
version=version,
install_requires=install_requires,
)
built_distribution_archive = self._build_package(
build_directory=build_directory, name=name, version=version)
source_distribution = SourceDistribution.from_archive(
built_distribution_archive,
logger=self._logger,
requirement_parser=self._requirement_parser,
)
self._move_package_target_directory(built_distribution_archive)
return source_distribution
class Index:
UrlEntry = namedtuple("UrlEntry", ["url", "sha256"])
GitEntry = namedtuple("GitEntry", ["url", "sha256", "rev"])
Entry = Union[UrlEntry, GitEntry]
logger: Logger = attrib()
path: Path = attrib(default=Path(os.path.dirname(__file__)) /
"index.json", )
def __getitem__(self, key: str) -> "Index.Entry":
with self._index_json() as index:
entry = index[key]
if self._is_schema_valid(entry, URL_SCHEMA):
return Index.UrlEntry(url=entry["url"], sha256=entry["sha256"])
elif self._is_schema_valid(entry, GIT_SCHEMA):
return Index.GitEntry(url=entry["url"],
sha256=entry["sha256"],
rev=entry["rev"])
else:
raise Exception()
def __setitem__(self, key: str, value: "Index.Entry") -> None:
with self._index_json(write=True) as index:
if isinstance(value, self.UrlEntry):
index[key] = {
"url": value.url,
"sha256": value.sha256,
"__type__": "fetchurl",
}
if isinstance(value, self.GitEntry):
index[key] = {
"url": value.url,
"sha256": value.sha256,
"rev": value.rev,
"__type__": "fetchgit",
}
def is_valid(self) -> bool:
with self._index_json() as index:
return self._is_schema_valid(index, INDEX_SCHEMA)
@contextmanager
def _index_json(self,
write: bool = False
) -> Iterator[Dict[str, Dict[str, str]]]:
with open(str(self.path)) as f:
index = json.load(f)
yield index
if write:
with open(str(self.path), "w") as f:
json.dump(index, f, sort_keys=True, indent=4)
def _is_schema_valid(self, json_value: Any, schema: Any) -> bool:
try:
validate(json_value, schema)
except ValidationError as e:
self.logger.error(str(e))
return False
else:
return True
def pip(
request,
nix: Nix,
project_dir: str,
current_platform: TargetPlatform,
logger: Logger,
requirement_parser: RequirementParser,
):
if request.param == "nix":
return NixPip(
nix=nix,
project_directory=Path(project_dir),
extra_build_inputs=[],
extra_env="",
wheels_cache=[],
target_platform=current_platform,
logger=logger,
requirement_parser=requirement_parser,
)
else:
pip = VirtualenvPip(
logger=logger,
target_platform=current_platform,
target_directory=os.path.join(project_dir, "venv-pip"),
env_builder=venv.EnvBuilder(with_pip=True),
requirement_parser=requirement_parser,
)
pip.prepare_virtualenv()
return pip
def test_flake_renderer_creates_flake_nix_file(flake_renderer: FlakeRenderer,
flake_path: Path):
flake_renderer.render_expression(
packages_metadata=[],
sources=Sources(),
)
assert flake_path.is_file()
def _generate_setup_py(self, target_directory: Path, name: str,
version: str) -> None:
content = render_template(
Path("setup.py"),
context={},
)
(target_directory / "setup.py").write_text(content)
def test_pip_can_install_wheels_previously_downloaded(
pip: Pip,
project_dir: str,
current_platform: TargetPlatform,
requirement_parser: RequirementParser,
download_dir: Path,
wheels_dir: Path,
):
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip.download_sources(requirements, download_dir)
pip.build_wheels(
requirements=requirements,
source_directories=[download_dir],
target_directory=wheels_dir,
)
assert wheels_dir.list_files()
assert any(map(lambda x: x.endswith(".whl"), wheels_dir.list_files()))
def test_pip_wheel_does_not_build_wheels_if_requirements_are_empty(
pip: Pip, wheels_dir: Path, download_dir: Path, current_platform: TargetPlatform
):
pip.build_wheels(
requirements=RequirementSet(current_platform),
target_directory=wheels_dir,
source_directories=[download_dir],
)
assert not wheels_dir.list_files()
def test_that_path_paths_from_requirement_files_are_preserved_in_sources(
collector: RequirementsCollector, tmpdir: Any) -> None:
with current_working_directory(str(tmpdir)):
requirements_file_path = tmpdir.join("requirements.txt")
with open(requirements_file_path, "w") as f:
print("path/to/egg#egg=testegg", file=f)
collector.add_file(str(requirements_file_path))
testegg_source = collector.sources()["testegg"]
assert isinstance(testegg_source, PathSource)
assert testegg_source.path == Path("path/to/egg")
def test_install_six_yields_non_empty_freeze_output(
pip: Pip,
project_dir: str,
download_dir: Path,
current_platform: TargetPlatform,
requirement_parser,
):
lib_dir = Path(os.path.join(project_dir, "lib"))
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
pip.download_sources(requirements, download_dir)
pip.install(requirements,
source_directories=[download_dir],
target_directory=lib_dir)
assert pip.freeze([lib_dir])
def test_can_build_a_flake(tmpdir, nix: Nix):
build_dir: Path = Path(str(tmpdir))
out_link_path = build_dir / "result"
flake_path = build_dir / "flake.nix"
flake_path.write_text(
"""{
description = "A flake for building Hello World";
inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03;
outputs = { self, nixpkgs }: {
defaultPackage.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.hello;
};
}"""
)
nix.build_flake(flake_path / "..", out_link=out_link_path)
completed_process = subprocess.run(str(out_link_path / "bin" / "hello"))
assert completed_process.returncode == 0
def _generate_setup_cfg(
self,
target_directory: Path,
name: str,
version: str,
install_requires: List[str],
) -> None:
content = render_template(
Path("setup.cfg"),
context={
"name": name,
"version": version,
"install_requires": install_requires,
},
)
(target_directory / "setup.cfg").write_text(content)
class PathSource:
def __init__(self, path: str) -> None:
self.path = Path(path)
@property
def _normalized_path(self) -> Path:
return self.path.resolve()
def sha256(self) -> str:
return self._normalized_path.sha256_sum()
def nix_expression(self) -> str:
return " ".join([
"builtins.fetchurl {",
f'url = "file://{self._normalized_path}";',
f'sha256 = "{self.sha256()}";',
"}",
])
def wheel_builder(
pip: Pip,
project_dir: str,
logger: Logger,
requirement_parser: RequirementParser,
current_platform: TargetPlatform,
base_dependency_graph: DependencyGraph,
) -> WheelBuilder:
base_dir = Path(project_dir)
return WheelBuilder(
pip=pip,
download_directory=base_dir / "downloads",
lib_directory=base_dir / "lib",
wheel_directory=base_dir / "wheels",
extracted_wheel_directory=base_dir / "extracted-wheels",
logger=logger,
requirement_parser=requirement_parser,
target_platform=current_platform,
base_dependency_graph=base_dependency_graph,
)
def generate_setuptools_package(
self,
name: str,
version: str = "1.0",
install_requires: List[str] = [],
extras_require: Dict[str, List[str]] = {},
) -> SourceDistribution:
with TemporaryDirectory() as directory_path_string:
build_directory: Path = Path(directory_path_string)
self._generate_setup_py(build_directory,
name=name,
version=version)
self._generate_setup_cfg(
build_directory,
name=name,
version=version,
install_requires=install_requires,
extras_require=extras_require,
)
self._generate_python_module(
build_directory,
name=name,
)
built_distribution_archive = self._build_package(
build_directory=build_directory, name=name, version=version)
source_distribution = SourceDistribution.from_archive(
built_distribution_archive,
logger=self._logger,
requirement_parser=self._requirement_parser,
)
self._move_package_target_directory(built_distribution_archive)
self._sources.add(
name=name,
source=PathSource(
path=str(self._get_distribution_path(name, version))),
)
return source_distribution
def build_wheel(target_directory: Path, requirement: str) -> str:
logger = StreamLogger(sys.stdout)
requirement_parser = RequirementParser(logger=logger)
package_directory: str = str(ROOT / "unittests" / "data")
escaped_requirement = shlex.quote(requirement)
target_directory = target_directory.resolve()
with tempfile.TemporaryDirectory() as build_directory:
os.chdir(str(build_directory))
nix = Nix(logger=logger)
nix.shell(
command=
f"pip wheel {escaped_requirement} --find-links {str(package_directory)} --no-deps",
derivation_path=DERIVATION_PATH,
nix_arguments=dict(),
)
try:
parsed_requirement = requirement_parser.parse(requirement)
except ParsingFailed:
for path in os.listdir("."):
if path.endswith(".whl"):
wheel_path = path
break
else:
raise Exception("Build process did not produce .whl file")
else:
for path in os.listdir("."):
if path.endswith(".whl") and parsed_requirement.name() in path:
wheel_path = path
break
else:
raise Exception("Build process did not produce .whl file")
target_file_name = os.path.basename(wheel_path)
target_path = target_directory / target_file_name
shutil.move(wheel_path, str(target_path))
return target_file_name
def test_install_to_target_directory_does_not_install_to_default_directory(
pip: Pip,
project_dir: str,
download_dir: Path,
current_platform: TargetPlatform,
requirement_parser: RequirementParser,
):
requirements = RequirementSet(current_platform)
requirements.add(requirement_parser.parse("six"))
target_directory = Path(project_dir) / "target-directory"
target_directory.ensure_directory()
pip.download_sources(requirements, download_dir)
assert not target_directory.list_files()
pip.install(
requirements,
source_directories=[download_dir],
target_directory=target_directory,
)
assert target_directory.list_files()
def test_install_does_not_install_anything_with_empty_requirements(
pip: Pip, project_dir: str, current_platform: TargetPlatform):
target_directory = Path(project_dir) / "target_dir"
target_directory.ensure_directory()
pip.install(RequirementSet(current_platform), [], target_directory)
assert not target_directory.list_files()
def wheels_dir(project_dir: str) -> Path:
path = os.path.join(project_dir, "wheels")
os.makedirs(path)
return Path(path)
def _output_files(self) -> Set[Path]:
return {
Path(self.example_directory()) / "requirements.nix",
Path(self.flake_path()),
}
def package_source_directory(tmpdir_factory: Any) -> Path:
path_as_str: str = str(tmpdir_factory.mktemp("package_source_directory"))
return Path(path_as_str)
def download_dir(project_dir: str) -> Path:
path: str = os.path.join(project_dir, "download")
os.makedirs(path)
return Path(path)
def render_expression(
self,
packages_metadata: Iterable[Wheel],
sources: Sources,
) -> None:
"""Create Nix expressions.
"""
default_file = os.path.join(self._target_directory,
f"{self._requirements_name}.nix")
overrides_file = os.path.join(
self._target_directory, f"{self._requirements_name}_override.nix")
metadata_by_name: Dict[str,
Wheel] = {x.name: x
for x in packages_metadata}
generated_packages_metadata = []
for item in sorted(packages_metadata, key=lambda x: x.name):
if item.build_dependencies:
buildInputs = "\n".join(
sorted([
' self."{}"'.format(dependency.name())
for dependency in item.build_dependencies(
self._target_platform)
]))
buildInputs = "[\n" + buildInputs + "\n ]"
else:
buildInputs = "[ ]"
propagatedBuildInputs = "[ ]"
dependencies = item.dependencies(extras=[])
if dependencies:
deps = [
x.name() for x in dependencies
if x.name() in metadata_by_name.keys()
]
if deps:
propagatedBuildInputs = "[\n%s\n ]" % ("\n".join(
sorted([
' self."%s"' % (metadata_by_name[x].name)
for x in deps if x != item.name
])))
source = sources[item.name]
fetch_expression = source.nix_expression()
package_format = item.package_format
generated_packages_metadata.append(
dict(
name=item.name,
version=item.version,
fetch_expression=fetch_expression,
buildInputs=buildInputs,
propagatedBuildInputs=propagatedBuildInputs,
homepage=item.homepage,
license=item.license,
description=escape_string(item.description),
package_format=package_format,
))
generated_template = TEMPLATES.get_template("generated.nix.j2")
generated = "\n\n".join(
generated_template.render(**x)
for x in generated_packages_metadata)
overrides = TEMPLATES.get_template("overrides.nix.j2").render()
common_overrides_expressions = [
" (" + override.nix_expression(self._logger) + ")"
for override in self._common_overrides
]
default_template = TEMPLATES.get_template("requirements.nix.j2")
overrides_file_nix_path = os.path.join(
".",
os.path.split(overrides_file)[1])
default = default_template.render(
version=pypi2nix_version,
command_arguments=" ".join(map(shlex.quote, sys.argv[1:])),
python_version=self._python_version.derivation_name(),
extra_build_inputs=(self._extra_build_inputs
and "with pkgs; [ %s ]" %
(" ".join(self._extra_build_inputs)) or "[]"),
overrides_file=overrides_file_nix_path,
enable_tests="false",
generated_package_nix=generated,
common_overrides="\n".join(common_overrides_expressions),
python_major_version=self._python_version.major_version(),
)
if not os.path.exists(overrides_file):
with open(overrides_file, "w+") as f:
f.write(overrides.strip())
self._logger.info("|-> writing %s" % overrides_file)
with open(default_file, "w+") as f:
f.write(default.strip())
with open(self._frozen_file, "w+") as f:
f.write(self._requirements_frozen)
self._code_formatter.format_file(Path(default_file))
self._code_formatter.format_file(Path(overrides_file))
def path_source(tmpdir_factory):
path = Path(str(tmpdir_factory.mktemp("path_source") / "test.txt"))
path.write_text("")
return PathSource(str(path))
import os
from typing import Dict
import jinja2
from pypi2nix.path import Path
HERE = Path(os.path.dirname(__file__))
_templates = jinja2.Environment(loader=jinja2.FileSystemLoader(str(HERE / "templates")))
def render_template(template_path: Path, context=Dict[str, str]) -> str:
template = _templates.get_template(str(template_path))
return template.render(**context)
def install_target(tmpdir_factory) -> Path:
return Path(str(tmpdir_factory.mktemp("install-target")))
def target_directory(tmpdir_factory) -> Path:
return Path(str(tmpdir_factory.mktemp("target-directory")))
def find_root(start: Path = Path(".")) -> Path:
absolute_location = start.resolve()
if (absolute_location / ".git").is_directory():
return absolute_location
else:
return find_root(absolute_location / "..")
def run(self) -> None:
requirements = self.requirements_collector().requirements()
self.logger().info("pypi2nix v{} running ...".format(pypi2nix_version))
if not requirements:
self.logger().info(
"No requirements were specified. Ending program.")
return
setup_requirements = self.setup_requirements_collector().requirements()
requirements_name = os.path.join(self.configuration.target_directory,
self.configuration.output_basename)
sources = Sources()
sources.update(setup_requirements.sources())
sources.update(requirements.sources())
sources.update(self.setup_requirements_collector().sources())
sources.update(self.requirements_collector().sources())
self.logger().info("Downloading wheels and creating wheelhouse ...")
pip = NixPip(
nix=self.nix(),
project_directory=self.configuration.project_directory,
extra_env=self.configuration.extra_environment,
extra_build_inputs=self._extra_build_inputs(),
wheels_cache=self.configuration.wheels_caches,
target_platform=self.target_platform(),
logger=self.logger(),
requirement_parser=self.requirement_parser(),
)
wheel_builder = WheelBuilder(
pip=pip,
download_directory=self.configuration.project_directory /
"downloads",
lib_directory=self.configuration.project_directory / "lib",
extracted_wheel_directory=self.configuration.project_directory /
"extracted-wheels",
wheel_directory=self.configuration.project_directory / "wheels",
logger=self.logger(),
requirement_parser=self.requirement_parser(),
target_platform=self.target_platform(),
base_dependency_graph=self.base_dependency_graph(),
)
wheels = wheel_builder.build(requirements=requirements,
setup_requirements=setup_requirements)
requirements_frozen = wheel_builder.get_frozen_requirements()
source_distributions = wheel_builder.source_distributions
self.logger().info("Extracting metadata from pypi.python.org ...")
metadata_fetcher = MetadataFetcher(
sources=sources,
logger=self.logger(),
requirement_parser=self.requirement_parser(),
pypi=Pypi(logger=self.logger()),
)
packages_metadata = metadata_fetcher.main(
wheel_paths=wheels,
target_platform=self.target_platform(),
source_distributions=source_distributions,
)
self.logger().info("Generating Nix expressions ...")
renderers: List[ExpressionRenderer] = []
renderers.append(
RequirementsRenderer(
requirements_name=requirements_name,
extra_build_inputs=self.extra_build_inputs(),
python_version=self.configuration.python_version,
target_directory=self.configuration.target_directory,
logger=self.logger(),
common_overrides=self.configuration.overrides,
target_platform=self.target_platform(),
requirements_frozen=requirements_frozen,
code_formatter=self.code_formatter(),
))
renderers.append(
FlakeRenderer(
target_path=Path(self.configuration.target_directory) /
"flake.nix",
target_platform=self.target_platform(),
logger=self.logger(),
overrides=self.configuration.overrides,
extra_build_inputs=self.extra_build_inputs(),
code_formatter=self.code_formatter(),
))
for renderer in renderers:
renderer.render_expression(
packages_metadata=packages_metadata,
sources=sources,
)
if self.configuration.dependency_graph_output_location:
dependency_graph = DependencyGraph()
for wheel in packages_metadata:
dependency_graph.import_wheel(wheel, self.requirement_parser())
with open(str(self.configuration.dependency_graph_output_location),
"w") as output_file:
output_file.write(dependency_graph.serialize())
self.print_user_information()
