class RequirementsCollector:
def __init__(
self,
platform: TargetPlatform,
requirement_parser: RequirementParser,
logger: Logger,
project_directory: str,
):
self.platform = platform
self.requirement_set = RequirementSet(platform)
self.requirement_parser = requirement_parser
self.logger = logger
self._project_directory = project_directory
self._sources = Sources()
def requirements(self) -> RequirementSet:
return self.requirement_set
def add_line(self, line: str) -> None:
requirement = self.requirement_parser.parse(line)
if isinstance(requirement, PathRequirement):
requirement = requirement.change_path(
lambda path: self._handle_requirements_path(
name=requirement.name(), path=path))
self.requirement_set.add(requirement)
def add_file(self, file_path: str) -> None:
requirements_file = RequirementsFile(file_path,
self._project_directory,
self.requirement_parser,
self.logger)
requirements_file.process()
self._sources.update(requirements_file.sources())
added_requirements = RequirementSet.from_file(requirements_file,
self.platform,
self.requirement_parser,
self.logger)
self.requirement_set += added_requirements
def sources(self) -> Sources:
sources = Sources()
sources.update(self.requirement_set.sources())
sources.update(self._sources)
return sources
def _handle_requirements_path(self, name: str, path: str) -> str:
self._sources.add(name, PathSource(path))
return os.path.abspath(path)
def run(self) -> None:
requirements = self.requirements_collector().requirements()
self.logger().info("pypi2nix v{} running ...".format(pypi2nix_version))
if not requirements:
self.logger().info(
"No requirements were specified. Ending program.")
return
setup_requirements = self.setup_requirements_collector().requirements()
requirements_name = os.path.join(self.configuration.target_directory,
self.configuration.output_basename)
sources = Sources()
sources.update(setup_requirements.sources())
sources.update(requirements.sources())
sources.update(self.setup_requirements_collector().sources())
sources.update(self.requirements_collector().sources())
self.logger().info("Downloading wheels and creating wheelhouse ...")
pip = NixPip(
nix=self.nix(),
project_directory=self.configuration.project_directory,
extra_env=self.configuration.extra_environment,
extra_build_inputs=self.configuration.extra_build_inputs,
wheels_cache=self.configuration.wheels_caches,
target_platform=self.target_platform(),
logger=self.logger(),
requirement_parser=self.requirement_parser(),
)
wheel_builder = WheelBuilder(
pip=pip,
project_directory=self.configuration.project_directory,
logger=self.logger(),
requirement_parser=self.requirement_parser(),
target_platform=self.target_platform(),
)
wheels = wheel_builder.build(requirements=requirements,
setup_requirements=setup_requirements)
requirements_frozen = wheel_builder.get_frozen_requirements()
source_distributions = wheel_builder.source_distributions
self.logger().info("Extracting metadata from pypi.python.org ...")
metadata_fetcher = MetadataFetcher(
sources=sources,
logger=self.logger(),
requirement_parser=self.requirement_parser(),
pypi=Pypi(logger=self.logger()),
)
packages_metadata = metadata_fetcher.main(
wheel_paths=wheels,
target_platform=self.target_platform(),
source_distributions=source_distributions,
)
self.logger().info("Generating Nix expressions ...")
render_expression(
packages_metadata=packages_metadata,
sources=sources,
requirements_name=requirements_name,
requirements_frozen=requirements_frozen,
extra_build_inputs=(self.configuration.extra_build_inputs
if self.configuration.emit_extra_build_inputs
else []),
enable_tests=self.configuration.enable_tests,
python_version=self.configuration.python_version,
target_directory=self.configuration.target_directory,
logger=self.logger(),
common_overrides=self.configuration.overrides,
)
self.print_user_information()
def main(
version: str,
verbose: int,
nix_shell: str,
nix_path: List[str],
basename: str,
cache_dir: str,
extra_build_inputs: List[str],
extra_env: str,
enable_tests: bool,
python_version: str,
requirements: List[str],
editable: List[str],
setup_requires: List[str],
overrides: List[AnyOverrides],
default_overrides: bool,
wheels_cache: List[str],
) -> None:
"""SPECIFICATION should be requirements.txt (output of pip freeze).
"""
logger = Logger(output=sys.stdout)
nix_executable_directory: Optional[str] = (os.path.abspath(
os.path.dirname(nix_shell)) if os.path.exists(nix_shell) else None)
nix = Nix(
nix_path=nix_path,
executable_directory=nix_executable_directory,
verbose=verbose != 0,
)
platform_generator = PlatformGenerator(nix=nix)
if default_overrides:
overrides += tuple([
pypi2nix.overrides.OverridesGithub(owner="garbas",
repo="nixpkgs-python",
path="overrides.nix")
])
with open(os.path.join(os.path.dirname(__file__), "VERSION")) as f:
pypi2nix_version = f.read()
if version:
click.echo(pypi2nix_version)
return
python_version_argument = python_version
python_versions = pypi2nix.utils.PYTHON_VERSIONS.keys()
if not python_version:
raise click.exceptions.UsageError(
'Missing option "-V" / "--python-version". Choose from ' +
(", ".join(python_versions)))
python_version = pypi2nix.utils.PYTHON_VERSIONS[python_version]
target_platform = platform_generator.from_python_version(
python_version_argument)
requirement_collector = RequirementsCollector(target_platform)
setup_requirement_collector = RequirementsCollector(target_platform)
extra_build_inputs = pypi2nix.utils.args_as_list(extra_build_inputs)
setup_requires = pypi2nix.utils.args_as_list(setup_requires)
for item in editable:
requirement_collector.add_line(item)
for build_input in setup_requires:
setup_requirement_collector.add_line(build_input)
# temporary pypi2nix folder and make sure it exists
tmp_dir = os.path.join(tempfile.gettempdir(), "pypi2nix")
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
current_dir = os.getcwd()
requirements_name = os.path.join(current_dir, basename)
if not cache_dir:
cache_dir = os.path.join(tmp_dir, "cache")
download_cache_dir = os.path.join(cache_dir, "download")
wheel_cache_dir = os.path.join(cache_dir, "wheel")
if not os.path.exists(download_cache_dir):
os.makedirs(download_cache_dir)
if not os.path.exists(wheel_cache_dir):
os.makedirs(wheel_cache_dir)
assert requirements is not None
project_hash = md5_sum_of_files_with_file_names(requirements)
project_dir = os.path.join(tmp_dir, project_hash)
if os.path.exists(project_dir):
shutil.rmtree(project_dir)
os.makedirs(project_dir)
for requirement_file_path in requirements:
requirement_collector.add_file(requirement_file_path)
requirement_set = requirement_collector.requirements()
setup_requirements = setup_requirement_collector.requirements()
sources = Sources()
sources.update(requirement_set.sources())
sources.update(setup_requirements.sources())
click.echo("pypi2nix v{} running ...".format(pypi2nix_version))
click.echo("")
click.echo("Stage1: Downloading wheels and creating wheelhouse ...")
pip = Pip(
nix=nix,
project_directory=project_dir,
extra_env=extra_env,
extra_build_inputs=extra_build_inputs,
verbose=verbose,
wheels_cache=wheels_cache,
target_platform=target_platform,
)
wheel_builder = pypi2nix.stage1.WheelBuilder(pip=pip,
project_directory=project_dir,
logger=logger)
wheels = wheel_builder.build(requirements=requirement_set,
setup_requirements=setup_requirements)
requirements_frozen = wheel_builder.get_frozen_requirements()
default_environment = pip.default_environment()
additional_dependency_graph = wheel_builder.additional_build_dependencies
click.echo("Stage2: Extracting metadata from pypi.python.org ...")
stage2 = pypi2nix.stage2.Stage2(sources=sources, verbose=verbose)
packages_metadata = stage2.main(
wheel_paths=wheels,
default_environment=default_environment,
wheel_cache_dir=wheel_cache_dir,
additional_dependencies=additional_dependency_graph,
)
click.echo("Stage3: Generating Nix expressions ...")
pypi2nix.stage3.main(
packages_metadata=packages_metadata,
sources=sources,
requirements_name=requirements_name,
requirements_frozen=requirements_frozen,
extra_build_inputs=extra_build_inputs,
enable_tests=enable_tests,
python_version=python_version,
current_dir=current_dir,
common_overrides=overrides,
)
click.echo("")
click.echo("Nix expressions generated successfully.")
click.echo("")
click.echo("To start development run:")
click.echo(" nix-shell requirements.nix -A interpreter")
click.echo("")
click.echo("More information you can find at")
click.echo(" https://github.com/garbas/pypi2nix")
click.echo("")
def run(self) -> None:
requirements = self.requirements_collector().requirements()
self.logger().info("pypi2nix v{} running ...".format(pypi2nix_version))
if not requirements:
self.logger().info(
"No requirements were specified. Ending program.")
return
setup_requirements = self.setup_requirements_collector().requirements()
requirements_name = os.path.join(self.configuration.target_directory,
self.configuration.output_basename)
sources = Sources()
sources.update(setup_requirements.sources())
sources.update(requirements.sources())
sources.update(self.setup_requirements_collector().sources())
sources.update(self.requirements_collector().sources())
self.logger().info("Downloading wheels and creating wheelhouse ...")
pip = NixPip(
nix=self.nix(),
project_directory=self.configuration.project_directory,
extra_env=self.configuration.extra_environment,
extra_build_inputs=self._extra_build_inputs(),
wheels_cache=self.configuration.wheels_caches,
target_platform=self.target_platform(),
logger=self.logger(),
requirement_parser=self.requirement_parser(),
)
wheel_builder = WheelBuilder(
pip=pip,
download_directory=self.configuration.project_directory /
"downloads",
lib_directory=self.configuration.project_directory / "lib",
extracted_wheel_directory=self.configuration.project_directory /
"extracted-wheels",
wheel_directory=self.configuration.project_directory / "wheels",
logger=self.logger(),
requirement_parser=self.requirement_parser(),
target_platform=self.target_platform(),
base_dependency_graph=self.base_dependency_graph(),
)
wheels = wheel_builder.build(requirements=requirements,
setup_requirements=setup_requirements)
requirements_frozen = wheel_builder.get_frozen_requirements()
source_distributions = wheel_builder.source_distributions
self.logger().info("Extracting metadata from pypi.python.org ...")
metadata_fetcher = MetadataFetcher(
sources=sources,
logger=self.logger(),
requirement_parser=self.requirement_parser(),
pypi=Pypi(logger=self.logger()),
)
packages_metadata = metadata_fetcher.main(
wheel_paths=wheels,
target_platform=self.target_platform(),
source_distributions=source_distributions,
)
self.logger().info("Generating Nix expressions ...")
renderers: List[ExpressionRenderer] = []
renderers.append(
RequirementsRenderer(
requirements_name=requirements_name,
extra_build_inputs=self.extra_build_inputs(),
python_version=self.configuration.python_version,
target_directory=self.configuration.target_directory,
logger=self.logger(),
common_overrides=self.configuration.overrides,
target_platform=self.target_platform(),
requirements_frozen=requirements_frozen,
code_formatter=self.code_formatter(),
))
renderers.append(
FlakeRenderer(
target_path=Path(self.configuration.target_directory) /
"flake.nix",
target_platform=self.target_platform(),
logger=self.logger(),
overrides=self.configuration.overrides,
extra_build_inputs=self.extra_build_inputs(),
code_formatter=self.code_formatter(),
))
for renderer in renderers:
renderer.render_expression(
packages_metadata=packages_metadata,
sources=sources,
)
if self.configuration.dependency_graph_output_location:
dependency_graph = DependencyGraph()
for wheel in packages_metadata:
dependency_graph.import_wheel(wheel, self.requirement_parser())
with open(str(self.configuration.dependency_graph_output_location),
"w") as output_file:
output_file.write(dependency_graph.serialize())
self.print_user_information()
class RequirementsFile:
def __init__(
self,
path: str,
project_dir: str,
requirement_parser: RequirementParser,
logger: Logger,
):
self.project_dir: str = project_dir
self.original_path: str = path
self.requirement_parser = requirement_parser
self._logger = logger
self._sources = Sources()
@classmethod
def from_lines(
constructor: "Type[RequirementsFile]",
lines: List[str],
project_dir: str,
requirement_parser: RequirementParser,
logger: Logger,
) -> "RequirementsFile":
assert not isinstance(lines, str)
temporary_file_descriptor, temporary_file_path = tempfile.mkstemp(
dir=project_dir, text=True
)
try:
with open(temporary_file_descriptor, "w") as f:
for line in lines:
f.write(line)
f.write("\n")
requirements_file = constructor(
project_dir=project_dir,
path=temporary_file_path,
requirement_parser=requirement_parser,
logger=logger,
)
requirements_file.process()
finally:
os.remove(temporary_file_path)
return requirements_file
def read(self) -> str:
if os.path.exists(self.processed_requirements_file_path()):
path = self.processed_requirements_file_path()
else:
path = self.original_path
with open(path) as f:
return f.read()
def process(self) -> None:
new_requirements_file = self.processed_requirements_file_path()
with open(self.original_path) as original_file, open(
new_requirements_file, "w+"
) as new_file:
for requirements_line in original_file.readlines():
requirements_line = requirements_line.strip()
if requirements_line:
processed_requirements_line = self._process_line(requirements_line)
print(processed_requirements_line, file=new_file)
self._logger.debug(f"Created requirements file {new_requirements_file}")
def _process_line(self, requirements_line: str) -> str:
line_handler: LineHandler
if self.is_include_line(requirements_line):
line_handler = _RequirementIncludeLineHandler(
line=requirements_line,
original_path=self.original_path,
project_directory=self.project_dir,
requirement_parser=self.requirement_parser,
logger=self._logger,
)
elif self.is_editable_line(requirements_line):
line_handler = _EditableLineHandler(
line=requirements_line,
original_path=self.original_path,
requirement_parser=self.requirement_parser,
)
else:
line_handler = _RequirementLineHandler(
line=requirements_line,
requirement_parser=self.requirement_parser,
original_path=self.original_path,
)
line, sources = line_handler.process()
self._sources.update(sources)
return line
def processed_requirements_file_path(self) -> str:
return "%s/%s.txt" % (
self.project_dir,
hashlib.md5(self.original_path.encode()).hexdigest(),
)
def is_include_line(self, line: str) -> bool:
return line.startswith("-r ") or line.startswith("-c ")
def is_vcs_line(self, line: str) -> bool:
return line.startswith("-e git+") or line.startswith("-e hg+")
def is_editable_line(self, line: str) -> bool:
return line.startswith("-e ") and not self.is_vcs_line(line)
def sources(self) -> Sources:
return self._sources
def sources(self) -> Sources:
sources = Sources()
sources.update(self.requirement_set.sources())
sources.update(self._sources)
return sources
class RequirementsCollector:
def __init__(
self,
platform: TargetPlatform,
requirement_parser: RequirementParser,
logger: Logger,
project_directory: str,
base_dependency_graph: DependencyGraph,
):
self.platform = platform
self.requirement_set = RequirementSet(platform)
self.requirement_parser = requirement_parser
self.logger = logger
self._project_directory = project_directory
self._sources = Sources()
self._base_dependency_graph = base_dependency_graph
def requirements(self) -> RequirementSet:
return self.requirement_set
def add_line(self, line: str) -> None:
original_dependency = self.requirement_parser.parse(line)
transitive_requirements = self._base_dependency_graph.get_all_build_dependency_names(
original_dependency)
self._add_line_without_dependency_check(line)
for requirement in transitive_requirements:
self._add_line_without_dependency_check(requirement)
def add_file(self, file_path: str) -> None:
requirements_file = RequirementsFile(file_path,
self._project_directory,
self.requirement_parser,
self.logger)
requirements_file.process()
self._sources.update(requirements_file.sources())
added_requirements = RequirementSet.from_file(requirements_file,
self.platform,
self.requirement_parser,
self.logger)
transitive_requirements = set()
for requirement in added_requirements:
transitive_requirements.update(
self._base_dependency_graph.get_all_build_dependency_names(
requirement))
for line in transitive_requirements:
self._add_line_without_dependency_check(line)
self.requirement_set += added_requirements
def sources(self) -> Sources:
sources = Sources()
sources.update(self.requirement_set.sources())
sources.update(self._sources)
return sources
def _add_line_without_dependency_check(self, line: str) -> None:
requirement = self.requirement_parser.parse(line)
if isinstance(requirement, PathRequirement):
requirement = requirement.change_path(
lambda path: self._handle_requirements_path(
name=requirement.name(), path=path))
self.requirement_set.add(requirement)
def _handle_requirements_path(self, name: str, path: str) -> str:
self._sources.add(name, PathSource(path))
return os.path.abspath(path)