def test_that_logger_with_high_verbosity_level_does_emit_warning_logs(
logger: Logger, level: Verbosity):
logger.set_verbosity(level)
logger.warning("test")
output = get_logger_output(logger)
assert "WARNING" in output
def test_that_logger_with_high_verbosity_level_does_emit_debug_logs(
logger: Logger, level: Verbosity):
logger.set_verbosity(level)
logger.debug("test")
output = get_logger_output(logger)
assert "DEBUG" in output
def cmd(command: Union[str, List[str]],
logger: Logger,
stderr: Optional[int] = None) -> Tuple[int, str]:
if isinstance(command, str):
command = shlex.split(command)
logger.debug("|-> " + " ".join(map(shlex.quote, command)))
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=stderr)
try:
out = []
while True:
line = p.stdout.readline().decode()
if line == "" and p.poll() is not None:
break
if line != "":
logger.debug(" " + line.rstrip("\n"))
out.append(line)
except Exception:
p.kill()
raise
finally:
p.communicate()
return p.returncode, "".join(out)
def test_that_logger_with_low_verbosity_level_does_not_emit_info_logs(
logger: Logger, level):
logger.set_verbosity(level)
logger.info("test")
output = get_logger_output(logger)
assert "INFO" not in output
def test_that_logger_with_high_verbosity_level_does_emit_info_logs(
logger: Logger, level: Verbosity):
logger.set_verbosity(level)
logger.info("test")
output = get_logger_output(logger)
assert "INFO" in output
def cmd(
command: Union[str, List[str]],
logger: Logger,
stderr: Optional[int] = None,
cwd: Optional[str] = None,
input: Optional[str] = None,
) -> Tuple[int, str]:
if isinstance(command, str):
command = shlex.split(command)
logger.debug("|-> " + " ".join(map(shlex.quote, command)))
p = subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=stderr,
cwd=cwd,
universal_newlines=True,
)
try:
out, _ = p.communicate(input)
except Exception:
p.kill()
p.communicate()
raise
logger.debug(out)
return p.returncode, out
def format_nix_files(logger: Logger) -> None:
if is_nixfmt_installed():
logger.info("Formatting nix files")
integration_test_nix_files = find_nix_files_in_integration_tests()
subprocess.run(
["nixfmt", "default.nix", "src/pypi2nix/pip/bootstrap.nix"] +
integration_test_nix_files,
check=True,
)
else:
logger.warning(
"Could not find `nixfmt` executable. Cannot format .nix files")
def get_inner_logger(logger: Logger) -> StreamLogger:
if isinstance(logger, StreamLogger):
return logger
elif isinstance(logger, ProxyLogger):
inner_logger = logger.get_target_logger()
if inner_logger is None:
raise Exception(
"ProxyLogger is not connected, cannot get output")
else:
return get_inner_logger(inner_logger)
else:
raise Exception("Unhandled Logger implementation", type(logger))
def main(
packages_metadata: Iterable[Wheel],
sources: Sources,
requirements_name: str,
requirements_frozen: str,
extra_build_inputs: Iterable[str],
enable_tests: bool,
python_version: PythonVersion,
target_directory: str,
logger: Logger,
common_overrides: Iterable[Overrides] = [],
) -> None:
"""Create Nix expressions.
"""
default_file = os.path.join(target_directory, f"{requirements_name}.nix")
overrides_file = os.path.join(target_directory,
f"{requirements_name}_override.nix")
frozen_file = os.path.join(target_directory,
f"{requirements_name}_frozen.txt")
metadata_by_name: Dict[str, Wheel] = {x.name: x for x in packages_metadata}
generated_packages_metadata = []
for item in sorted(packages_metadata, key=lambda x: x.name):
if item.build_dependencies:
buildInputs = "\n".join(
sorted([
' self."{}"'.format(dependency.name())
for dependency in item.build_dependencies
]))
buildInputs = "[\n" + buildInputs + "\n ]"
else:
buildInputs = "[ ]"
propagatedBuildInputs = "[ ]"
dependencies = item.dependencies(extras=[])
if dependencies:
deps = [
x.name() for x in dependencies
if x.name() in metadata_by_name.keys()
]
if deps:
propagatedBuildInputs = "[\n%s\n ]" % ("\n".join(
sorted([
' self."%s"' % (metadata_by_name[x].name)
for x in deps if x != item.name
])))
source = sources[item.name]
fetch_expression = source.nix_expression()
package_format = item.package_format
generated_packages_metadata.append(
dict(
name=item.name,
version=item.version,
fetch_expression=fetch_expression,
buildInputs=buildInputs,
propagatedBuildInputs=propagatedBuildInputs,
homepage=item.homepage,
license=item.license,
description=item.description,
package_format=package_format,
))
templates = jinja2.Environment(
loader=jinja2.FileSystemLoader(HERE + "/templates"))
generated_template = templates.get_template("generated.nix.j2")
generated = "\n\n".join(
generated_template.render(**x) for x in generated_packages_metadata)
overrides = templates.get_template("overrides.nix.j2").render()
common_overrides_expressions = [
" (" + override.nix_expression(logger) + ")"
for override in common_overrides
]
default_template = templates.get_template("requirements.nix.j2")
overrides_file_nix_path = os.path.join(".",
os.path.split(overrides_file)[1])
default = default_template.render(
version=pypi2nix_version,
command_arguments=" ".join(map(shlex.quote, sys.argv[1:])),
python_version=python_version.derivation_name(),
extra_build_inputs=(extra_build_inputs and "with pkgs; [ %s ]" %
(" ".join(extra_build_inputs)) or "[]"),
overrides_file=overrides_file_nix_path,
enable_tests=str(enable_tests).lower(),
generated_package_nix=generated,
common_overrides="\n".join(common_overrides_expressions),
python_major_version=python_version.major_version(),
)
if not os.path.exists(overrides_file):
with open(overrides_file, "w+") as f:
f.write(overrides.strip())
logger.info("|-> writing %s" % overrides_file)
with open(default_file, "w+") as f:
f.write(default.strip())
with open(frozen_file, "w+") as f:
f.write(requirements_frozen)
def main(
version: str,
verbose: int,
nix_shell: str,
nix_path: List[str],
basename: str,
cache_dir: str,
extra_build_inputs: List[str],
extra_env: str,
enable_tests: bool,
python_version: str,
requirements: List[str],
editable: List[str],
setup_requires: List[str],
overrides: List[AnyOverrides],
default_overrides: bool,
wheels_cache: List[str],
) -> None:
"""SPECIFICATION should be requirements.txt (output of pip freeze).
"""
logger = Logger(output=sys.stdout)
nix_executable_directory: Optional[str] = (os.path.abspath(
os.path.dirname(nix_shell)) if os.path.exists(nix_shell) else None)
nix = Nix(
nix_path=nix_path,
executable_directory=nix_executable_directory,
verbose=verbose != 0,
)
platform_generator = PlatformGenerator(nix=nix)
if default_overrides:
overrides += tuple([
pypi2nix.overrides.OverridesGithub(owner="garbas",
repo="nixpkgs-python",
path="overrides.nix")
])
with open(os.path.join(os.path.dirname(__file__), "VERSION")) as f:
pypi2nix_version = f.read()
if version:
click.echo(pypi2nix_version)
return
python_version_argument = python_version
python_versions = pypi2nix.utils.PYTHON_VERSIONS.keys()
if not python_version:
raise click.exceptions.UsageError(
'Missing option "-V" / "--python-version". Choose from ' +
(", ".join(python_versions)))
python_version = pypi2nix.utils.PYTHON_VERSIONS[python_version]
target_platform = platform_generator.from_python_version(
python_version_argument)
requirement_collector = RequirementsCollector(target_platform)
setup_requirement_collector = RequirementsCollector(target_platform)
extra_build_inputs = pypi2nix.utils.args_as_list(extra_build_inputs)
setup_requires = pypi2nix.utils.args_as_list(setup_requires)
for item in editable:
requirement_collector.add_line(item)
for build_input in setup_requires:
setup_requirement_collector.add_line(build_input)
# temporary pypi2nix folder and make sure it exists
tmp_dir = os.path.join(tempfile.gettempdir(), "pypi2nix")
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
current_dir = os.getcwd()
requirements_name = os.path.join(current_dir, basename)
if not cache_dir:
cache_dir = os.path.join(tmp_dir, "cache")
download_cache_dir = os.path.join(cache_dir, "download")
wheel_cache_dir = os.path.join(cache_dir, "wheel")
if not os.path.exists(download_cache_dir):
os.makedirs(download_cache_dir)
if not os.path.exists(wheel_cache_dir):
os.makedirs(wheel_cache_dir)
assert requirements is not None
project_hash = md5_sum_of_files_with_file_names(requirements)
project_dir = os.path.join(tmp_dir, project_hash)
if os.path.exists(project_dir):
shutil.rmtree(project_dir)
os.makedirs(project_dir)
for requirement_file_path in requirements:
requirement_collector.add_file(requirement_file_path)
requirement_set = requirement_collector.requirements()
setup_requirements = setup_requirement_collector.requirements()
sources = Sources()
sources.update(requirement_set.sources())
sources.update(setup_requirements.sources())
click.echo("pypi2nix v{} running ...".format(pypi2nix_version))
click.echo("")
click.echo("Stage1: Downloading wheels and creating wheelhouse ...")
pip = Pip(
nix=nix,
project_directory=project_dir,
extra_env=extra_env,
extra_build_inputs=extra_build_inputs,
verbose=verbose,
wheels_cache=wheels_cache,
target_platform=target_platform,
)
wheel_builder = pypi2nix.stage1.WheelBuilder(pip=pip,
project_directory=project_dir,
logger=logger)
wheels = wheel_builder.build(requirements=requirement_set,
setup_requirements=setup_requirements)
requirements_frozen = wheel_builder.get_frozen_requirements()
default_environment = pip.default_environment()
additional_dependency_graph = wheel_builder.additional_build_dependencies
click.echo("Stage2: Extracting metadata from pypi.python.org ...")
stage2 = pypi2nix.stage2.Stage2(sources=sources, verbose=verbose)
packages_metadata = stage2.main(
wheel_paths=wheels,
default_environment=default_environment,
wheel_cache_dir=wheel_cache_dir,
additional_dependencies=additional_dependency_graph,
)
click.echo("Stage3: Generating Nix expressions ...")
pypi2nix.stage3.main(
packages_metadata=packages_metadata,
sources=sources,
requirements_name=requirements_name,
requirements_frozen=requirements_frozen,
extra_build_inputs=extra_build_inputs,
enable_tests=enable_tests,
python_version=python_version,
current_dir=current_dir,
common_overrides=overrides,
)
click.echo("")
click.echo("Nix expressions generated successfully.")
click.echo("")
click.echo("To start development run:")
click.echo(" nix-shell requirements.nix -A interpreter")
click.echo("")
click.echo("More information you can find at")
click.echo(" https://github.com/garbas/pypi2nix")
click.echo("")
def logger():
with StringIO() as f:
yield Logger(output=f)
def test_every_info_line_is_prefixed(logger: Logger):
logger.info("line1\nline2")
output = get_logger_output(logger)
assert "INFO: line1" in output
assert "INFO: line2" in output
def test_every_debug_line_is_prefixed(logger: Logger):
logger.debug("line1\nline2")
output = get_logger_output(logger)
assert "DEBUG: line1" in output
assert "DEBUG: line2" in output
def test_every_line_of_error_is_prefixed(logger: Logger):
logger.error("line1\nline2")
output = get_logger_output(logger)
assert "ERROR: line1" in output
assert "ERROR: line2" in output
def test_can_log_warning(logger: Logger):
logger.warning("test")
assert "WARNING: test" in get_logger_output(logger)
def logger():
stream = StringIO("")
logger = Logger(output=stream)
return logger
def test_that_logger_always_emits_errors(logger: Logger, level: Verbosity):
logger.set_verbosity(level)
logger.error("test")
output = get_logger_output(logger)
assert "ERROR" in output
def from_wheel_directory_path(
wheel_class: "Type[Wheel]",
wheel_directory_path: str,
target_platform: TargetPlatform,
logger: Logger,
requirement_parser: RequirementParser,
) -> "Wheel":
metadata_file = os.path.join(wheel_directory_path, "METADATA")
if os.path.exists(metadata_file):
with open(metadata_file,
"r",
encoding="ascii",
errors="surrogateescape") as headers:
metadata = email.parser.Parser().parse(headers)
license_string = str_from_message(metadata, "license")
if license_string is None:
license_string = ""
classifiers = list_from_message(metadata, "Classifier")
if classifiers is None:
classifiers = []
license = find_license(classifiers=classifiers,
license_string=license_string)
if license is None:
license = '"' + safe(license_string) + '"'
logger.warning(
"Couldn't recognize license `{}` for `{}`".format(
license_string, metadata.get("name")))
name = str_from_message(metadata, "name")
if name is None:
raise Exception("Could not extract name from wheel metadata")
else:
name = canonicalize_name(name)
version = str_from_message(metadata, "version")
if version is None:
raise Exception(
"Could not extract version from wheel metadata")
dependencies = list_from_message(metadata, "requires-dist")
if dependencies is None:
dependencies = []
description = str_from_message(metadata, "summary")
if description is None:
description = ""
return wheel_class(
name=name,
version=version,
deps=wheel_class._extract_deps(
dependencies,
target_platform,
requirement_parser,
current_wheel_name=name,
),
homepage=safe(find_homepage(metadata)),
license=license,
description=safe(description),
build_dependencies=RequirementSet(target_platform),
target_platform=target_platform,
)
raise click.ClickException("Unable to find METADATA in `%s` folder." %
wheel_directory_path)
def test_can_log_info(logger: Logger):
logger.info("test")
assert "INFO: test" in get_logger_output(logger)
def test_can_log_error(logger: Logger):
logger.error("test")
assert "ERROR: test" in get_logger_output(logger)
def test_can_log_debug(logger: Logger):
logger.debug("test")
assert "DEBUG: test" in get_logger_output(logger)