def test_metadata_header_round_trip() -> None:
input_metadata = LockfileMetadata.new(
InterpreterConstraints(
["CPython==2.7.*", "PyPy", "CPython>=3.6,<4,!=3.7.*"]),
reqset("ansicolors==0.1.0"),
)
serialized_lockfile = input_metadata.add_header_to_lockfile(
b"req1==1.0", regenerate_command="./pants lock")
output_metadata = LockfileMetadata.from_lockfile(serialized_lockfile)
assert input_metadata == output_metadata
def test_add_header_to_lockfile() -> None:
input_lockfile = b"""dave==3.1.4 \\
--hash=sha256:cab0c0c0c0c0dadacafec0c0c0c0cafedadabeefc0c0c0c0feedbeeffeedbeef \\
"""
expected = b"""
# This lockfile was autogenerated by Pants. To regenerate, run:
#
# ./pants lock
#
# --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
# {
# "version": 2,
# "valid_for_interpreter_constraints": [
# "CPython>=3.7"
# ],
# "generated_with_requirements": [
# "ansicolors==0.1.0"
# ]
# }
# --- END PANTS LOCKFILE METADATA ---
dave==3.1.4 \\
--hash=sha256:cab0c0c0c0c0dadacafec0c0c0c0cafedadabeefc0c0c0c0feedbeeffeedbeef \\
"""
def line_by_line(b: bytes) -> list[bytes]:
return [i for i in (j.strip() for j in b.splitlines()) if i]
metadata = LockfileMetadata.new(InterpreterConstraints([">=3.7"]),
reqset("ansicolors==0.1.0"))
result = metadata.add_header_to_lockfile(input_lockfile,
regenerate_command="./pants lock")
assert line_by_line(result) == line_by_line(expected)
def _validate_metadata(
metadata: LockfileMetadata,
request: PexRequest,
requirements: (Lockfile | LockfileContent),
python_setup: PythonSetup,
) -> None:
# TODO(#12314): Improve this message: `Requirement.parse` raises `InvalidRequirement`, which
# doesn't have mypy stubs at the moment; it may be hard to catch this exception and typecheck.
req_strings = ({PipRequirement.parse(i)
for i in requirements.req_strings}
if requirements.req_strings is not None else None)
validation = metadata.is_valid_for(
requirements.lockfile_hex_digest,
request.interpreter_constraints,
python_setup.interpreter_universe,
req_strings,
)
if validation:
return
def tool_message_parts(
requirements: (ToolCustomLockfile | ToolDefaultLockfile),
) -> Iterator[str]:
tool_name = requirements.options_scope_name
uses_source_plugins = requirements.uses_source_plugins
uses_project_interpreter_constraints = requirements.uses_project_interpreter_constraints
yield "You are using "
if isinstance(requirements, ToolDefaultLockfile):
yield "the `<default>` lockfile provided by Pants "
elif isinstance(requirements, ToolCustomLockfile):
yield f"the lockfile at {requirements.file_path} "
yield (
f"to install the tool `{tool_name}`, but it is not compatible with your "
"configuration: "
"\n\n")
if any(i == InvalidLockfileReason.INVALIDATION_DIGEST_MISMATCH
or i == InvalidLockfileReason.REQUIREMENTS_MISMATCH
for i in validation.failure_reasons):
# TODO(12314): Add message showing _which_ requirements diverged.
yield (
"- You have set different requirements than those used to generate the lockfile. "
f"You can fix this by not setting `[{tool_name}].version`, ")
if uses_source_plugins:
yield f"`[{tool_name}].source_plugins`, "
yield (
f"and `[{tool_name}].extra_requirements`, or by using a new "
"custom lockfile."
"\n")
if InvalidLockfileReason.INTERPRETER_CONSTRAINTS_MISMATCH in validation.failure_reasons:
yield (
f"- You have set interpreter constraints (`{request.interpreter_constraints}`) that "
"are not compatible with those used to generate the lockfile "
f"(`{metadata.valid_for_interpreter_constraints}`). ")
if not uses_project_interpreter_constraints:
yield (
f"You can fix this by not setting `[{tool_name}].interpreter_constraints`, "
"or by using a new custom lockfile. ")
else:
yield (
f"`{tool_name}` determines its interpreter constraints based on your code's own "
"constraints. To fix this error, you can either change your code's constraints "
f"(see {doc_url('python-interpreter-compatibility')}) or by generating a new "
"custom lockfile. ")
yield "\n"
yield "\n"
if not isinstance(requirements, ToolCustomLockfile):
yield (
"To generate a custom lockfile based on your current configuration, set "
f"`[{tool_name}].lockfile` to where you want to create the lockfile, then run "
f"`./pants generate-lockfiles --resolve={tool_name}`. ")
else:
yield (
"To regenerate your lockfile based on your current configuration, run "
f"`./pants generate-lockfiles --resolve={tool_name}`. ")
message: str
if isinstance(requirements, (ToolCustomLockfile, ToolDefaultLockfile)):
message = "".join(tool_message_parts(requirements)).strip()
else:
# TODO(12314): Improve this message
raise InvalidLockfileError(f"{validation.failure_reasons}")
if python_setup.invalid_lockfile_behavior == InvalidLockfileBehavior.error:
raise ValueError(message)
else:
logger.warning("%s", message)
async def generate_lockfile(
req: PythonLockfileRequest,
poetry_subsystem: PoetrySubsystem,
generate_lockfiles_subsystem: GenerateLockfilesSubsystem,
) -> PythonLockfile:
pyproject_toml = create_pyproject_toml(req.requirements, req.interpreter_constraints).encode()
pyproject_toml_digest, launcher_digest = await MultiGet(
Get(Digest, CreateDigest([FileContent("pyproject.toml", pyproject_toml)])),
Get(Digest, CreateDigest([POETRY_LAUNCHER])),
)
poetry_pex = await Get(
VenvPex,
PexRequest(
output_filename="poetry.pex",
internal_only=True,
requirements=poetry_subsystem.pex_requirements(),
interpreter_constraints=poetry_subsystem.interpreter_constraints,
main=EntryPoint(PurePath(POETRY_LAUNCHER.path).stem),
sources=launcher_digest,
),
)
# WONTFIX(#12314): Wire up Poetry to named_caches.
# WONTFIX(#12314): Wire up all the pip options like indexes.
poetry_lock_result = await Get(
ProcessResult,
VenvPexProcess(
poetry_pex,
argv=("lock",),
input_digest=pyproject_toml_digest,
output_files=("poetry.lock", "pyproject.toml"),
description=req._description or f"Generate lockfile for {req.resolve_name}",
# Instead of caching lockfile generation with LMDB, we instead use the invalidation
# scheme from `lockfile_metadata.py` to check for stale/invalid lockfiles. This is
# necessary so that our invalidation is resilient to deleting LMDB or running on a
# new machine.
#
# We disable caching with LMDB so that when you generate a lockfile, you always get
# the most up-to-date snapshot of the world. This is generally desirable and also
# necessary to avoid an awkward edge case where different developers generate different
# lockfiles even when generating at the same time. See
# https://github.com/pantsbuild/pants/issues/12591.
cache_scope=ProcessCacheScope.PER_SESSION,
),
)
poetry_export_result = await Get(
ProcessResult,
VenvPexProcess(
poetry_pex,
argv=("export", "-o", req.lockfile_dest),
input_digest=poetry_lock_result.output_digest,
output_files=(req.lockfile_dest,),
description=(
f"Exporting Poetry lockfile to requirements.txt format for {req.resolve_name}"
),
level=LogLevel.DEBUG,
),
)
initial_lockfile_digest_contents = await Get(
DigestContents, Digest, poetry_export_result.output_digest
)
# TODO(#12314) Improve error message on `Requirement.parse`
metadata = LockfileMetadata.new(
req.interpreter_constraints,
{PipRequirement.parse(i) for i in req.requirements},
)
lockfile_with_header = metadata.add_header_to_lockfile(
initial_lockfile_digest_contents[0].content,
regenerate_command=(
generate_lockfiles_subsystem.custom_command
or req._regenerate_command
or f"./pants generate-lockfiles --resolve={req.resolve_name}"
),
)
final_lockfile_digest = await Get(
Digest, CreateDigest([FileContent(req.lockfile_dest, lockfile_with_header)])
)
return PythonLockfile(final_lockfile_digest, req.resolve_name, req.lockfile_dest)
async def build_pex(
request: PexRequest,
python_setup: PythonSetup,
python_repos: PythonRepos,
platform: Platform,
pex_runtime_env: PexRuntimeEnvironment,
) -> BuildPexResult:
"""Returns a PEX with the given settings."""
argv = ["--output-file", request.output_filename, *request.additional_args]
repository_pex = (request.requirements.repository_pex if isinstance(
request.requirements, PexRequirements) else None)
if repository_pex:
argv.extend(["--pex-repository", repository_pex.name])
else:
# NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes`
# including PyPI, which will override `--no-pypi` and result in using PyPI in the default
# case. Why set `--no-pypi`, then? We need to do this so that
# `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI.
argv.extend([
"--no-pypi",
*(f"--index={index}" for index in python_repos.indexes),
*(f"--repo={repo}" for repo in python_repos.repos),
"--resolver-version",
"pip-2020-resolver",
])
python: PythonExecutable | None = None
# NB: If `--platform` is specified, this signals that the PEX should not be built locally.
# `--interpreter-constraint` only makes sense in the context of building locally. These two
# flags are mutually exclusive. See https://github.com/pantsbuild/pex/issues/957.
if request.platforms:
# TODO(#9560): consider validating that these platforms are valid with the interpreter
# constraints.
argv.extend(request.platforms.generate_pex_arg_list())
elif request.python:
python = request.python
elif request.internal_only:
# NB: If it's an internal_only PEX, we do our own lookup of the interpreter based on the
# interpreter constraints, and then will run the PEX with that specific interpreter. We
# will have already validated that there were no platforms.
python = await Get(PythonExecutable, InterpreterConstraints,
request.interpreter_constraints)
else:
# `--interpreter-constraint` options are mutually exclusive with the `--python` option,
# so we only specify them if we have not already located a concrete Python.
argv.extend(request.interpreter_constraints.generate_pex_arg_list())
if python:
argv.extend(["--python", python.path])
argv.append("--no-emit-warnings")
if python_setup.resolver_jobs:
argv.extend(["--jobs", str(python_setup.resolver_jobs)])
if python_setup.manylinux:
argv.extend(["--manylinux", python_setup.manylinux])
else:
argv.append("--no-manylinux")
if request.main is not None:
argv.extend(request.main.iter_pex_args())
# TODO(John Sirois): Right now any request requirements will shadow corresponding pex path
# requirements, which could lead to problems. Support shading python binaries.
# See: https://github.com/pantsbuild/pants/issues/9206
if request.pex_path:
argv.extend(
["--pex-path", ":".join(pex.name for pex in request.pex_path)])
source_dir_name = "source_files"
argv.append(f"--sources-directory={source_dir_name}")
sources_digest_as_subdir = await Get(
Digest, AddPrefix(request.sources or EMPTY_DIGEST, source_dir_name))
additional_inputs_digest = request.additional_inputs or EMPTY_DIGEST
repository_pex_digest = repository_pex.digest if repository_pex else EMPTY_DIGEST
constraint_file_digest = EMPTY_DIGEST
requirements_file_digest = EMPTY_DIGEST
# TODO(#12314): Capture the resolve name for multiple user lockfiles.
resolve_name = (request.requirements.options_scope_name if isinstance(
request.requirements,
(ToolDefaultLockfile, ToolCustomLockfile)) else None)
if isinstance(request.requirements, Lockfile):
is_monolithic_resolve = True
argv.extend(["--requirement", request.requirements.file_path])
argv.append("--no-transitive")
globs = PathGlobs(
[request.requirements.file_path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=request.requirements.
file_path_description_of_origin,
)
if python_setup.invalid_lockfile_behavior in {
InvalidLockfileBehavior.warn,
InvalidLockfileBehavior.error,
}:
requirements_file_digest_contents = await Get(
DigestContents, PathGlobs, globs)
metadata = LockfileMetadata.from_lockfile(
requirements_file_digest_contents[0].content,
request.requirements.file_path,
resolve_name,
)
_validate_metadata(metadata, request, request.requirements,
python_setup)
requirements_file_digest = await Get(Digest, PathGlobs, globs)
elif isinstance(request.requirements, LockfileContent):
is_monolithic_resolve = True
file_content = request.requirements.file_content
argv.extend(["--requirement", file_content.path])
argv.append("--no-transitive")
if python_setup.invalid_lockfile_behavior in {
InvalidLockfileBehavior.warn,
InvalidLockfileBehavior.error,
}:
metadata = LockfileMetadata.from_lockfile(
file_content.content, resolve_name=resolve_name)
_validate_metadata(metadata, request, request.requirements,
python_setup)
requirements_file_digest = await Get(Digest,
CreateDigest([file_content]))
else:
assert isinstance(request.requirements, PexRequirements)
is_monolithic_resolve = request.requirements.is_all_constraints_resolve
if (request.requirements.apply_constraints
and python_setup.requirement_constraints is not None):
argv.extend(
["--constraints", python_setup.requirement_constraints])
constraint_file_digest = await Get(
Digest,
PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=
"the option `[python].requirement_constraints`",
),
)
argv.extend(request.requirements.req_strings)
merged_digest = await Get(
Digest,
MergeDigests((
sources_digest_as_subdir,
additional_inputs_digest,
constraint_file_digest,
requirements_file_digest,
repository_pex_digest,
*(pex.digest for pex in request.pex_path),
)),
)
output_files: Iterable[str] | None = None
output_directories: Iterable[str] | None = None
if request.internal_only or is_monolithic_resolve:
# This is a much friendlier layout for the CAS than the default zipapp.
argv.extend(["--layout", "packed"])
output_directories = [request.output_filename]
else:
output_files = [request.output_filename]
process = await Get(
Process,
PexCliProcess(
python=python,
argv=argv,
additional_input_digest=merged_digest,
description=_build_pex_description(request),
output_files=output_files,
output_directories=output_directories,
),
)
# NB: Building a Pex is platform dependent, so in order to get a PEX that we can use locally
# without cross-building, we specify that our PEX command should be run on the current local
# platform.
result = await Get(ProcessResult, MultiPlatformProcess({platform:
process}))
if pex_runtime_env.verbosity > 0:
log_output = result.stderr.decode()
if log_output:
logger.info("%s", log_output)
digest = (await Get(
Digest,
MergeDigests(
(result.output_digest, *(pex.digest for pex in request.pex_path))))
if request.pex_path else result.output_digest)
return BuildPexResult(result=result,
pex_filename=request.output_filename,
digest=digest,
python=python)