def test_metadata_header_round_trip() -> None:
input_metadata = LockfileMetadata.new(
InterpreterConstraints(
["CPython==2.7.*", "PyPy", "CPython>=3.6,<4,!=3.7.*"]),
reqset("ansicolors==0.1.0"),
)
serialized_lockfile = input_metadata.add_header_to_lockfile(
b"req1==1.0", regenerate_command="./pants lock")
output_metadata = LockfileMetadata.from_lockfile(serialized_lockfile)
assert input_metadata == output_metadata
async def build_pex(
request: PexRequest,
python_setup: PythonSetup,
python_repos: PythonRepos,
platform: Platform,
pex_runtime_env: PexRuntimeEnvironment,
) -> BuildPexResult:
"""Returns a PEX with the given settings."""
argv = ["--output-file", request.output_filename, *request.additional_args]
repository_pex = (request.requirements.repository_pex if isinstance(
request.requirements, PexRequirements) else None)
if repository_pex:
argv.extend(["--pex-repository", repository_pex.name])
else:
# NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes`
# including PyPI, which will override `--no-pypi` and result in using PyPI in the default
# case. Why set `--no-pypi`, then? We need to do this so that
# `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI.
argv.extend([
"--no-pypi",
*(f"--index={index}" for index in python_repos.indexes),
*(f"--repo={repo}" for repo in python_repos.repos),
"--resolver-version",
"pip-2020-resolver",
])
python: PythonExecutable | None = None
# NB: If `--platform` is specified, this signals that the PEX should not be built locally.
# `--interpreter-constraint` only makes sense in the context of building locally. These two
# flags are mutually exclusive. See https://github.com/pantsbuild/pex/issues/957.
if request.platforms:
# TODO(#9560): consider validating that these platforms are valid with the interpreter
# constraints.
argv.extend(request.platforms.generate_pex_arg_list())
elif request.python:
python = request.python
elif request.internal_only:
# NB: If it's an internal_only PEX, we do our own lookup of the interpreter based on the
# interpreter constraints, and then will run the PEX with that specific interpreter. We
# will have already validated that there were no platforms.
python = await Get(PythonExecutable, InterpreterConstraints,
request.interpreter_constraints)
else:
# `--interpreter-constraint` options are mutually exclusive with the `--python` option,
# so we only specify them if we have not already located a concrete Python.
argv.extend(request.interpreter_constraints.generate_pex_arg_list())
if python:
argv.extend(["--python", python.path])
argv.append("--no-emit-warnings")
if python_setup.resolver_jobs:
argv.extend(["--jobs", str(python_setup.resolver_jobs)])
if python_setup.manylinux:
argv.extend(["--manylinux", python_setup.manylinux])
else:
argv.append("--no-manylinux")
if request.main is not None:
argv.extend(request.main.iter_pex_args())
# TODO(John Sirois): Right now any request requirements will shadow corresponding pex path
# requirements, which could lead to problems. Support shading python binaries.
# See: https://github.com/pantsbuild/pants/issues/9206
if request.pex_path:
argv.extend(
["--pex-path", ":".join(pex.name for pex in request.pex_path)])
source_dir_name = "source_files"
argv.append(f"--sources-directory={source_dir_name}")
sources_digest_as_subdir = await Get(
Digest, AddPrefix(request.sources or EMPTY_DIGEST, source_dir_name))
additional_inputs_digest = request.additional_inputs or EMPTY_DIGEST
repository_pex_digest = repository_pex.digest if repository_pex else EMPTY_DIGEST
constraint_file_digest = EMPTY_DIGEST
requirements_file_digest = EMPTY_DIGEST
# TODO(#12314): Capture the resolve name for multiple user lockfiles.
resolve_name = (request.requirements.options_scope_name if isinstance(
request.requirements,
(ToolDefaultLockfile, ToolCustomLockfile)) else None)
if isinstance(request.requirements, Lockfile):
is_monolithic_resolve = True
argv.extend(["--requirement", request.requirements.file_path])
argv.append("--no-transitive")
globs = PathGlobs(
[request.requirements.file_path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=request.requirements.
file_path_description_of_origin,
)
if python_setup.invalid_lockfile_behavior in {
InvalidLockfileBehavior.warn,
InvalidLockfileBehavior.error,
}:
requirements_file_digest_contents = await Get(
DigestContents, PathGlobs, globs)
metadata = LockfileMetadata.from_lockfile(
requirements_file_digest_contents[0].content,
request.requirements.file_path,
resolve_name,
)
_validate_metadata(metadata, request, request.requirements,
python_setup)
requirements_file_digest = await Get(Digest, PathGlobs, globs)
elif isinstance(request.requirements, LockfileContent):
is_monolithic_resolve = True
file_content = request.requirements.file_content
argv.extend(["--requirement", file_content.path])
argv.append("--no-transitive")
if python_setup.invalid_lockfile_behavior in {
InvalidLockfileBehavior.warn,
InvalidLockfileBehavior.error,
}:
metadata = LockfileMetadata.from_lockfile(
file_content.content, resolve_name=resolve_name)
_validate_metadata(metadata, request, request.requirements,
python_setup)
requirements_file_digest = await Get(Digest,
CreateDigest([file_content]))
else:
assert isinstance(request.requirements, PexRequirements)
is_monolithic_resolve = request.requirements.is_all_constraints_resolve
if (request.requirements.apply_constraints
and python_setup.requirement_constraints is not None):
argv.extend(
["--constraints", python_setup.requirement_constraints])
constraint_file_digest = await Get(
Digest,
PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=
"the option `[python].requirement_constraints`",
),
)
argv.extend(request.requirements.req_strings)
merged_digest = await Get(
Digest,
MergeDigests((
sources_digest_as_subdir,
additional_inputs_digest,
constraint_file_digest,
requirements_file_digest,
repository_pex_digest,
*(pex.digest for pex in request.pex_path),
)),
)
output_files: Iterable[str] | None = None
output_directories: Iterable[str] | None = None
if request.internal_only or is_monolithic_resolve:
# This is a much friendlier layout for the CAS than the default zipapp.
argv.extend(["--layout", "packed"])
output_directories = [request.output_filename]
else:
output_files = [request.output_filename]
process = await Get(
Process,
PexCliProcess(
python=python,
argv=argv,
additional_input_digest=merged_digest,
description=_build_pex_description(request),
output_files=output_files,
output_directories=output_directories,
),
)
# NB: Building a Pex is platform dependent, so in order to get a PEX that we can use locally
# without cross-building, we specify that our PEX command should be run on the current local
# platform.
result = await Get(ProcessResult, MultiPlatformProcess({platform:
process}))
if pex_runtime_env.verbosity > 0:
log_output = result.stderr.decode()
if log_output:
logger.info("%s", log_output)
digest = (await Get(
Digest,
MergeDigests(
(result.output_digest, *(pex.digest for pex in request.pex_path))))
if request.pex_path else result.output_digest)
return BuildPexResult(result=result,
pex_filename=request.output_filename,
digest=digest,
python=python)