async def a_rule() -> TrueResult:
proc = Process(
["/bin/sh", "-c", "true"],
description="always true",
)
_ = await Get(ProcessResult, MultiPlatformProcess({None: proc}))
return TrueResult()
async def a_rule() -> TrueResult:
proc = Process(
["/bin/sh", "-c", "true"],
description="always true",
cache_scope=ProcessCacheScope.PER_SESSION,
)
_ = await Get(ProcessResult, MultiPlatformProcess({None: proc}))
return TrueResult()
async def build_pex(
request: PexRequest,
python_setup: PythonSetup,
python_repos: PythonRepos,
platform: Platform,
pex_runtime_env: PexRuntimeEnvironment,
) -> BuildPexResult:
"""Returns a PEX with the given settings."""
argv = ["--output-file", request.output_filename, *request.additional_args]
repository_pex = (request.requirements.repository_pex if isinstance(
request.requirements, PexRequirements) else None)
if repository_pex:
argv.extend(["--pex-repository", repository_pex.name])
else:
# NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes`
# including PyPI, which will override `--no-pypi` and result in using PyPI in the default
# case. Why set `--no-pypi`, then? We need to do this so that
# `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI.
argv.extend([
"--no-pypi",
*(f"--index={index}" for index in python_repos.indexes),
*(f"--repo={repo}" for repo in python_repos.repos),
"--resolver-version",
"pip-2020-resolver",
])
python: PythonExecutable | None = None
# NB: If `--platform` is specified, this signals that the PEX should not be built locally.
# `--interpreter-constraint` only makes sense in the context of building locally. These two
# flags are mutually exclusive. See https://github.com/pantsbuild/pex/issues/957.
if request.platforms:
# TODO(#9560): consider validating that these platforms are valid with the interpreter
# constraints.
argv.extend(request.platforms.generate_pex_arg_list())
elif request.python:
python = request.python
elif request.internal_only:
# NB: If it's an internal_only PEX, we do our own lookup of the interpreter based on the
# interpreter constraints, and then will run the PEX with that specific interpreter. We
# will have already validated that there were no platforms.
python = await Get(PythonExecutable, InterpreterConstraints,
request.interpreter_constraints)
else:
# `--interpreter-constraint` options are mutually exclusive with the `--python` option,
# so we only specify them if we have not already located a concrete Python.
argv.extend(request.interpreter_constraints.generate_pex_arg_list())
if python:
argv.extend(["--python", python.path])
argv.append("--no-emit-warnings")
if python_setup.resolver_jobs:
argv.extend(["--jobs", str(python_setup.resolver_jobs)])
if python_setup.manylinux:
argv.extend(["--manylinux", python_setup.manylinux])
else:
argv.append("--no-manylinux")
if request.main is not None:
argv.extend(request.main.iter_pex_args())
# TODO(John Sirois): Right now any request requirements will shadow corresponding pex path
# requirements, which could lead to problems. Support shading python binaries.
# See: https://github.com/pantsbuild/pants/issues/9206
if request.pex_path:
argv.extend(
["--pex-path", ":".join(pex.name for pex in request.pex_path)])
source_dir_name = "source_files"
argv.append(f"--sources-directory={source_dir_name}")
sources_digest_as_subdir = await Get(
Digest, AddPrefix(request.sources or EMPTY_DIGEST, source_dir_name))
additional_inputs_digest = request.additional_inputs or EMPTY_DIGEST
repository_pex_digest = repository_pex.digest if repository_pex else EMPTY_DIGEST
constraint_file_digest = EMPTY_DIGEST
requirements_file_digest = EMPTY_DIGEST
# TODO(#12314): Capture the resolve name for multiple user lockfiles.
resolve_name = (request.requirements.options_scope_name if isinstance(
request.requirements,
(ToolDefaultLockfile, ToolCustomLockfile)) else None)
if isinstance(request.requirements, Lockfile):
is_monolithic_resolve = True
argv.extend(["--requirement", request.requirements.file_path])
argv.append("--no-transitive")
globs = PathGlobs(
[request.requirements.file_path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=request.requirements.
file_path_description_of_origin,
)
if python_setup.invalid_lockfile_behavior in {
InvalidLockfileBehavior.warn,
InvalidLockfileBehavior.error,
}:
requirements_file_digest_contents = await Get(
DigestContents, PathGlobs, globs)
metadata = LockfileMetadata.from_lockfile(
requirements_file_digest_contents[0].content,
request.requirements.file_path,
resolve_name,
)
_validate_metadata(metadata, request, request.requirements,
python_setup)
requirements_file_digest = await Get(Digest, PathGlobs, globs)
elif isinstance(request.requirements, LockfileContent):
is_monolithic_resolve = True
file_content = request.requirements.file_content
argv.extend(["--requirement", file_content.path])
argv.append("--no-transitive")
if python_setup.invalid_lockfile_behavior in {
InvalidLockfileBehavior.warn,
InvalidLockfileBehavior.error,
}:
metadata = LockfileMetadata.from_lockfile(
file_content.content, resolve_name=resolve_name)
_validate_metadata(metadata, request, request.requirements,
python_setup)
requirements_file_digest = await Get(Digest,
CreateDigest([file_content]))
else:
assert isinstance(request.requirements, PexRequirements)
is_monolithic_resolve = request.requirements.is_all_constraints_resolve
if (request.requirements.apply_constraints
and python_setup.requirement_constraints is not None):
argv.extend(
["--constraints", python_setup.requirement_constraints])
constraint_file_digest = await Get(
Digest,
PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=
"the option `[python].requirement_constraints`",
),
)
argv.extend(request.requirements.req_strings)
merged_digest = await Get(
Digest,
MergeDigests((
sources_digest_as_subdir,
additional_inputs_digest,
constraint_file_digest,
requirements_file_digest,
repository_pex_digest,
*(pex.digest for pex in request.pex_path),
)),
)
output_files: Iterable[str] | None = None
output_directories: Iterable[str] | None = None
if request.internal_only or is_monolithic_resolve:
# This is a much friendlier layout for the CAS than the default zipapp.
argv.extend(["--layout", "packed"])
output_directories = [request.output_filename]
else:
output_files = [request.output_filename]
process = await Get(
Process,
PexCliProcess(
python=python,
argv=argv,
additional_input_digest=merged_digest,
description=_build_pex_description(request),
output_files=output_files,
output_directories=output_directories,
),
)
# NB: Building a Pex is platform dependent, so in order to get a PEX that we can use locally
# without cross-building, we specify that our PEX command should be run on the current local
# platform.
result = await Get(ProcessResult, MultiPlatformProcess({platform:
process}))
if pex_runtime_env.verbosity > 0:
log_output = result.stderr.decode()
if log_output:
logger.info("%s", log_output)
digest = (await Get(
Digest,
MergeDigests(
(result.output_digest, *(pex.digest for pex in request.pex_path))))
if request.pex_path else result.output_digest)
return BuildPexResult(result=result,
pex_filename=request.output_filename,
digest=digest,
python=python)
async def build_pex(
request: PexRequest,
python_setup: PythonSetup,
python_repos: PythonRepos,
platform: Platform,
pex_runtime_env: PexRuntimeEnvironment,
) -> BuildPexResult:
"""Returns a PEX with the given settings."""
argv = [
"--output-file",
request.output_filename,
# NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes`
# including PyPI, which will override `--no-pypi` and result in using PyPI in the default
# case. Why set `--no-pypi`, then? We need to do this so that
# `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI.
"--no-pypi",
*(f"--index={index}" for index in python_repos.indexes),
*(f"--repo={repo}" for repo in python_repos.repos),
"--resolver-version",
python_setup.resolver_version.value,
*request.additional_args,
]
python: PythonExecutable | None = None
# NB: If `--platform` is specified, this signals that the PEX should not be built locally.
# `--interpreter-constraint` only makes sense in the context of building locally. These two
# flags are mutually exclusive. See https://github.com/pantsbuild/pex/issues/957.
if request.platforms:
# TODO(#9560): consider validating that these platforms are valid with the interpreter
# constraints.
argv.extend(request.platforms.generate_pex_arg_list())
else:
# NB: If it's an internal_only PEX, we do our own lookup of the interpreter based on the
# interpreter constraints, and then will run the PEX with that specific interpreter. We
# will have already validated that there were no platforms.
# Otherwise, we let Pex resolve the constraints.
if request.internal_only:
python = await Get(
PythonExecutable, PexInterpreterConstraints, request.interpreter_constraints
)
else:
argv.extend(request.interpreter_constraints.generate_pex_arg_list())
argv.append("--no-emit-warnings")
if python_setup.resolver_jobs:
argv.extend(["--jobs", str(python_setup.resolver_jobs)])
if python_setup.manylinux:
argv.extend(["--manylinux", python_setup.manylinux])
else:
argv.append("--no-manylinux")
if request.main is not None:
argv.extend(request.main.iter_pex_args())
source_dir_name = "source_files"
argv.append(f"--sources-directory={source_dir_name}")
argv.extend(request.requirements)
constraint_file_digest = EMPTY_DIGEST
if request.apply_requirement_constraints and python_setup.requirement_constraints is not None:
argv.extend(["--constraints", python_setup.requirement_constraints])
constraint_file_digest = await Get(
Digest,
PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin="the option `--python-setup-requirement-constraints`",
),
)
sources_digest_as_subdir = await Get(
Digest, AddPrefix(request.sources or EMPTY_DIGEST, source_dir_name)
)
additional_inputs_digest = request.additional_inputs or EMPTY_DIGEST
merged_digest = await Get(
Digest,
MergeDigests(
(
sources_digest_as_subdir,
additional_inputs_digest,
constraint_file_digest,
*(pex.digest for pex in request.pex_path),
)
),
)
# TODO(John Sirois): Right now any request requirements will shadow corresponding pex path
# requirements, which could lead to problems. Support shading python binaries.
# See: https://github.com/pantsbuild/pants/issues/9206
if request.pex_path:
argv.extend(["--pex-path", ":".join(pex.name for pex in request.pex_path)])
description = request.description
if description is None:
if request.requirements:
description = (
f"Building {request.output_filename} with "
f"{pluralize(len(request.requirements), 'requirement')}: "
f"{', '.join(request.requirements)}"
)
else:
description = f"Building {request.output_filename}"
process = await Get(
Process,
PexCliProcess(
python=python,
argv=argv,
additional_input_digest=merged_digest,
description=description,
output_files=[request.output_filename],
),
)
# NB: Building a Pex is platform dependent, so in order to get a PEX that we can use locally
# without cross-building, we specify that our PEX command should be run on the current local
# platform.
result = await Get(ProcessResult, MultiPlatformProcess({platform: process}))
if pex_runtime_env.verbosity > 0:
log_output = result.stderr.decode()
if log_output:
logger.info("%s", log_output)
digest = (
await Get(
Digest, MergeDigests((result.output_digest, *(pex.digest for pex in request.pex_path)))
)
if request.pex_path
else result.output_digest
)
return BuildPexResult(
result=result, pex_filename=request.output_filename, digest=digest, python=python
)
async def create_pex(
request: PexRequest,
pex_bin: DownloadedPexBin,
python_setup: PythonSetup,
python_repos: PythonRepos,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
pex_build_environment: PexBuildEnvironment,
platform: Platform,
log_level: LogLevel,
) -> Pex:
"""Returns a PEX with the given requirements, optional entry point, optional interpreter
constraints, and optional requirement constraints."""
argv = [
"--output-file",
request.output_filename,
# NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes`
# including PyPI, which will override `--no-pypi` and result in using PyPI in the default
# case. Why set `--no-pypi`, then? We need to do this so that
# `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI.
"--no-pypi",
*(f"--index={index}" for index in python_repos.indexes),
*(f"--repo={repo}" for repo in python_repos.repos),
*request.additional_args,
]
# NB: If `--platform` is specified, this signals that the PEX should not be built locally.
# `--interpreter-constraint` only makes sense in the context of building locally. These two
# flags are mutually exclusive. See https://github.com/pantsbuild/pex/issues/957.
if request.platforms:
# TODO(#9560): consider validating that these platforms are valid with the interpreter
# constraints.
argv.extend(request.platforms.generate_pex_arg_list())
else:
argv.extend(request.interpreter_constraints.generate_pex_arg_list())
pex_debug = PexDebug(log_level)
argv.extend(pex_debug.iter_pex_args())
if python_setup.resolver_jobs:
argv.extend(["--jobs", str(python_setup.resolver_jobs)])
if python_setup.manylinux:
argv.extend(["--manylinux", python_setup.manylinux])
else:
argv.append("--no-manylinux")
if request.entry_point is not None:
argv.extend(["--entry-point", request.entry_point])
if python_setup.requirement_constraints is not None:
argv.extend(["--constraints", python_setup.requirement_constraints])
source_dir_name = "source_files"
argv.append(f"--sources-directory={source_dir_name}")
argv.extend(request.requirements)
constraint_file_snapshot = EMPTY_SNAPSHOT
if python_setup.requirement_constraints is not None:
constraint_file_snapshot = await Get[Snapshot](
PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin="the option `--python-setup-requirement-constraints`",
)
)
sources_digest_as_subdir = await Get[Digest](
AddPrefix(request.sources or EMPTY_DIGEST, source_dir_name)
)
additional_inputs_digest = request.additional_inputs or EMPTY_DIGEST
merged_digest = await Get[Digest](
MergeDigests(
(
pex_bin.digest,
sources_digest_as_subdir,
additional_inputs_digest,
constraint_file_snapshot.digest,
)
)
)
# NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without
# cross-building, we specify that our PEX command be run on the current local platform. When we
# support cross-building through CLI flags we can configure requests that build a PEX for our
# local platform that are able to execute on a different platform, but for now in order to
# guarantee correct build we need to restrict this command to execute on the same platform type
# that the output is intended for. The correct way to interpret the keys
# (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of
# this command is intended for `target_platform_constraint` iff it is run on `execution_platform
# constraint`".
description = request.description
if description is None:
if request.requirements:
description = (
f"Building {request.output_filename} with "
f"{pluralize(len(request.requirements), 'requirement')}: "
f"{', '.join(request.requirements)}"
)
else:
description = f"Building {request.output_filename}"
process = MultiPlatformProcess(
{
(
PlatformConstraint(platform.value),
PlatformConstraint(platform.value),
): pex_bin.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_build_environment=pex_build_environment,
pex_args=argv,
input_digest=merged_digest,
description=description,
output_files=(request.output_filename,),
)
}
)
result = await Get[ProcessResult](MultiPlatformProcess, process)
if pex_debug.might_log:
lines = result.stderr.decode().splitlines()
if lines:
pex_debug.log(f"Debug output from Pex for: {process}")
for line in lines:
pex_debug.log(line)
return Pex(digest=result.output_digest, output_filename=request.output_filename)
async def create_pex(
request: PexRequest,
python_setup: PythonSetup,
python_repos: PythonRepos,
platform: Platform,
pex_runtime_environment: PexRuntimeEnvironment,
) -> Pex:
"""Returns a PEX with the given settings."""
argv = [
"--output-file",
request.output_filename,
# NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes`
# including PyPI, which will override `--no-pypi` and result in using PyPI in the default
# case. Why set `--no-pypi`, then? We need to do this so that
# `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI.
"--no-pypi",
*(f"--index={index}" for index in python_repos.indexes),
*(f"--repo={repo}" for repo in python_repos.repos),
*request.additional_args,
]
if request.internal_only:
# This will result in a faster build, but worse compatibility at runtime.
argv.append("--use-first-matching-interpreter")
# NB: If `--platform` is specified, this signals that the PEX should not be built locally.
# `--interpreter-constraint` only makes sense in the context of building locally. These two
# flags are mutually exclusive. See https://github.com/pantsbuild/pex/issues/957.
if request.platforms:
# TODO(#9560): consider validating that these platforms are valid with the interpreter
# constraints.
argv.extend(request.platforms.generate_pex_arg_list())
else:
argv.extend(request.interpreter_constraints.generate_pex_arg_list())
argv.append("--no-emit-warnings")
verbosity = pex_runtime_environment.verbosity
if verbosity > 0:
argv.append(f"-{'v' * verbosity}")
if python_setup.resolver_jobs:
argv.extend(["--jobs", str(python_setup.resolver_jobs)])
if python_setup.manylinux:
argv.extend(["--manylinux", python_setup.manylinux])
else:
argv.append("--no-manylinux")
if request.entry_point is not None:
argv.extend(["--entry-point", request.entry_point])
if python_setup.requirement_constraints is not None:
argv.extend(["--constraints", python_setup.requirement_constraints])
source_dir_name = "source_files"
argv.append(f"--sources-directory={source_dir_name}")
argv.extend(request.requirements)
constraint_file_digest = EMPTY_DIGEST
if python_setup.requirement_constraints is not None:
constraint_file_digest = await Get(
Digest,
PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin=
"the option `--python-setup-requirement-constraints`",
),
)
sources_digest_as_subdir = await Get(
Digest, AddPrefix(request.sources or EMPTY_DIGEST, source_dir_name))
additional_inputs_digest = request.additional_inputs or EMPTY_DIGEST
merged_digest = await Get(
Digest,
MergeDigests((
sources_digest_as_subdir,
additional_inputs_digest,
constraint_file_digest,
)),
)
description = request.description
if description is None:
if request.requirements:
description = (
f"Building {request.output_filename} with "
f"{pluralize(len(request.requirements), 'requirement')}: "
f"{', '.join(request.requirements)}")
else:
description = f"Building {request.output_filename}"
process = await Get(
Process,
PexCliProcess(
argv=argv,
additional_input_digest=merged_digest,
description=description,
output_files=[request.output_filename],
),
)
# NB: Building a Pex is platform dependent, so in order to get a PEX that we can use locally
# without cross-building, we specify that our PEX command should be run on the current local
# platform.
result = await Get(
ProcessResult,
MultiPlatformProcess({
(PlatformConstraint(platform.value),
PlatformConstraint(platform.value)):
process
}),
)
if verbosity > 0:
log_output = result.stderr.decode()
if log_output:
logger.info("%s", log_output)
return Pex(
digest=result.output_digest,
name=request.output_filename,
internal_only=request.internal_only,
)