def create_pex(request: CreatePex, pex_bin: DownloadedPexBin,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
pex_build_environment: PexBuildEnvironment,
platform: Platform) -> Pex:
"""Returns a PEX with the given requirements, optional entry point, and optional
interpreter constraints."""
interpreter_constraint_args = []
for constraint in request.interpreter_constraints:
interpreter_constraint_args.extend(
["--interpreter-constraint", constraint])
argv = ["--output-file", request.output_filename]
if request.entry_point is not None:
argv.extend(["--entry-point", request.entry_point])
argv.extend(interpreter_constraint_args + list(request.requirements))
source_dir_name = 'source_files'
argv.append(f'--sources-directory={source_dir_name}')
sources_digest = request.input_files_digest if request.input_files_digest else EMPTY_DIRECTORY_DIGEST
sources_digest_as_subdir = yield Get(
Digest, DirectoryWithPrefixToAdd(sources_digest, source_dir_name))
all_inputs = (
pex_bin.directory_digest,
sources_digest_as_subdir,
)
merged_digest = yield Get(Digest,
DirectoriesToMerge(directories=all_inputs))
# NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without
# cross-building we specify that out PEX command be run on the current local platform. When we
# support cross-building through CLI flags we can configure requests that build a PEX for out
# local platform that are able to execute on a different platform, but for now in order to
# guarantee correct build we need to restrict this command to execute on the same platform type
# that the output is intended for. The correct way to interpret the keys
# (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of
# this command is intended for `target_platform_constraint` iff it is run on `execution_platform
# constraint`".
execute_process_request = MultiPlatformExecuteProcessRequest(
{
(PlatformConstraint(platform.value),
PlatformConstraint(platform.value)):
pex_bin.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_build_environment=pex_build_environment,
pex_args=argv,
input_files=merged_digest,
description=
f"Create a requirements PEX: {', '.join(request.requirements)}",
output_files=(request.output_filename, ))
})
result = yield Get(ExecuteProcessResult,
MultiPlatformExecuteProcessRequest,
execute_process_request)
yield Pex(directory_digest=result.output_directory_digest)
def create_requirements_pex(request, pex_bin, python_setup,
pex_build_environment, platform):
"""Returns a PEX with the given requirements, optional entry point, and optional
interpreter constraints."""
interpreter_search_paths = create_path_env_var(
python_setup.interpreter_search_paths)
env = {
"PATH": interpreter_search_paths,
**pex_build_environment.invocation_environment_dict
}
interpreter_constraint_args = []
for constraint in request.interpreter_constraints:
interpreter_constraint_args.extend(
["--interpreter-constraint", constraint])
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
# TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target
# platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735.
argv = [
"python", f"./{pex_bin.executable}", "--output-file",
request.output_filename
]
if request.entry_point is not None:
argv.extend(["--entry-point", request.entry_point])
argv.extend(interpreter_constraint_args + list(request.requirements))
# NOTE
# PEX outputs are platform dependent so in order to get a PEX that we can use locally, without cross-building
# we specify that out PEX command be run on the current local platform. When we support cross-building
# through CLI flags we can configure requests that build a PEX for out local platform that are
# able to execute on a different platform, but for now in order to guarantee correct build we need
# to restrict this command to execute on the same platform type that the output is intended for.
# The correct way to interpret the keys (execution_platform_constraint, target_platform_constraint)
# of this dictionary is "The output of this command is intended for `target_platform_constraint` iff
# it is run on `execution_platform_constraint`".
execute_process_request = MultiPlatformExecuteProcessRequest(
{
(PlatformConstraint(platform.value),
PlatformConstraint(platform.value)):
ExecuteProcessRequest(
argv=tuple(argv),
env=env,
input_files=pex_bin.directory_digest,
description=
f"Create a requirements PEX: {', '.join(request.requirements)}",
output_files=(request.output_filename, ))
})
result = yield Get(ExecuteProcessResult,
MultiPlatformExecuteProcessRequest,
execute_process_request)
yield RequirementsPex(directory_digest=result.output_directory_digest)
def _deserialized_mapping(
self, ) -> Dict[PlatformConstraint, ToolForPlatform]:
deserialized: Dict[PlatformConstraint, ToolForPlatform] = {}
for platform_constraint, (version, fingerprint,
size_bytes) in self.version_digest_mapping:
deserialized[PlatformConstraint(
platform_constraint)] = ToolForPlatform(
version=ToolVersion(version),
digest=Digest(fingerprint, size_bytes),
)
return deserialized
def __new__(cls, request_dict):
if len(request_dict) == 0:
raise cls.make_type_error("At least one platform constrained ExecuteProcessRequest must be passed.")
# validate the platform constraints using the platforms enum an flatten the keys.
validated_constraints = tuple(
constraint.value
for pair in request_dict.keys() for constraint in pair
if PlatformConstraint(constraint.value)
)
if len({req.description for req in request_dict.values()}) != 1:
raise ValueError(f"The `description` of all execute_process_requests in a {cls.__name__} must be identical.")
return super().__new__(
cls,
validated_constraints,
tuple(request_dict.values())
)
def __init__(
self, request_dict: Dict[Tuple[PlatformConstraint, PlatformConstraint], Process],
) -> None:
if len(request_dict) == 0:
raise ValueError("At least one platform constrained Process must be passed.")
validated_constraints = tuple(
constraint.value
for pair in request_dict.keys()
for constraint in pair
if PlatformConstraint(constraint.value)
)
if len({req.description for req in request_dict.values()}) != 1:
raise ValueError(
f"The `description` of all processes in a {MultiPlatformProcess.__name__} must be identical."
)
self.platform_constraints = validated_constraints
self.processes = tuple(request_dict.values())
async def create_pex(
request: CreatePex,
pex_bin: DownloadedPexBin,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
pex_build_environment: PexBuildEnvironment,
platform: Platform,
) -> Pex:
"""Returns a PEX with the given requirements, optional entry point, optional interpreter
constraints, and optional requirement constraints."""
argv = [
"--output-file",
request.output_filename,
*request.interpreter_constraints.generate_pex_arg_list(),
*request.additional_args,
]
if python_setup.resolver_jobs:
argv.extend(["--jobs", python_setup.resolver_jobs])
if python_setup.manylinux:
argv.extend(["--manylinux", python_setup.manylinux])
else:
argv.append("--no-manylinux")
if request.entry_point is not None:
argv.extend(["--entry-point", request.entry_point])
if python_setup.requirement_constraints is not None:
argv.extend(["--constraints", python_setup.requirement_constraints])
source_dir_name = "source_files"
argv.append(f"--sources-directory={source_dir_name}")
argv.extend(request.requirements.requirements)
constraint_file_snapshot = EMPTY_SNAPSHOT
if python_setup.requirement_constraints is not None:
constraint_file_snapshot = await Get[Snapshot](PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin=
"the option `--python-setup-requirement-constraints`",
))
sources_digest = (request.input_files_digest if request.input_files_digest
else EMPTY_DIRECTORY_DIGEST)
sources_digest_as_subdir = await Get[Digest](DirectoryWithPrefixToAdd(
sources_digest, source_dir_name))
merged_digest = await Get[Digest](DirectoriesToMerge(directories=(
pex_bin.directory_digest,
sources_digest_as_subdir,
constraint_file_snapshot.directory_digest,
)))
# NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without
# cross-building, we specify that our PEX command be run on the current local platform. When we
# support cross-building through CLI flags we can configure requests that build a PEX for our
# local platform that are able to execute on a different platform, but for now in order to
# guarantee correct build we need to restrict this command to execute on the same platform type
# that the output is intended for. The correct way to interpret the keys
# (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of
# this command is intended for `target_platform_constraint` iff it is run on `execution_platform
# constraint`".
execute_process_request = MultiPlatformExecuteProcessRequest({
(
PlatformConstraint(platform.value),
PlatformConstraint(platform.value),
):
pex_bin.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_build_environment=pex_build_environment,
pex_args=argv,
input_files=merged_digest,
description=
f"Create a requirements PEX: {', '.join(request.requirements.requirements)}",
output_files=(request.output_filename, ),
)
})
result = await Get[ExecuteProcessResult](
MultiPlatformExecuteProcessRequest, execute_process_request)
return Pex(directory_digest=result.output_directory_digest,
output_filename=request.output_filename)
async def create_pex(
request: PexRequest,
pex_bin: DownloadedPexBin,
python_setup: PythonSetup,
python_repos: PythonRepos,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
pex_build_environment: PexBuildEnvironment,
platform: Platform,
log_level: LogLevel,
) -> Pex:
"""Returns a PEX with the given requirements, optional entry point, optional interpreter
constraints, and optional requirement constraints."""
argv = [
"--output-file",
request.output_filename,
*request.interpreter_constraints.generate_pex_arg_list(),
# NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes`
# including PyPI, which will override `--no-pypi` and result in using PyPI in the default
# case. Why set `--no-pypi`, then? We need to do this so that
# `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI.
"--no-pypi",
*(f"--index={index}" for index in python_repos.indexes),
*(f"--repo={repo}" for repo in python_repos.repos),
*request.additional_args,
]
pex_debug = PexDebug(log_level)
argv.extend(pex_debug.iter_pex_args())
if python_setup.resolver_jobs:
argv.extend(["--jobs", python_setup.resolver_jobs])
if python_setup.manylinux:
argv.extend(["--manylinux", python_setup.manylinux])
else:
argv.append("--no-manylinux")
if request.entry_point is not None:
argv.extend(["--entry-point", request.entry_point])
if python_setup.requirement_constraints is not None:
argv.extend(["--constraints", python_setup.requirement_constraints])
source_dir_name = "source_files"
argv.append(f"--sources-directory={source_dir_name}")
argv.extend(request.requirements.requirements)
constraint_file_snapshot = EMPTY_SNAPSHOT
if python_setup.requirement_constraints is not None:
constraint_file_snapshot = await Get[Snapshot](PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin=
"the option `--python-setup-requirement-constraints`",
))
sources_digest_as_subdir = await Get[Digest](DirectoryWithPrefixToAdd(
request.sources or EMPTY_DIRECTORY_DIGEST, source_dir_name))
additional_inputs_digest = request.additional_inputs or EMPTY_DIRECTORY_DIGEST
merged_digest = await Get[Digest](DirectoriesToMerge(directories=(
pex_bin.directory_digest,
sources_digest_as_subdir,
additional_inputs_digest,
constraint_file_snapshot.directory_digest,
)))
# NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without
# cross-building, we specify that our PEX command be run on the current local platform. When we
# support cross-building through CLI flags we can configure requests that build a PEX for our
# local platform that are able to execute on a different platform, but for now in order to
# guarantee correct build we need to restrict this command to execute on the same platform type
# that the output is intended for. The correct way to interpret the keys
# (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of
# this command is intended for `target_platform_constraint` iff it is run on `execution_platform
# constraint`".
description = request.description
if description is None:
if request.requirements.requirements:
description = f"Resolving {', '.join(request.requirements.requirements)}"
else:
description = f"Building PEX"
execute_process_request = MultiPlatformExecuteProcessRequest({
(
PlatformConstraint(platform.value),
PlatformConstraint(platform.value),
):
pex_bin.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_build_environment=pex_build_environment,
pex_args=argv,
input_files=merged_digest,
description=description,
output_files=(request.output_filename, ),
)
})
result = await Get[ExecuteProcessResult](
MultiPlatformExecuteProcessRequest, execute_process_request)
if pex_debug.might_log:
lines = result.stderr.decode().splitlines()
if lines:
pex_debug.log(
f"Debug output from Pex for: {execute_process_request}")
for line in lines:
pex_debug.log(line)
return Pex(directory_digest=result.output_directory_digest,
output_filename=request.output_filename)
async def create_pex(
request: PexRequest,
python_setup: PythonSetup,
python_repos: PythonRepos,
platform: Platform,
pex_runtime_environment: PexRuntimeEnvironment,
) -> Pex:
"""Returns a PEX with the given settings."""
argv = [
"--output-file",
request.output_filename,
# NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes`
# including PyPI, which will override `--no-pypi` and result in using PyPI in the default
# case. Why set `--no-pypi`, then? We need to do this so that
# `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI.
"--no-pypi",
*(f"--index={index}" for index in python_repos.indexes),
*(f"--repo={repo}" for repo in python_repos.repos),
*request.additional_args,
]
if request.internal_only:
# This will result in a faster build, but worse compatibility at runtime.
argv.append("--use-first-matching-interpreter")
# NB: If `--platform` is specified, this signals that the PEX should not be built locally.
# `--interpreter-constraint` only makes sense in the context of building locally. These two
# flags are mutually exclusive. See https://github.com/pantsbuild/pex/issues/957.
if request.platforms:
# TODO(#9560): consider validating that these platforms are valid with the interpreter
# constraints.
argv.extend(request.platforms.generate_pex_arg_list())
else:
argv.extend(request.interpreter_constraints.generate_pex_arg_list())
argv.append("--no-emit-warnings")
verbosity = pex_runtime_environment.verbosity
if verbosity > 0:
argv.append(f"-{'v' * verbosity}")
if python_setup.resolver_jobs:
argv.extend(["--jobs", str(python_setup.resolver_jobs)])
if python_setup.manylinux:
argv.extend(["--manylinux", python_setup.manylinux])
else:
argv.append("--no-manylinux")
if request.entry_point is not None:
argv.extend(["--entry-point", request.entry_point])
if python_setup.requirement_constraints is not None:
argv.extend(["--constraints", python_setup.requirement_constraints])
source_dir_name = "source_files"
argv.append(f"--sources-directory={source_dir_name}")
argv.extend(request.requirements)
constraint_file_digest = EMPTY_DIGEST
if python_setup.requirement_constraints is not None:
constraint_file_digest = await Get(
Digest,
PathGlobs(
[python_setup.requirement_constraints],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin=
"the option `--python-setup-requirement-constraints`",
),
)
sources_digest_as_subdir = await Get(
Digest, AddPrefix(request.sources or EMPTY_DIGEST, source_dir_name))
additional_inputs_digest = request.additional_inputs or EMPTY_DIGEST
merged_digest = await Get(
Digest,
MergeDigests((
sources_digest_as_subdir,
additional_inputs_digest,
constraint_file_digest,
)),
)
description = request.description
if description is None:
if request.requirements:
description = (
f"Building {request.output_filename} with "
f"{pluralize(len(request.requirements), 'requirement')}: "
f"{', '.join(request.requirements)}")
else:
description = f"Building {request.output_filename}"
process = await Get(
Process,
PexCliProcess(
argv=argv,
additional_input_digest=merged_digest,
description=description,
output_files=[request.output_filename],
),
)
# NB: Building a Pex is platform dependent, so in order to get a PEX that we can use locally
# without cross-building, we specify that our PEX command should be run on the current local
# platform.
result = await Get(
ProcessResult,
MultiPlatformProcess({
(PlatformConstraint(platform.value),
PlatformConstraint(platform.value)):
process
}),
)
if verbosity > 0:
log_output = result.stderr.decode()
if log_output:
logger.info("%s", log_output)
return Pex(
digest=result.output_digest,
name=request.output_filename,
internal_only=request.internal_only,
)