async def setup_pex_process(request: PexProcess,
pex_environment: PexEnvironment) -> Process:
argv = pex_environment.create_argv(
f"./{request.pex.name}",
*request.argv,
python=request.pex.python,
)
env = {
**pex_environment.environment_dict(python_configured=request.pex.python is not None),
**(request.extra_env or {}),
}
process = Process(
argv,
description=request.description,
level=request.level,
input_digest=request.input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
timeout_seconds=request.timeout_seconds,
execution_slot_variable=request.execution_slot_variable,
)
return await Get(
Process,
UncacheableProcess(process)) if request.uncacheable else process
async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment) -> Process:
pex = request.pex
argv = pex_environment.create_argv(pex.name, *request.argv, python=pex.python)
env = {
**pex_environment.environment_dict(python_configured=pex.python is not None),
**(request.extra_env or {}),
}
input_digest = (
await Get(Digest, MergeDigests((pex.digest, request.input_digest)))
if request.input_digest
else pex.digest
)
return Process(
argv,
description=request.description,
level=request.level,
input_digest=input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches=pex_environment.append_only_caches(),
timeout_seconds=request.timeout_seconds,
execution_slot_variable=request.execution_slot_variable,
cache_scope=request.cache_scope,
)
async def create_python_repl_request(repl: PythonRepl,
pex_env: PexEnvironment) -> ReplRequest:
requirements_request = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in repl.targets), internal_only=True),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(repl.targets, include_files=True))
requirements_pex, sources = await MultiGet(requirements_request,
sources_request)
merged_digest = await Get(
Digest,
MergeDigests(
(requirements_pex.digest, sources.source_files.snapshot.digest)))
args = pex_env.create_argv(repl.in_chroot(requirements_pex.name),
python=requirements_pex.python)
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=requirements_pex.python is not None),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_ipython_repl_request(repl: IPythonRepl, ipython: IPython,
pex_env: PexEnvironment) -> ReplRequest:
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in ipython_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in repl.targets), internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(repl.targets, include_files=True))
ipython_request = Get(
Pex,
PexRequest(
output_filename="ipython.pex",
main=ipython.main,
requirements=PexRequirements(ipython.all_requirements),
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
internal_only=True,
),
)
requirements_pex, sources, ipython_pex = await MultiGet(
requirements_request, sources_request, ipython_request)
merged_digest = await Get(
Digest,
MergeDigests(
(requirements_pex.digest, sources.source_files.snapshot.digest,
ipython_pex.digest)),
)
args = list(
pex_env.create_argv(repl.in_chroot(ipython_pex.name),
python=ipython_pex.python))
if ipython.options.ignore_cwd:
args.append("--ignore-cwd")
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=ipython_pex.python is not None),
"PEX_PATH":
repl.in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
def _create_venv_script(
self,
bash: BashBinary,
pex_environment: PexEnvironment,
*,
script_path: PurePath,
venv_executable: PurePath,
) -> VenvScript:
env_vars = (
f"{name}={shlex.quote(value)}"
for name, value in pex_environment.environment_dict(python_configured=True).items()
)
target_venv_executable = shlex.quote(str(venv_executable))
venv_dir = shlex.quote(str(self.venv_dir))
execute_pex_args = " ".join(
shlex.quote(arg)
for arg in pex_environment.create_argv(self.pex.name, python=self.pex.python)
)
script = dedent(
f"""\
#!{bash.path}
set -euo pipefail
export {" ".join(env_vars)}
# Let PEX_TOOLS invocations pass through to the original PEX file since venvs don't come
# with tools support.
if [ -n "${{PEX_TOOLS:-}}" ]; then
exec {execute_pex_args} "$@"
fi
# If the seeded venv has been removed from the PEX_ROOT, we re-seed from the original
# `--venv` mode PEX file.
if [ ! -e {target_venv_executable} ]; then
rm -rf {venv_dir} || true
PEX_INTERPRETER=1 {execute_pex_args} -c ''
fi
exec {target_venv_executable} "$@"
"""
)
return VenvScript(
script=Script(script_path),
content=FileContent(path=str(script_path), content=script.encode(), is_executable=True),
)
async def create_pex_binary_run_request(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([field_set.address],
internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
internal_only=True,
# Note that the entry point file is not in the PEX itself. It's loaded by setting
# `PEX_EXTRA_SYS_PATH`.
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=entry_point.val,
),
)
requirements, sources, runner_pex = await MultiGet(requirements_request,
sources_request,
runner_pex_request)
merged_digest = await Get(
Digest,
MergeDigests([
requirements.digest, sources.source_files.snapshot.digest,
runner_pex.digest
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
args = pex_env.create_argv(in_chroot(runner_pex.name),
python=runner_pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=runner_pex.python is not None),
"PEX_PATH":
in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pex_cli_process(
request: PexCliProcess,
pex_binary: PexBinary,
pex_env: PexEnvironment,
python_native_code: PythonNativeCode,
global_options: GlobalOptions,
pex_runtime_env: PexRuntimeEnvironment,
) -> Process:
tmpdir = ".tmp"
gets: List[Get] = [
Get(DownloadedExternalTool, ExternalToolRequest, pex_binary.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(tmpdir)])),
]
cert_args = []
# The certs file will typically not be in the repo, so we can't digest it via a PathGlobs.
# Instead we manually create a FileContent for it.
if global_options.options.ca_certs_path:
ca_certs_content = Path(global_options.options.ca_certs_path).read_bytes()
chrooted_ca_certs_path = os.path.basename(global_options.options.ca_certs_path)
gets.append(
Get(
Digest,
CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content),)),
)
)
cert_args = ["--cert", chrooted_ca_certs_path]
downloaded_pex_bin, *digests_to_merge = await MultiGet(gets)
digests_to_merge.append(downloaded_pex_bin.digest)
if request.additional_input_digest:
digests_to_merge.append(request.additional_input_digest)
input_digest = await Get(Digest, MergeDigests(digests_to_merge))
pex_root_path = ".cache/pex_root"
argv = [
downloaded_pex_bin.exe,
*cert_args,
"--python-path",
create_path_env_var(pex_env.interpreter_search_paths),
"--pex-root",
pex_root_path,
# Ensure Pex and its subprocesses create temporary files in the the process execution
# sandbox. It may make sense to do this generally for Processes, but in the short term we
# have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to
# perform. Making the TMPDIR local to the sandbox allows control via
# --local-execution-root-dir for the local case and should work well with remote cases where
# a remoting implementation has to allow for processes producing large binaries in a
# sandbox to support reasonable workloads. Communicating TMPDIR via --tmpdir instead of via
# environment variable allows Pex to absolutize the path ensuring subprocesses that change
# CWD can find the TMPDIR.
"--tmpdir",
tmpdir,
]
if pex_runtime_env.verbosity > 0:
argv.append(f"-{'v' * pex_runtime_env.verbosity}")
# NB: This comes at the end of the argv because the request may use `--` passthrough args,
# which must come at the end.
argv.extend(request.argv)
normalized_argv = pex_env.create_argv(*argv, python=request.python)
env = {
**pex_env.environment_dict(python_configured=request.python is not None),
**python_native_code.environment_dict,
**(request.extra_env or {}),
}
return Process(
normalized_argv,
description=request.description,
input_digest=input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches={"pex_root": pex_root_path},
level=request.level,
cache_scope=request.cache_scope,
)
async def create_python_binary_run_request(
field_set: PythonBinaryFieldSet,
python_binary_defaults: PythonBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point = field_set.entry_point.value
if entry_point is None:
binary_source_paths = await Get(
Paths, PathGlobs,
field_set.sources.path_globs(FilesNotFoundBehavior.error))
if len(binary_source_paths.files) != 1:
raise InvalidFieldException(
"No `entry_point` was set for the target "
f"{repr(field_set.address)}, so it must have exactly one source, but it has "
f"{len(binary_source_paths.files)}")
entry_point_path = binary_source_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
os.path.relpath(entry_point_path, source_root.path))
transitive_targets = await Get(TransitiveTargets,
Addresses([field_set.address]))
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([field_set.address],
internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
additional_args=field_set.generate_additional_args(
python_binary_defaults),
internal_only=True,
# Note that the entry point file is not in the Pex itself, but on the
# PEX_PATH. This works fine!
entry_point=entry_point,
),
)
requirements, sources, runner_pex = await MultiGet(requirements_request,
sources_request,
runner_pex_request)
merged_digest = await Get(
Digest,
MergeDigests([
requirements.digest, sources.source_files.snapshot.digest,
runner_pex.digest
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
args = pex_env.create_argv(in_chroot(runner_pex.name),
python=runner_pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=runner_pex.python is not None),
"PEX_PATH":
in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pex_cli_process(
request: PexCliProcess,
pex_binary: PexBinary,
pex_env: PexEnvironment,
python_native_code: PythonNativeCode,
global_options: GlobalOptions,
) -> Process:
tmpdir = ".tmp"
gets: List[Get] = [
Get(DownloadedExternalTool, ExternalToolRequest,
pex_binary.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(f"{tmpdir}/.reserve")])),
]
cert_args = []
# The certs file will typically not be in the repo, so we can't digest it via a PathGlobs.
# Instead we manually create a FileContent for it.
if global_options.options.ca_certs_path:
ca_certs_content = Path(
global_options.options.ca_certs_path).read_bytes()
chrooted_ca_certs_path = os.path.basename(
global_options.options.ca_certs_path)
gets.append(
Get(
Digest,
CreateDigest((FileContent(chrooted_ca_certs_path,
ca_certs_content), )),
))
cert_args = ["--cert", chrooted_ca_certs_path]
downloaded_pex_bin, *digests_to_merge = await MultiGet(gets)
digests_to_merge.append(downloaded_pex_bin.digest)
if request.additional_input_digest:
digests_to_merge.append(request.additional_input_digest)
input_digest = await Get(Digest, MergeDigests(digests_to_merge))
pex_root_path = ".cache/pex_root"
argv = pex_env.create_argv(
downloaded_pex_bin.exe,
*request.argv,
*cert_args,
"--pex-root",
pex_root_path,
python=request.python,
)
env = {
# Ensure Pex and its subprocesses create temporary files in the the process execution
# sandbox. It may make sense to do this generally for Processes, but in the short term we
# have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to
# perform. Making the TMPDIR local to the sandbox allows control via
# --local-execution-root-dir for the local case and should work well with remote cases where
# a remoting implementation has to allow for processes producing large binaries in a
# sandbox to support reasonable workloads.
"TMPDIR": tmpdir,
**pex_env.environment_dict(python_configured=request.python is not None),
**python_native_code.environment_dict,
**(request.extra_env or {}),
}
return Process(
argv,
description=request.description,
input_digest=input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches={"pex_root": pex_root_path},
level=request.level,
)