async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment) -> Process:
pex = request.pex
argv = pex_environment.create_argv(pex.name, *request.argv, python=pex.python)
env = {
**pex_environment.environment_dict(python_configured=pex.python is not None),
**(request.extra_env or {}),
}
input_digest = (
await Get(Digest, MergeDigests((pex.digest, request.input_digest)))
if request.input_digest
else pex.digest
)
return Process(
argv,
description=request.description,
level=request.level,
input_digest=input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches=pex_environment.append_only_caches(),
timeout_seconds=request.timeout_seconds,
execution_slot_variable=request.execution_slot_variable,
cache_scope=request.cache_scope,
)
async def create_python_repl_request(repl: PythonRepl,
pex_env: PexEnvironment) -> ReplRequest:
requirements_request = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in repl.targets), internal_only=True),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(repl.targets, include_files=True))
requirements_pex, sources = await MultiGet(requirements_request,
sources_request)
merged_digest = await Get(
Digest,
MergeDigests(
(requirements_pex.digest, sources.source_files.snapshot.digest)))
args = pex_env.create_argv(repl.in_chroot(requirements_pex.name),
python=requirements_pex.python)
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=requirements_pex.python is not None),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pex_process(request: PexProcess,
pex_environment: PexEnvironment) -> Process:
argv = pex_environment.create_argv(
f"./{request.pex.name}",
*request.argv,
python=request.pex.python,
)
env = {
**pex_environment.environment_dict(python_configured=request.pex.python is not None),
**(request.extra_env or {}),
}
process = Process(
argv,
description=request.description,
level=request.level,
input_digest=request.input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
timeout_seconds=request.timeout_seconds,
execution_slot_variable=request.execution_slot_variable,
)
return await Get(
Process,
UncacheableProcess(process)) if request.uncacheable else process
async def create_ipython_repl_request(repl: IPythonRepl, ipython: IPython,
pex_env: PexEnvironment) -> ReplRequest:
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in ipython_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in repl.targets), internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(repl.targets, include_files=True))
ipython_request = Get(
Pex,
PexRequest(
output_filename="ipython.pex",
main=ipython.main,
requirements=PexRequirements(ipython.all_requirements),
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
internal_only=True,
),
)
requirements_pex, sources, ipython_pex = await MultiGet(
requirements_request, sources_request, ipython_request)
merged_digest = await Get(
Digest,
MergeDigests(
(requirements_pex.digest, sources.source_files.snapshot.digest,
ipython_pex.digest)),
)
args = list(
pex_env.create_argv(repl.in_chroot(ipython_pex.name),
python=ipython_pex.python))
if ipython.options.ignore_cwd:
args.append("--ignore-cwd")
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=ipython_pex.python is not None),
"PEX_PATH":
repl.in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_venv_pex_process(
request: VenvPexProcess, pex_environment: PexEnvironment
) -> Process:
venv_pex = request.venv_pex
pex_bin = (
os.path.relpath(venv_pex.pex.argv0, request.working_directory)
if request.working_directory
else venv_pex.pex.argv0
)
argv = (pex_bin, *request.argv)
input_digest = (
await Get(Digest, MergeDigests((venv_pex.digest, request.input_digest)))
if request.input_digest
else venv_pex.digest
)
return Process(
argv=argv,
description=request.description,
level=request.level,
input_digest=input_digest,
working_directory=request.working_directory,
env=request.extra_env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches=pex_environment.in_sandbox(
working_directory=request.working_directory
).append_only_caches,
timeout_seconds=request.timeout_seconds,
execution_slot_variable=request.execution_slot_variable,
cache_scope=request.cache_scope,
)
def _create_venv_script(
self,
bash: BashBinary,
pex_environment: PexEnvironment,
*,
script_path: PurePath,
venv_executable: PurePath,
) -> VenvScript:
env_vars = (
f"{name}={shlex.quote(value)}"
for name, value in pex_environment.environment_dict(python_configured=True).items()
)
target_venv_executable = shlex.quote(str(venv_executable))
venv_dir = shlex.quote(str(self.venv_dir))
execute_pex_args = " ".join(
shlex.quote(arg)
for arg in pex_environment.create_argv(self.pex.name, python=self.pex.python)
)
script = dedent(
f"""\
#!{bash.path}
set -euo pipefail
export {" ".join(env_vars)}
# Let PEX_TOOLS invocations pass through to the original PEX file since venvs don't come
# with tools support.
if [ -n "${{PEX_TOOLS:-}}" ]; then
exec {execute_pex_args} "$@"
fi
# If the seeded venv has been removed from the PEX_ROOT, we re-seed from the original
# `--venv` mode PEX file.
if [ ! -e {target_venv_executable} ]; then
rm -rf {venv_dir} || true
PEX_INTERPRETER=1 {execute_pex_args} -c ''
fi
exec {target_venv_executable} "$@"
"""
)
return VenvScript(
script=Script(script_path),
content=FileContent(path=str(script_path), content=script.encode(), is_executable=True),
)
def create(
cls, pex_environment: PexEnvironment, pex: Pex, venv_rel_dir: PurePath
) -> VenvScriptWriter:
# N.B.: We don't know the working directory that will be used in any given
# invocation of the venv scripts; so we deal with working_directory inside the scripts
# themselves by absolutifying all relevant paths at runtime.
complete_pex_env = pex_environment.in_sandbox(working_directory=None)
venv_dir = complete_pex_env.pex_root / venv_rel_dir
return cls(complete_pex_env=complete_pex_env, pex=pex, venv_dir=venv_dir)
def create(cls, pex_environment: PexEnvironment, pex: Pex,
venv_rel_dir: PurePath) -> VenvScriptWriter:
# N.B.: We don't know the working directory that will be used in any given
# invocation of the venv scripts; so we deal with working_directory once in an
# `adjust_relative_paths` function inside the script to save rule authors from having to do
# CWD offset math in every rule for all the relative paths their process depends on.
complete_pex_env = pex_environment.in_sandbox(working_directory=None)
venv_dir = complete_pex_env.pex_root / venv_rel_dir
return cls(complete_pex_env=complete_pex_env,
pex=pex,
venv_dir=venv_dir)
async def export_venv(request: ExportedVenvRequest, python_setup: PythonSetup,
pex_env: PexEnvironment) -> ExportableData:
# Pick a single interpreter for the venv.
interpreter_constraints = InterpreterConstraints.create_from_targets(
request.targets, python_setup)
if not interpreter_constraints:
# If there were no targets that defined any constraints, fall back to the global ones.
interpreter_constraints = InterpreterConstraints(
python_setup.interpreter_constraints)
min_interpreter = interpreter_constraints.snap_to_minimum(
python_setup.interpreter_universe)
if not min_interpreter:
raise ExportError(
"The following interpreter constraints were computed for all the targets for which "
f"export was requested: {interpreter_constraints}. There is no python interpreter "
"compatible with these constraints. Please restrict the target set to one that shares "
"a compatible interpreter.")
venv_pex = await Get(
VenvPex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in request.targets),
internal_only=True,
hardcoded_interpreter_constraints=min_interpreter,
),
)
complete_pex_env = pex_env.in_workspace()
venv_abspath = os.path.join(complete_pex_env.pex_root,
venv_pex.venv_rel_dir)
# Run the venv_pex to get the full python version (including patch #), so we
# can use it in the symlink name.
res = await Get(
ProcessResult,
VenvPexProcess(
venv_pex=venv_pex,
description="Create virtualenv",
argv=[
"-c",
"import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))"
],
input_digest=venv_pex.digest,
),
)
py_version = res.stdout.strip().decode()
return ExportableData(
f"virtualenv for {min_interpreter}",
os.path.join("python", "virtualenv"),
symlinks=[Symlink(venv_abspath, py_version)],
)
async def create_python_repl_request(request: PythonRepl,
pex_env: PexEnvironment,
python_setup: PythonSetup) -> ReplRequest:
validate_compatible_resolve(request.targets, python_setup)
interpreter_constraints, transitive_targets = await MultiGet(
Get(InterpreterConstraints,
InterpreterConstraintsRequest(request.addresses)),
Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)),
)
requirements_request = Get(Pex, RequirementsPexRequest(request.addresses))
local_dists_request = Get(
LocalDistsPex,
LocalDistsPexRequest(
request.addresses,
internal_only=True,
interpreter_constraints=interpreter_constraints,
),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
requirements_pex, local_dists, sources = await MultiGet(
requirements_request, local_dists_request, sources_request)
merged_digest = await Get(
Digest,
MergeDigests((requirements_pex.digest, local_dists.pex.digest,
sources.source_files.snapshot.digest)),
)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(request.in_chroot(
requirements_pex.name),
python=requirements_pex.python)
chrooted_source_roots = [
request.in_chroot(sr) for sr in sources.source_roots
]
extra_env = {
**complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
"PEX_PATH":
request.in_chroot(local_dists.pex.name),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_python_repl_request(repl: PythonRepl,
pex_env: PexEnvironment) -> ReplRequest:
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex) so
# that we can get the interpreter constraints for use in local_dists_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in repl.targets), internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
local_dists_request = Get(
LocalDistsPex,
LocalDistsPexRequest(
Addresses(tgt.address for tgt in repl.targets),
internal_only=True,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(repl.targets, include_files=True))
requirements_pex, local_dists, sources = await MultiGet(
requirements_request, local_dists_request, sources_request)
merged_digest = await Get(
Digest,
MergeDigests((requirements_pex.digest, local_dists.pex.digest,
sources.source_files.snapshot.digest)),
)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(repl.in_chroot(requirements_pex.name),
python=requirements_pex.python)
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
"PEX_PATH":
repl.in_chroot(local_dists.pex.name),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment) -> RunRequest:
run_in_sandbox = field_set.run_in_sandbox.value
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
addresses = [field_set.address]
interpreter_constraints = await Get(
InterpreterConstraints, InterpreterConstraintsRequest(addresses))
pex_filename = (field_set.address.generated_name.replace(".", "_")
if field_set.address.generated_name else
field_set.address.target_name)
pex_get = Get(
Pex,
PexFromTargetsRequest(
[field_set.address],
output_filename=f"{pex_filename}.pex",
internal_only=True,
include_source_files=False,
# Note that the file for first-party entry points is not in the PEX itself. In that
# case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
main=entry_point.val or field_set.script.value,
additional_args=(
*field_set.generate_additional_args(pex_binary_defaults),
# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
# below, it's important for any app that re-executes itself that these environment
# variables are not stripped.
"--no-strip-pex-env",
),
),
)
sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
pex, sources = await MultiGet(pex_get, sources_get)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[field_set.address],
internal_only=True,
interpreter_constraints=interpreter_constraints,
sources=sources,
),
)
input_digests = [
pex.digest,
local_dists.pex.digest,
# Note regarding inline mode: You might think that the sources don't need to be copied
# into the chroot when using inline sources. But they do, because some of them might be
# codegenned, and those won't exist in the inline source tree. Rather than incurring the
# complexity of figuring out here which sources were codegenned, we copy everything.
# The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline
# sources will take precedence and their copies in the chroot will be ignored.
local_dists.remaining_sources.source_files.snapshot.digest,
]
merged_digest = await Get(Digest, MergeDigests(input_digests))
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
# The order here is important: we want the in-repo sources to take precedence over their
# copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
source_roots = [
*([] if run_in_sandbox else sources.source_roots),
*chrooted_source_roots,
]
extra_env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
"PEX_PATH":
in_chroot(local_dists.pex.name),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([field_set.address],
internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
internal_only=True,
# Note that the entry point file is not in the PEX itself. It's loaded by setting
# `PEX_EXTRA_SYS_PATH`.
# TODO(John Sirois): Support ConsoleScript in PexBinary targets:
# https://github.com/pantsbuild/pants/issues/11619
main=entry_point.val,
),
)
requirements, sources, runner_pex = await MultiGet(requirements_request,
sources_request,
runner_pex_request)
merged_digest = await Get(
Digest,
MergeDigests([
requirements.digest, sources.source_files.snapshot.digest,
runner_pex.digest
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
args = pex_env.create_argv(in_chroot(runner_pex.name),
python=runner_pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=runner_pex.python is not None),
"PEX_PATH":
in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pex_cli_process(
request: PexCliProcess,
pex_binary: PexBinary,
pex_env: PexEnvironment,
python_native_code: PythonNativeCode,
global_options: GlobalOptions,
pex_runtime_env: PexRuntimeEnvironment,
) -> Process:
tmpdir = ".tmp"
gets: List[Get] = [
Get(DownloadedExternalTool, ExternalToolRequest, pex_binary.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(tmpdir)])),
]
cert_args = []
# The certs file will typically not be in the repo, so we can't digest it via a PathGlobs.
# Instead we manually create a FileContent for it.
if global_options.options.ca_certs_path:
ca_certs_content = Path(global_options.options.ca_certs_path).read_bytes()
chrooted_ca_certs_path = os.path.basename(global_options.options.ca_certs_path)
gets.append(
Get(
Digest,
CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content),)),
)
)
cert_args = ["--cert", chrooted_ca_certs_path]
downloaded_pex_bin, *digests_to_merge = await MultiGet(gets)
digests_to_merge.append(downloaded_pex_bin.digest)
if request.additional_input_digest:
digests_to_merge.append(request.additional_input_digest)
input_digest = await Get(Digest, MergeDigests(digests_to_merge))
pex_root_path = ".cache/pex_root"
argv = [
downloaded_pex_bin.exe,
*cert_args,
"--python-path",
create_path_env_var(pex_env.interpreter_search_paths),
"--pex-root",
pex_root_path,
# Ensure Pex and its subprocesses create temporary files in the the process execution
# sandbox. It may make sense to do this generally for Processes, but in the short term we
# have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to
# perform. Making the TMPDIR local to the sandbox allows control via
# --local-execution-root-dir for the local case and should work well with remote cases where
# a remoting implementation has to allow for processes producing large binaries in a
# sandbox to support reasonable workloads. Communicating TMPDIR via --tmpdir instead of via
# environment variable allows Pex to absolutize the path ensuring subprocesses that change
# CWD can find the TMPDIR.
"--tmpdir",
tmpdir,
]
if pex_runtime_env.verbosity > 0:
argv.append(f"-{'v' * pex_runtime_env.verbosity}")
# NB: This comes at the end of the argv because the request may use `--` passthrough args,
# which must come at the end.
argv.extend(request.argv)
normalized_argv = pex_env.create_argv(*argv, python=request.python)
env = {
**pex_env.environment_dict(python_configured=request.python is not None),
**python_native_code.environment_dict,
**(request.extra_env or {}),
}
return Process(
normalized_argv,
description=request.description,
input_digest=input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches={"pex_root": pex_root_path},
level=request.level,
cache_scope=request.cache_scope,
)
async def setup_pex_cli_process(
request: PexCliProcess,
pex_pex: PexPEX,
pex_env: PexEnvironment,
python_native_code: PythonNativeCode,
global_options: GlobalOptions,
pex_runtime_env: PexRuntimeEnvironment,
) -> Process:
tmpdir = ".tmp"
gets: List[Get] = [Get(Digest, CreateDigest([Directory(tmpdir)]))]
# The certs file will typically not be in the repo, so we can't digest it via a PathGlobs.
# Instead we manually create a FileContent for it.
cert_args = []
if global_options.ca_certs_path:
ca_certs_content = Path(global_options.ca_certs_path).read_bytes()
chrooted_ca_certs_path = os.path.basename(global_options.ca_certs_path)
gets.append(
Get(
Digest,
CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content),)),
)
)
cert_args = ["--cert", chrooted_ca_certs_path]
digests_to_merge = [pex_pex.digest]
digests_to_merge.extend(await MultiGet(gets))
if request.additional_input_digest:
digests_to_merge.append(request.additional_input_digest)
input_digest = await Get(Digest, MergeDigests(digests_to_merge))
global_args = [
# Ensure Pex and its subprocesses create temporary files in the the process execution
# sandbox. It may make sense to do this generally for Processes, but in the short term we
# have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to
# perform. Making the TMPDIR local to the sandbox allows control via
# --local-execution-root-dir for the local case and should work well with remote cases where
# a remoting implementation has to allow for processes producing large binaries in a
# sandbox to support reasonable workloads. Communicating TMPDIR via --tmpdir instead of via
# environment variable allows Pex to absolutize the path ensuring subprocesses that change
# CWD can find the TMPDIR.
"--tmpdir",
tmpdir,
]
if request.concurrency_available > 0:
global_args.extend(["--jobs", "{pants_concurrency}"])
if pex_runtime_env.verbosity > 0:
global_args.append(f"-{'v' * pex_runtime_env.verbosity}")
resolve_args = (
[*cert_args, "--python-path", create_path_env_var(pex_env.interpreter_search_paths)]
if request.set_resolve_args
else []
)
args = [
*global_args,
*request.subcommand,
*resolve_args,
# NB: This comes at the end because it may use `--` passthrough args, # which must come at
# the end.
*request.extra_args,
]
complete_pex_env = pex_env.in_sandbox(working_directory=None)
normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args, python=request.python)
env = {
**complete_pex_env.environment_dict(python_configured=request.python is not None),
**python_native_code.environment_dict,
**(request.extra_env or {}),
# If a subcommand is used, we need to use the `pex3` console script.
**({"PEX_SCRIPT": "pex3"} if request.subcommand else {}),
}
return Process(
normalized_argv,
description=request.description,
input_digest=input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches=complete_pex_env.append_only_caches,
level=request.level,
concurrency_available=request.concurrency_available,
cache_scope=request.cache_scope,
)
async def create_python_binary_run_request(
field_set: PythonBinaryFieldSet,
python_binary_defaults: PythonBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point = field_set.entry_point.value
if entry_point is None:
binary_source_paths = await Get(
Paths, PathGlobs,
field_set.sources.path_globs(FilesNotFoundBehavior.error))
if len(binary_source_paths.files) != 1:
raise InvalidFieldException(
"No `entry_point` was set for the target "
f"{repr(field_set.address)}, so it must have exactly one source, but it has "
f"{len(binary_source_paths.files)}")
entry_point_path = binary_source_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
os.path.relpath(entry_point_path, source_root.path))
transitive_targets = await Get(TransitiveTargets,
Addresses([field_set.address]))
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([field_set.address],
internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
additional_args=field_set.generate_additional_args(
python_binary_defaults),
internal_only=True,
# Note that the entry point file is not in the Pex itself, but on the
# PEX_PATH. This works fine!
entry_point=entry_point,
),
)
requirements, sources, runner_pex = await MultiGet(requirements_request,
sources_request,
runner_pex_request)
merged_digest = await Get(
Digest,
MergeDigests([
requirements.digest, sources.source_files.snapshot.digest,
runner_pex.digest
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
args = pex_env.create_argv(in_chroot(runner_pex.name),
python=runner_pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=runner_pex.python is not None),
"PEX_PATH":
in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest,
protoc: Protoc,
grpc_python_plugin: GrpcPythonPlugin,
python_protobuf_subsystem: PythonProtobufSubsystem,
python_protobuf_mypy_plugin: PythonProtobufMypyPlugin,
pex_environment: PexEnvironment,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
protoc_gen_mypy_script = "protoc-gen-mypy"
protoc_gen_mypy_grpc_script = "protoc-gen-mypy_grpc"
mypy_pex = None
mypy_request = PexRequest(
output_filename="mypy_protobuf.pex",
internal_only=True,
requirements=python_protobuf_mypy_plugin.pex_requirements(),
interpreter_constraints=python_protobuf_mypy_plugin.
interpreter_constraints,
)
if python_protobuf_subsystem.mypy_plugin:
mypy_pex = await Get(
VenvPex,
VenvPexRequest(bin_names=[protoc_gen_mypy_script],
pex_request=mypy_request),
)
if request.protocol_target.get(ProtobufGrpcToggleField).value:
mypy_info = await Get(PexResolveInfo, VenvPex, mypy_pex)
# In order to generate stubs for gRPC code, we need mypy-protobuf 2.0 or above.
if any(dist_info.project_name == "mypy-protobuf"
and dist_info.version.major >= 2
for dist_info in mypy_info):
# TODO: Use `pex_path` once VenvPex stores a Pex field.
mypy_pex = await Get(
VenvPex,
VenvPexRequest(
bin_names=[
protoc_gen_mypy_script, protoc_gen_mypy_grpc_script
],
pex_request=mypy_request,
),
)
downloaded_grpc_plugin = (await Get(
DownloadedExternalTool,
ExternalToolRequest,
grpc_python_plugin.get_request(Platform.current),
) if request.protocol_target.get(ProtobufGrpcToggleField).value else None)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
if mypy_pex:
unmerged_digests.append(mypy_pex.digest)
if downloaded_grpc_plugin:
unmerged_digests.append(downloaded_grpc_plugin.digest)
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--python_out", output_dir]
if mypy_pex:
argv.extend([
f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}",
"--mypy_out",
output_dir,
])
if downloaded_grpc_plugin:
argv.extend([
f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}",
"--grpc_out", output_dir
])
if mypy_pex and protoc_gen_mypy_grpc_script in mypy_pex.bin:
argv.extend([
f"--plugin=protoc-gen-mypy_grpc={mypy_pex.bin[protoc_gen_mypy_grpc_script].argv0}",
"--mypy_grpc_out",
output_dir,
])
argv.extend(target_sources_stripped.snapshot.files)
result = await Get(
ProcessResult,
Process(
argv,
input_digest=input_digest,
description=
f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
append_only_caches=pex_environment.in_sandbox(
working_directory=None).append_only_caches,
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_source's source root.
source_root_request = SourceRootRequest.for_target(
request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def create_venv_pex(request: VenvPexRequest, bash: BashBinary,
pex_environment: PexEnvironment) -> VenvPex:
# VenvPex is motivated by improving performance of Python tools by eliminating traditional PEX
# file startup overhead.
#
# To achieve the minimal overhead (on the order of 1ms) we discard:
# 1. Using Pex default mode:
# Although this does reduce initial tool execution overhead, it still leaves a minimum
# O(100ms) of overhead per subsequent tool invocation. Fundamentally, Pex still needs to
# execute its `sys.path` isolation bootstrap code in this case.
# 2. Using the Pex `venv` tool:
# The idea here would be to create a tool venv as a Process output and then use the tool
# venv as an input digest for all tool invocations. This was tried and netted ~500ms of
# overhead over raw venv use.
#
# Instead we use Pex's `--venv` mode. In this mode you can run the Pex file and it will create a
# venv on the fly in the PEX_ROOT as needed. Since the PEX_ROOT is a named_cache, we avoid the
# digest materialization overhead present in 2 above. Since the venv is naturally isolated we
# avoid the `sys.path` isolation overhead of Pex itself present in 1 above.
#
# This does leave O(50ms) of overhead though for the PEX bootstrap code to detect an already
# created venv in the PEX_ROOT and re-exec into it. To eliminate this overhead we execute the
# `pex` venv script in the PEX_ROOT directly. This is not robust on its own though, since the
# named caches store might be pruned at any time. To guard against that case we introduce a shim
# bash script that checks to see if the `pex` venv script exists in the PEX_ROOT and re-creates
# the PEX_ROOT venv if not. Using the shim script to run Python tools gets us down to the ~1ms
# of overhead we currently enjoy.
pex_request = request.pex_request
seeded_venv_request = dataclasses.replace(
pex_request,
additional_args=pex_request.additional_args + (
"--venv",
"--seed",
"verbose",
pex_environment.venv_site_packages_copies_option(
use_copies=request.site_packages_copies),
),
)
venv_pex_result = await Get(BuildPexResult, PexRequest,
seeded_venv_request)
# Pex verbose --seed mode outputs the absolute path of the PEX executable as well as the
# absolute path of the PEX_ROOT. In the --venv case this is the `pex` script in the venv root
# directory.
seed_info = json.loads(venv_pex_result.result.stdout.decode())
abs_pex_root = PurePath(seed_info["pex_root"])
abs_pex_path = PurePath(seed_info["pex"])
venv_rel_dir = abs_pex_path.relative_to(abs_pex_root).parent
venv_script_writer = VenvScriptWriter.create(
pex_environment=pex_environment,
pex=venv_pex_result.create_pex(),
venv_rel_dir=venv_rel_dir)
pex = venv_script_writer.exe(bash)
python = venv_script_writer.python(bash)
scripts = {
bin_name: venv_script_writer.bin(bash, bin_name)
for bin_name in request.bin_names
}
scripts_digest = await Get(
Digest,
CreateDigest((
pex.content,
python.content,
*(venv_script.content for venv_script in scripts.values()),
)),
)
input_digest = await Get(
Digest, MergeDigests((venv_script_writer.pex.digest, scripts_digest)))
return VenvPex(
digest=input_digest,
pex_filename=venv_pex_result.pex_filename,
pex=pex.script,
python=python.script,
bin=FrozenDict((bin_name, venv_script.script)
for bin_name, venv_script in scripts.items()),
venv_rel_dir=venv_rel_dir.as_posix(),
)
async def create_pex_binary_run_request(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment,
python_setup: PythonSetup,
) -> RunRequest:
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in local_dists_get.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest(
[field_set.address],
output_filename=f"{field_set.address.target_name}.pex",
internal_only=True,
include_source_files=False,
# Note that the file for first-party entry points is not in the PEX itself. In that
# case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
main=entry_point.val or field_set.script.value,
resolve_and_lockfile=field_set.resolve.resolve_and_lockfile(
python_setup),
additional_args=(
*field_set.generate_additional_args(pex_binary_defaults),
# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
# below, it's important for any app that re-executes itself that these environment
# variables are not stripped.
"--no-strip-pex-env",
),
),
)
pex_get = Get(Pex, PexRequest, requirements_pex_request)
sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
pex, sources = await MultiGet(pex_get, sources_get)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[field_set.address],
internal_only=True,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
sources=sources,
),
)
merged_digest = await Get(
Digest,
MergeDigests([
pex.digest,
local_dists.pex.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
"PEX_PATH":
in_chroot(local_dists.pex.name),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pex_cli_process(
request: PexCliProcess,
pex_binary: PexBinary,
pex_env: PexEnvironment,
python_native_code: PythonNativeCode,
global_options: GlobalOptions,
) -> Process:
tmpdir = ".tmp"
gets: List[Get] = [
Get(DownloadedExternalTool, ExternalToolRequest,
pex_binary.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(f"{tmpdir}/.reserve")])),
]
cert_args = []
# The certs file will typically not be in the repo, so we can't digest it via a PathGlobs.
# Instead we manually create a FileContent for it.
if global_options.options.ca_certs_path:
ca_certs_content = Path(
global_options.options.ca_certs_path).read_bytes()
chrooted_ca_certs_path = os.path.basename(
global_options.options.ca_certs_path)
gets.append(
Get(
Digest,
CreateDigest((FileContent(chrooted_ca_certs_path,
ca_certs_content), )),
))
cert_args = ["--cert", chrooted_ca_certs_path]
downloaded_pex_bin, *digests_to_merge = await MultiGet(gets)
digests_to_merge.append(downloaded_pex_bin.digest)
if request.additional_input_digest:
digests_to_merge.append(request.additional_input_digest)
input_digest = await Get(Digest, MergeDigests(digests_to_merge))
pex_root_path = ".cache/pex_root"
argv = pex_env.create_argv(
downloaded_pex_bin.exe,
*request.argv,
*cert_args,
"--pex-root",
pex_root_path,
python=request.python,
)
env = {
# Ensure Pex and its subprocesses create temporary files in the the process execution
# sandbox. It may make sense to do this generally for Processes, but in the short term we
# have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to
# perform. Making the TMPDIR local to the sandbox allows control via
# --local-execution-root-dir for the local case and should work well with remote cases where
# a remoting implementation has to allow for processes producing large binaries in a
# sandbox to support reasonable workloads.
"TMPDIR": tmpdir,
**pex_env.environment_dict(python_configured=request.python is not None),
**python_native_code.environment_dict,
**(request.extra_env or {}),
}
return Process(
argv,
description=request.description,
input_digest=input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches={"pex_root": pex_root_path},
level=request.level,
)