def create(cls, pex_environment: PexEnvironment, pex: Pex,
venv_rel_dir: PurePath) -> VenvScriptWriter:
# N.B.: We don't know the working directory that will be used in any given
# invocation of the venv scripts; so we deal with working_directory inside the scripts
# themselves by absolutifying all relevant paths at runtime.
complete_pex_env = pex_environment.in_sandbox(working_directory=None)
venv_dir = complete_pex_env.pex_root / venv_rel_dir
return cls(complete_pex_env=complete_pex_env,
pex=pex,
venv_dir=venv_dir)
def create(cls, pex_environment: PexEnvironment, pex: Pex,
venv_rel_dir: PurePath) -> VenvScriptWriter:
# N.B.: We don't know the working directory that will be used in any given
# invocation of the venv scripts; so we deal with working_directory once in an
# `adjust_relative_paths` function inside the script to save rule authors from having to do
# CWD offset math in every rule for all the relative paths their process depends on.
complete_pex_env = pex_environment.in_sandbox(working_directory=None)
venv_dir = complete_pex_env.pex_root / venv_rel_dir
return cls(complete_pex_env=complete_pex_env,
pex=pex,
venv_dir=venv_dir)
async def setup_venv_pex_process(request: VenvPexProcess,
pex_environment: PexEnvironment) -> Process:
venv_pex = request.venv_pex
pex_bin = (os.path.relpath(venv_pex.pex.argv0, request.working_directory)
if request.working_directory else venv_pex.pex.argv0)
argv = (pex_bin, *request.argv)
input_digest = (await Get(
Digest, MergeDigests((venv_pex.digest, request.input_digest)))
if request.input_digest else venv_pex.digest)
return Process(
argv=argv,
description=request.description,
level=request.level,
input_digest=input_digest,
working_directory=request.working_directory,
env=request.extra_env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches=pex_environment.in_sandbox(
working_directory=request.working_directory).append_only_caches,
timeout_seconds=request.timeout_seconds,
execution_slot_variable=request.execution_slot_variable,
cache_scope=request.cache_scope,
)
async def setup_pex_cli_process(
request: PexCliProcess,
pex_pex: PexPEX,
pex_env: PexEnvironment,
python_native_code: PythonNativeCode,
global_options: GlobalOptions,
pex_runtime_env: PexRuntimeEnvironment,
) -> Process:
tmpdir = ".tmp"
gets: List[Get] = [Get(Digest, CreateDigest([Directory(tmpdir)]))]
# The certs file will typically not be in the repo, so we can't digest it via a PathGlobs.
# Instead we manually create a FileContent for it.
cert_args = []
if global_options.ca_certs_path:
ca_certs_content = Path(global_options.ca_certs_path).read_bytes()
chrooted_ca_certs_path = os.path.basename(global_options.ca_certs_path)
gets.append(
Get(
Digest,
CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content),)),
)
)
cert_args = ["--cert", chrooted_ca_certs_path]
digests_to_merge = [pex_pex.digest]
digests_to_merge.extend(await MultiGet(gets))
if request.additional_input_digest:
digests_to_merge.append(request.additional_input_digest)
input_digest = await Get(Digest, MergeDigests(digests_to_merge))
global_args = [
# Ensure Pex and its subprocesses create temporary files in the the process execution
# sandbox. It may make sense to do this generally for Processes, but in the short term we
# have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to
# perform. Making the TMPDIR local to the sandbox allows control via
# --local-execution-root-dir for the local case and should work well with remote cases where
# a remoting implementation has to allow for processes producing large binaries in a
# sandbox to support reasonable workloads. Communicating TMPDIR via --tmpdir instead of via
# environment variable allows Pex to absolutize the path ensuring subprocesses that change
# CWD can find the TMPDIR.
"--tmpdir",
tmpdir,
]
if request.concurrency_available > 0:
global_args.extend(["--jobs", "{pants_concurrency}"])
if pex_runtime_env.verbosity > 0:
global_args.append(f"-{'v' * pex_runtime_env.verbosity}")
resolve_args = (
[*cert_args, "--python-path", create_path_env_var(pex_env.interpreter_search_paths)]
if request.set_resolve_args
else []
)
args = [
*global_args,
*request.subcommand,
*resolve_args,
# NB: This comes at the end because it may use `--` passthrough args, # which must come at
# the end.
*request.extra_args,
]
complete_pex_env = pex_env.in_sandbox(working_directory=None)
normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args, python=request.python)
env = {
**complete_pex_env.environment_dict(python_configured=request.python is not None),
**python_native_code.environment_dict,
**(request.extra_env or {}),
# If a subcommand is used, we need to use the `pex3` console script.
**({"PEX_SCRIPT": "pex3"} if request.subcommand else {}),
}
return Process(
normalized_argv,
description=request.description,
input_digest=input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches=complete_pex_env.append_only_caches,
level=request.level,
concurrency_available=request.concurrency_available,
cache_scope=request.cache_scope,
)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest,
protoc: Protoc,
grpc_python_plugin: GrpcPythonPlugin,
python_protobuf_subsystem: PythonProtobufSubsystem,
python_protobuf_mypy_plugin: PythonProtobufMypyPlugin,
pex_environment: PexEnvironment,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
protoc_gen_mypy_script = "protoc-gen-mypy"
protoc_gen_mypy_grpc_script = "protoc-gen-mypy_grpc"
mypy_pex = None
mypy_request = PexRequest(
output_filename="mypy_protobuf.pex",
internal_only=True,
requirements=python_protobuf_mypy_plugin.pex_requirements(),
interpreter_constraints=python_protobuf_mypy_plugin.
interpreter_constraints,
)
if python_protobuf_subsystem.mypy_plugin:
mypy_pex = await Get(
VenvPex,
VenvPexRequest(bin_names=[protoc_gen_mypy_script],
pex_request=mypy_request),
)
if request.protocol_target.get(ProtobufGrpcToggleField).value:
mypy_info = await Get(PexResolveInfo, VenvPex, mypy_pex)
# In order to generate stubs for gRPC code, we need mypy-protobuf 2.0 or above.
if any(dist_info.project_name == "mypy-protobuf"
and dist_info.version.major >= 2
for dist_info in mypy_info):
# TODO: Use `pex_path` once VenvPex stores a Pex field.
mypy_pex = await Get(
VenvPex,
VenvPexRequest(
bin_names=[
protoc_gen_mypy_script, protoc_gen_mypy_grpc_script
],
pex_request=mypy_request,
),
)
downloaded_grpc_plugin = (await Get(
DownloadedExternalTool,
ExternalToolRequest,
grpc_python_plugin.get_request(Platform.current),
) if request.protocol_target.get(ProtobufGrpcToggleField).value else None)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
if mypy_pex:
unmerged_digests.append(mypy_pex.digest)
if downloaded_grpc_plugin:
unmerged_digests.append(downloaded_grpc_plugin.digest)
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--python_out", output_dir]
if mypy_pex:
argv.extend([
f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}",
"--mypy_out",
output_dir,
])
if downloaded_grpc_plugin:
argv.extend([
f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}",
"--grpc_out", output_dir
])
if mypy_pex and protoc_gen_mypy_grpc_script in mypy_pex.bin:
argv.extend([
f"--plugin=protoc-gen-mypy_grpc={mypy_pex.bin[protoc_gen_mypy_grpc_script].argv0}",
"--mypy_grpc_out",
output_dir,
])
argv.extend(target_sources_stripped.snapshot.files)
result = await Get(
ProcessResult,
Process(
argv,
input_digest=input_digest,
description=
f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
append_only_caches=pex_environment.in_sandbox(
working_directory=None).append_only_caches,
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_source's source root.
source_root_request = SourceRootRequest.for_target(
request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)