def test_path_globs_to_digest_entries(rule_runner: RuleRunner) -> None:
setup_fs_test_tar(rule_runner)
def get_entries(globs: Iterable[str]) -> Set[Union[FileEntry, Directory]]:
return set(rule_runner.request(DigestEntries, [PathGlobs(globs)]))
assert get_entries(["4.txt", "a/4.txt.ln"]) == {
FileEntry(
"4.txt",
FileDigest("ab929fcd5594037960792ea0b98caf5fdaf6b60645e4ef248c28db74260f393e", 5),
),
FileEntry(
"a/4.txt.ln",
FileDigest("ab929fcd5594037960792ea0b98caf5fdaf6b60645e4ef248c28db74260f393e", 5),
),
}
assert get_entries(["c.ln/../3.txt"]) == {
FileEntry(
"c.ln/../3.txt",
FileDigest("f6936912184481f5edd4c304ce27c5a1a827804fc7f329f43d273b8621870776", 6),
)
}
# Directories are empty.
assert get_entries(["a/b"]) == {Directory("a/b")}
assert get_entries(["c.ln"]) == {Directory("c.ln")}
def test_create_empty_directory(rule_runner: RuleRunner) -> None:
res = rule_runner.request(Snapshot, [CreateDigest([Directory("a/")])])
assert res.dirs == ("a",)
assert not res.files
assert res.digest != EMPTY_DIGEST
res = rule_runner.request(
Snapshot, [CreateDigest([Directory("x/y/z"), Directory("m"), Directory("m/n")])]
)
assert res.dirs == ("m", "m/n", "x", "x/y", "x/y/z")
assert not res.files
assert res.digest != EMPTY_DIGEST
def test_digest_entries_handles_empty_directory(rule_runner: RuleRunner) -> None:
digest = rule_runner.request(
Digest, [CreateDigest([Directory("a/b"), FileContent("a/foo.txt", b"four\n")])]
)
entries = rule_runner.request(DigestEntries, [digest])
assert entries == DigestEntries(
[
Directory("a/b"),
FileEntry(
"a/foo.txt",
FileDigest("ab929fcd5594037960792ea0b98caf5fdaf6b60645e4ef248c28db74260f393e", 5),
),
]
)
def test_output_digest(rule_runner: RuleRunner, working_directory) -> None:
# Test that the output files are relative to the working directory, both in how
# they're specified, and their paths in the output_digest.
input_digest = (rule_runner.request(
Digest,
[CreateDigest([Directory(working_directory)])],
) if working_directory else EMPTY_DIGEST)
process = Process(
input_digest=input_digest,
argv=("/bin/bash", "-c", "echo -n 'European Burmese' > roland"),
description="echo roland",
output_files=("roland", ),
working_directory=working_directory,
)
result = rule_runner.request(ProcessResult, [process])
assert result.output_digest == Digest(
fingerprint=
"63949aa823baf765eff07b946050d76ec0033144c785a94d3ebd82baa931cd16",
serialized_bytes_length=80,
)
digest_contents = rule_runner.request(DigestContents,
[result.output_digest])
assert digest_contents == DigestContents(
[FileContent("roland", b"European Burmese", False)])
async def create_archive(request: CreateArchive) -> Digest:
if request.format == ArchiveFormat.ZIP:
zip_binary = await Get(ZipBinary, _ZipBinaryRequest())
argv = zip_binary.create_archive_argv(request)
env = {}
input_digest = request.snapshot.digest
else:
tar_binary = await Get(TarBinary, _TarBinaryRequest())
argv = tar_binary.create_archive_argv(request)
# `tar` expects to find a couple binaries like `gzip` and `xz` by looking on the PATH.
env = {"PATH": os.pathsep.join(SEARCH_PATHS)}
# `tar` requires that the output filename's parent directory exists.
output_dir_digest = await Get(
Digest,
CreateDigest([Directory(os.path.dirname(request.output_filename))
]))
input_digest = await Get(
Digest, MergeDigests([output_dir_digest, request.snapshot.digest]))
result = await Get(
ProcessResult,
Process(
argv=argv,
env=env,
input_digest=input_digest,
description=f"Create {request.output_filename}",
level=LogLevel.DEBUG,
output_files=(request.output_filename, ),
),
)
return result.output_digest
async def fetch_helm_artifacts(
request: FetchHelmArfifactsRequest) -> FetchedHelmArtifacts:
download_prefix = "__downloads"
empty_download_digest = await Get(
Digest, CreateDigest([Directory(download_prefix)]))
artifacts = await MultiGet(
Get(ResolvedHelmArtifact, HelmArtifact,
HelmArtifact.from_field_set(field_set))
for field_set in request.field_sets)
def create_fetch_process(artifact: ResolvedHelmArtifact) -> HelmProcess:
return HelmProcess(
argv=[
"pull",
artifact.name,
"--repo",
artifact.location_url,
"--version",
artifact.version,
"--destination",
download_prefix,
"--untar",
],
input_digest=empty_download_digest,
description=
f"Pulling Helm Chart '{artifact.name}' with version {artifact.version}",
output_directories=(download_prefix, ),
)
download_results = await MultiGet(
Get(
ProcessResult,
HelmProcess,
create_fetch_process(artifact),
) for artifact in artifacts)
stripped_artifact_digests = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, download_prefix))
for result in download_results)
# Avoid capturing the tarball that has been downloaded by Helm during the pull.
artifact_snapshots = await MultiGet(
Get(
Snapshot,
DigestSubset(digest, PathGlobs([os.path.join(artifact.name, "**")
])))
for artifact, digest in zip(artifacts, stripped_artifact_digests))
fetched_artifacts = [
FetchedHelmArtifact(artifact=artifact, snapshot=snapshot)
for artifact, snapshot in zip(artifacts, artifact_snapshots)
]
logger.debug(
f"Fetched {pluralize(len(fetched_artifacts), 'Helm artifact')} corresponding with:\n"
f"{bullet_list([artifact.address.spec for artifact in fetched_artifacts], max_elements=10)}"
)
return FetchedHelmArtifacts(fetched_artifacts)
async def maybe_extract_archive(digest: Digest) -> ExtractedArchive:
"""If digest contains a single archive file, extract it, otherwise return the input digest."""
extract_archive_dir = "__extract_archive_dir"
snapshot, output_dir_digest = await MultiGet(
Get(Snapshot, Digest, digest),
Get(Digest, CreateDigest([Directory(extract_archive_dir)])),
)
if len(snapshot.files) != 1:
return ExtractedArchive(digest)
archive_path = snapshot.files[0]
is_zip = archive_path.endswith(".zip")
is_tar = archive_path.endswith(
(".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz"))
is_gz = not is_tar and archive_path.endswith(".gz")
if not is_zip and not is_tar and not is_gz:
return ExtractedArchive(digest)
merge_digest_get = Get(Digest, MergeDigests((digest, output_dir_digest)))
if is_zip:
input_digest, unzip_binary = await MultiGet(
merge_digest_get,
Get(UnzipBinary, _UnzipBinaryRequest()),
)
argv = unzip_binary.extract_archive_argv(archive_path,
extract_archive_dir)
env = {}
elif is_tar:
input_digest, tar_binary = await MultiGet(
merge_digest_get,
Get(TarBinary, _TarBinaryRequest()),
)
argv = tar_binary.extract_archive_argv(archive_path,
extract_archive_dir)
# `tar` expects to find a couple binaries like `gzip` and `xz` by looking on the PATH.
env = {"PATH": os.pathsep.join(SEARCH_PATHS)}
else:
input_digest, gunzip = await MultiGet(
merge_digest_get,
Get(Gunzip, _GunzipRequest()),
)
argv = gunzip.extract_archive_argv(archive_path, extract_archive_dir)
env = {}
result = await Get(
ProcessResult,
Process(
argv=argv,
env=env,
input_digest=input_digest,
description=f"Extract {archive_path}",
level=LogLevel.DEBUG,
output_directories=(extract_archive_dir, ),
),
)
digest = await Get(Digest,
RemovePrefix(result.output_digest, extract_archive_dir))
return ExtractedArchive(digest)
async def build_processors(
bash: BashBinary, jdk_setup: JdkSetup) -> JavaParserCompiledClassfiles:
dest_dir = "classfiles"
materialized_classpath, source_digest = await MultiGet(
Get(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__toolcp",
artifact_requirements=(java_parser_artifact_requirements(), ),
),
),
Get(
Digest,
CreateDigest([
FileContent(
path=_LAUNCHER_BASENAME,
content=_load_javaparser_launcher_source(),
),
Directory(dest_dir),
]),
),
)
merged_digest = await Get(
Digest,
MergeDigests((
materialized_classpath.digest,
jdk_setup.digest,
source_digest,
)),
)
process_result = await Get(
ProcessResult,
Process(
argv=[
*jdk_setup.args(bash,
[f"{jdk_setup.java_home}/lib/tools.jar"]),
"com.sun.tools.javac.Main",
"-cp",
materialized_classpath.classpath_arg(),
"-d",
dest_dir,
_LAUNCHER_BASENAME,
],
input_digest=merged_digest,
use_nailgun=jdk_setup.digest,
output_directories=(dest_dir, ),
description=
f"Compile {_LAUNCHER_BASENAME} import processors with javac",
level=LogLevel.DEBUG,
),
)
stripped_classfiles_digest = await Get(
Digest, RemovePrefix(process_result.output_digest, dest_dir))
return JavaParserCompiledClassfiles(digest=stripped_classfiles_digest)
async def build_processors(jdk: InternalJdk) -> JavaParserCompiledClassfiles:
dest_dir = "classfiles"
parser_lockfile_request = await Get(GenerateJvmLockfileFromTool,
JavaParserToolLockfileSentinel())
materialized_classpath, source_digest = await MultiGet(
Get(
ToolClasspath,
ToolClasspathRequest(prefix="__toolcp",
lockfile=parser_lockfile_request),
),
Get(
Digest,
CreateDigest([
FileContent(
path=_LAUNCHER_BASENAME,
content=_load_javaparser_launcher_source(),
),
Directory(dest_dir),
]),
),
)
merged_digest = await Get(
Digest,
MergeDigests((
materialized_classpath.digest,
source_digest,
)),
)
process_result = await Get(
ProcessResult,
JvmProcess(
jdk=jdk,
classpath_entries=[f"{jdk.java_home}/lib/tools.jar"],
argv=[
"com.sun.tools.javac.Main",
"-cp",
":".join(materialized_classpath.classpath_entries()),
"-d",
dest_dir,
_LAUNCHER_BASENAME,
],
input_digest=merged_digest,
output_directories=(dest_dir, ),
description=
f"Compile {_LAUNCHER_BASENAME} import processors with javac",
level=LogLevel.DEBUG,
# NB: We do not use nailgun for this process, since it is launched exactly once.
use_nailgun=False,
),
)
stripped_classfiles_digest = await Get(
Digest, RemovePrefix(process_result.output_digest, dest_dir))
return JavaParserCompiledClassfiles(digest=stripped_classfiles_digest)
async def flake8_lint_partition(
partition: Flake8Partition, flake8: Flake8,
first_party_plugins: Flake8FirstPartyPlugins) -> LintResult:
flake8_pex_get = Get(
VenvPex,
PexRequest,
flake8.to_pex_request(
interpreter_constraints=partition.interpreter_constraints,
extra_requirements=first_party_plugins.requirement_strings,
),
)
config_files_get = Get(ConfigFiles, ConfigFilesRequest,
flake8.config_request)
source_files_get = Get(
SourceFiles,
SourceFilesRequest(field_set.source
for field_set in partition.field_sets))
# Ensure that the empty report dir exists.
report_directory_digest_get = Get(Digest,
CreateDigest([Directory(REPORT_DIR)]))
flake8_pex, config_files, report_directory, source_files = await MultiGet(
flake8_pex_get, config_files_get, report_directory_digest_get,
source_files_get)
input_digest = await Get(
Digest,
MergeDigests((
source_files.snapshot.digest,
first_party_plugins.sources_digest,
config_files.snapshot.digest,
report_directory,
)),
)
result = await Get(
FallibleProcessResult,
VenvPexProcess(
flake8_pex,
argv=generate_argv(source_files, flake8),
input_digest=input_digest,
output_directories=(REPORT_DIR, ),
extra_env={"PEX_EXTRA_SYS_PATH": first_party_plugins.PREFIX},
concurrency_available=len(partition.field_sets),
description=
f"Run Flake8 on {pluralize(len(partition.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR))
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def setup_helm(helm_subsytem: HelmSubsystem) -> HelmBinary:
cache_dir = "__cache"
config_dir = "__config"
data_dir = "__data"
downloaded_binary, cache_digest, config_digest, data_digest = await MultiGet(
Get(DownloadedExternalTool, ExternalToolRequest,
helm_subsytem.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(cache_dir)])),
Get(Digest, CreateDigest([Directory(config_dir)])),
Get(Digest, CreateDigest([Directory(data_dir)])),
)
tool_relpath = "__helm"
immutable_input_digests = {tool_relpath: downloaded_binary.digest}
helm_path = os.path.join(tool_relpath, downloaded_binary.exe)
helm_env = _build_helm_env(cache_dir, config_dir, data_dir)
# TODO Install Global Helm plugins
# TODO Configure Helm classic repositories
cache_subset_digest, config_subset_digest, data_subset_digest = await MultiGet(
Get(Digest, RemovePrefix(cache_digest, cache_dir)),
Get(Digest, RemovePrefix(config_digest, config_dir)),
Get(Digest, RemovePrefix(data_digest, data_dir)),
)
setup_immutable_digests = {
**immutable_input_digests,
cache_dir: cache_subset_digest,
config_dir: config_subset_digest,
data_dir: data_subset_digest,
}
local_env = await Get(Environment, EnvironmentRequest(["HOME", "PATH"]))
return HelmBinary(
path=helm_path,
helm_env=helm_env,
local_env=local_env,
immutable_input_digests=setup_immutable_digests,
)
async def bandit_lint_partition(partition: BanditPartition,
bandit: Bandit) -> LintResult:
bandit_pex_get = Get(
VenvPex,
PexRequest(
output_filename="bandit.pex",
internal_only=True,
requirements=bandit.pex_requirements(),
interpreter_constraints=partition.interpreter_constraints,
main=bandit.main,
),
)
config_files_get = Get(ConfigFiles, ConfigFilesRequest,
bandit.config_request)
source_files_get = Get(
SourceFiles,
SourceFilesRequest(field_set.source
for field_set in partition.field_sets))
# Ensure that the empty report dir exists.
report_directory_digest_get = Get(Digest,
CreateDigest([Directory(REPORT_DIR)]))
bandit_pex, config_files, report_directory, source_files = await MultiGet(
bandit_pex_get, config_files_get, report_directory_digest_get,
source_files_get)
input_digest = await Get(
Digest,
MergeDigests((source_files.snapshot.digest,
config_files.snapshot.digest, report_directory)),
)
result = await Get(
FallibleProcessResult,
VenvPexProcess(
bandit_pex,
argv=generate_argv(source_files, bandit),
input_digest=input_digest,
description=
f"Run Bandit on {pluralize(len(partition.field_sets), 'file')}.",
output_directories=(REPORT_DIR, ),
level=LogLevel.DEBUG,
),
)
report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR))
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def maybe_extract_archive(digest: Digest, tar_binary: TarBinary,
unzip_binary: UnzipBinary) -> ExtractedArchive:
"""If digest contains a single archive file, extract it, otherwise return the input digest."""
output_dir = "__output"
snapshot, output_dir_digest = await MultiGet(
Get(Snapshot, Digest, digest),
Get(Digest, CreateDigest([Directory(output_dir)])),
)
if len(snapshot.files) != 1:
return ExtractedArchive(digest)
input_digest = await Get(Digest, MergeDigests((digest, output_dir_digest)))
fp = snapshot.files[0]
if fp.endswith(".zip"):
argv = unzip_binary.extract_archive_argv(archive_path=fp,
output_dir=output_dir)
env = {}
elif fp.endswith(
(".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz")):
argv = tar_binary.extract_archive_argv(archive_path=fp,
output_dir=output_dir)
# `tar` expects to find a couple binaries like `gzip` and `xz` by looking on the PATH.
env = {"PATH": os.pathsep.join(SEARCH_PATHS)}
else:
return ExtractedArchive(digest)
result = await Get(
ProcessResult,
Process(
argv=argv,
env=env,
input_digest=input_digest,
description=f"Extract {fp}",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
strip_output_dir = await Get(
Digest, RemovePrefix(result.output_digest, output_dir))
return ExtractedArchive(strip_output_dir)
async def run_helm_package(field_set: HelmPackageFieldSet) -> BuiltPackage:
result_dir = "__out"
chart, result_digest = await MultiGet(
Get(HelmChart, HelmChartRequest(field_set)),
Get(Digest, CreateDigest([Directory(result_dir)])),
)
input_digest = await Get(
Digest, MergeDigests([chart.snapshot.digest, result_digest]))
process_output_file = os.path.join(result_dir,
_helm_artifact_filename(chart.metadata))
process_result = await Get(
ProcessResult,
HelmProcess(
argv=["package", chart.path, "-d", result_dir],
input_digest=input_digest,
output_files=(process_output_file, ),
description=f"Packaging Helm chart: {field_set.address.spec_path}",
),
)
stripped_output_digest = await Get(
Digest, RemovePrefix(process_result.output_digest, result_dir))
final_snapshot = await Get(
Snapshot,
AddPrefix(stripped_output_digest,
field_set.output_path.value_or_default(file_ending=None)),
)
return BuiltPackage(
final_snapshot.digest,
artifacts=tuple(
BuiltPackageArtifact(
file, extra_log_lines=(f"Built Helm chart artifact: {file}", ))
for file in final_snapshot.files),
)
async def setup_pex_cli_process(
request: PexCliProcess,
pex_pex: PexPEX,
pex_env: PexEnvironment,
python_native_code: PythonNativeCode,
global_options: GlobalOptions,
pex_runtime_env: PexRuntimeEnvironment,
) -> Process:
tmpdir = ".tmp"
gets: List[Get] = [Get(Digest, CreateDigest([Directory(tmpdir)]))]
# The certs file will typically not be in the repo, so we can't digest it via a PathGlobs.
# Instead we manually create a FileContent for it.
cert_args = []
if global_options.ca_certs_path:
ca_certs_content = Path(global_options.ca_certs_path).read_bytes()
chrooted_ca_certs_path = os.path.basename(global_options.ca_certs_path)
gets.append(
Get(
Digest,
CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content),)),
)
)
cert_args = ["--cert", chrooted_ca_certs_path]
digests_to_merge = [pex_pex.digest]
digests_to_merge.extend(await MultiGet(gets))
if request.additional_input_digest:
digests_to_merge.append(request.additional_input_digest)
input_digest = await Get(Digest, MergeDigests(digests_to_merge))
global_args = [
# Ensure Pex and its subprocesses create temporary files in the the process execution
# sandbox. It may make sense to do this generally for Processes, but in the short term we
# have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to
# perform. Making the TMPDIR local to the sandbox allows control via
# --local-execution-root-dir for the local case and should work well with remote cases where
# a remoting implementation has to allow for processes producing large binaries in a
# sandbox to support reasonable workloads. Communicating TMPDIR via --tmpdir instead of via
# environment variable allows Pex to absolutize the path ensuring subprocesses that change
# CWD can find the TMPDIR.
"--tmpdir",
tmpdir,
]
if request.concurrency_available > 0:
global_args.extend(["--jobs", "{pants_concurrency}"])
if pex_runtime_env.verbosity > 0:
global_args.append(f"-{'v' * pex_runtime_env.verbosity}")
resolve_args = (
[*cert_args, "--python-path", create_path_env_var(pex_env.interpreter_search_paths)]
if request.set_resolve_args
else []
)
args = [
*global_args,
*request.subcommand,
*resolve_args,
# NB: This comes at the end because it may use `--` passthrough args, # which must come at
# the end.
*request.extra_args,
]
complete_pex_env = pex_env.in_sandbox(working_directory=None)
normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args, python=request.python)
env = {
**complete_pex_env.environment_dict(python_configured=request.python is not None),
**python_native_code.environment_dict,
**(request.extra_env or {}),
# If a subcommand is used, we need to use the `pex3` console script.
**({"PEX_SCRIPT": "pex3"} if request.subcommand else {}),
}
return Process(
normalized_argv,
description=request.description,
input_digest=input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches=complete_pex_env.append_only_caches,
level=request.level,
concurrency_available=request.concurrency_available,
cache_scope=request.cache_scope,
)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest,
protoc: Protoc,
grpc_python_plugin: GrpcPythonPlugin,
python_protobuf_subsystem: PythonProtobufSubsystem,
python_protobuf_mypy_plugin: PythonProtobufMypyPlugin,
pex_environment: PexEnvironment,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
protoc_gen_mypy_script = "protoc-gen-mypy"
protoc_gen_mypy_grpc_script = "protoc-gen-mypy_grpc"
mypy_pex = None
mypy_request = PexRequest(
output_filename="mypy_protobuf.pex",
internal_only=True,
requirements=python_protobuf_mypy_plugin.pex_requirements(),
interpreter_constraints=python_protobuf_mypy_plugin.
interpreter_constraints,
)
if python_protobuf_subsystem.mypy_plugin:
mypy_pex = await Get(
VenvPex,
VenvPexRequest(bin_names=[protoc_gen_mypy_script],
pex_request=mypy_request),
)
if request.protocol_target.get(ProtobufGrpcToggleField).value:
mypy_info = await Get(PexResolveInfo, VenvPex, mypy_pex)
# In order to generate stubs for gRPC code, we need mypy-protobuf 2.0 or above.
if any(dist_info.project_name == "mypy-protobuf"
and dist_info.version.major >= 2
for dist_info in mypy_info):
# TODO: Use `pex_path` once VenvPex stores a Pex field.
mypy_pex = await Get(
VenvPex,
VenvPexRequest(
bin_names=[
protoc_gen_mypy_script, protoc_gen_mypy_grpc_script
],
pex_request=mypy_request,
),
)
downloaded_grpc_plugin = (await Get(
DownloadedExternalTool,
ExternalToolRequest,
grpc_python_plugin.get_request(Platform.current),
) if request.protocol_target.get(ProtobufGrpcToggleField).value else None)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
if mypy_pex:
unmerged_digests.append(mypy_pex.digest)
if downloaded_grpc_plugin:
unmerged_digests.append(downloaded_grpc_plugin.digest)
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--python_out", output_dir]
if mypy_pex:
argv.extend([
f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}",
"--mypy_out",
output_dir,
])
if downloaded_grpc_plugin:
argv.extend([
f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}",
"--grpc_out", output_dir
])
if mypy_pex and protoc_gen_mypy_grpc_script in mypy_pex.bin:
argv.extend([
f"--plugin=protoc-gen-mypy_grpc={mypy_pex.bin[protoc_gen_mypy_grpc_script].argv0}",
"--mypy_grpc_out",
output_dir,
])
argv.extend(target_sources_stripped.snapshot.files)
result = await Get(
ProcessResult,
Process(
argv,
input_digest=input_digest,
description=
f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
append_only_caches=pex_environment.in_sandbox(
working_directory=None).append_only_caches,
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_source's source root.
source_root_request = SourceRootRequest.for_target(
request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def prepare_shell_command_process(request: ShellCommandProcessRequest,
shell_setup: ShellSetup,
bash: BashBinary) -> Process:
shell_command = request.target
interactive = shell_command.has_field(ShellCommandRunWorkdirField)
if interactive:
working_directory = shell_command[
ShellCommandRunWorkdirField].value or ""
else:
working_directory = shell_command.address.spec_path
command = shell_command[ShellCommandCommandField].value
timeout = shell_command.get(ShellCommandTimeoutField).value
tools = shell_command.get(ShellCommandToolsField,
default_raw_value=()).value
outputs = shell_command.get(ShellCommandOutputsField).value or ()
extra_env_vars = shell_command.get(
ShellCommandExtraEnvVarsField).value or ()
if not command:
raise ValueError(
f"Missing `command` line in `{shell_command.alias}` target {shell_command.address}."
)
if interactive:
command_env = {
"CHROOT": "{chroot}",
}
else:
if not tools:
raise ValueError(
f"Must provide any `tools` used by the `{shell_command.alias}` {shell_command.address}."
)
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_path = shell_setup.executable_search_path(env)
tool_requests = [
BinaryPathRequest(
binary_name=tool,
search_path=search_path,
) for tool in {*tools, *["mkdir", "ln"]}
if tool not in BASH_BUILTIN_COMMANDS
]
tool_paths = await MultiGet(
Get(BinaryPaths, BinaryPathRequest, request)
for request in tool_requests)
command_env = {
"TOOLS":
" ".join(
_shell_tool_safe_env_name(tool.binary_name)
for tool in tool_requests),
}
for binary, tool_request in zip(tool_paths, tool_requests):
if binary.first_path:
command_env[_shell_tool_safe_env_name(
tool_request.binary_name)] = binary.first_path.path
else:
raise BinaryNotFoundError.from_request(
tool_request,
rationale=
f"execute `{shell_command.alias}` {shell_command.address}",
)
extra_env = await Get(Environment, EnvironmentRequest(extra_env_vars))
command_env.update(extra_env)
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([shell_command.address]),
)
sources, pkgs_per_target = await MultiGet(
Get(
SourceFiles,
SourceFilesRequest(
sources_fields=[
tgt.get(SourcesField)
for tgt in transitive_targets.dependencies
],
for_sources_types=(SourcesField, FileSourceField),
enable_codegen=True,
),
),
Get(
FieldSetsPerTarget,
FieldSetsPerTargetRequest(PackageFieldSet,
transitive_targets.dependencies),
),
)
packages = await MultiGet(
Get(BuiltPackage, PackageFieldSet, field_set)
for field_set in pkgs_per_target.field_sets)
if interactive or not working_directory or working_directory in sources.snapshot.dirs:
work_dir = EMPTY_DIGEST
else:
work_dir = await Get(Digest,
CreateDigest([Directory(working_directory)]))
input_digest = await Get(
Digest,
MergeDigests([
sources.snapshot.digest, work_dir,
*(pkg.digest for pkg in packages)
]))
output_files = [f for f in outputs if not f.endswith("/")]
output_directories = [d for d in outputs if d.endswith("/")]
if interactive:
relpath = os.path.relpath(
working_directory or ".",
start="/" if os.path.isabs(working_directory) else ".")
boot_script = f"cd {shlex.quote(relpath)}; " if relpath != "." else ""
else:
# Setup bin_relpath dir with symlinks to all requested tools, so that we can use PATH, force
# symlinks to avoid issues with repeat runs using the __run.sh script in the sandbox.
bin_relpath = ".bin"
boot_script = ";".join(
dedent(f"""\
$mkdir -p {bin_relpath}
for tool in $TOOLS; do $ln -sf ${{!tool}} {bin_relpath}; done
export PATH="$PWD/{bin_relpath}"
""").split("\n"))
return Process(
argv=(bash.path, "-c", boot_script + command),
description=f"Running {shell_command.alias} {shell_command.address}",
env=command_env,
input_digest=input_digest,
output_directories=output_directories,
output_files=output_files,
timeout_seconds=timeout,
working_directory=working_directory,
)
async def setup_pex_cli_process(
request: PexCliProcess,
pex_binary: PexBinary,
pex_env: PexEnvironment,
python_native_code: PythonNativeCode,
global_options: GlobalOptions,
pex_runtime_env: PexRuntimeEnvironment,
) -> Process:
tmpdir = ".tmp"
gets: List[Get] = [
Get(DownloadedExternalTool, ExternalToolRequest, pex_binary.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(tmpdir)])),
]
cert_args = []
# The certs file will typically not be in the repo, so we can't digest it via a PathGlobs.
# Instead we manually create a FileContent for it.
if global_options.options.ca_certs_path:
ca_certs_content = Path(global_options.options.ca_certs_path).read_bytes()
chrooted_ca_certs_path = os.path.basename(global_options.options.ca_certs_path)
gets.append(
Get(
Digest,
CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content),)),
)
)
cert_args = ["--cert", chrooted_ca_certs_path]
downloaded_pex_bin, *digests_to_merge = await MultiGet(gets)
digests_to_merge.append(downloaded_pex_bin.digest)
if request.additional_input_digest:
digests_to_merge.append(request.additional_input_digest)
input_digest = await Get(Digest, MergeDigests(digests_to_merge))
pex_root_path = ".cache/pex_root"
argv = [
downloaded_pex_bin.exe,
*cert_args,
"--python-path",
create_path_env_var(pex_env.interpreter_search_paths),
"--pex-root",
pex_root_path,
# Ensure Pex and its subprocesses create temporary files in the the process execution
# sandbox. It may make sense to do this generally for Processes, but in the short term we
# have known use cases where /tmp is too small to hold large wheel downloads Pex is asked to
# perform. Making the TMPDIR local to the sandbox allows control via
# --local-execution-root-dir for the local case and should work well with remote cases where
# a remoting implementation has to allow for processes producing large binaries in a
# sandbox to support reasonable workloads. Communicating TMPDIR via --tmpdir instead of via
# environment variable allows Pex to absolutize the path ensuring subprocesses that change
# CWD can find the TMPDIR.
"--tmpdir",
tmpdir,
]
if pex_runtime_env.verbosity > 0:
argv.append(f"-{'v' * pex_runtime_env.verbosity}")
# NB: This comes at the end of the argv because the request may use `--` passthrough args,
# which must come at the end.
argv.extend(request.argv)
normalized_argv = pex_env.create_argv(*argv, python=request.python)
env = {
**pex_env.environment_dict(python_configured=request.python is not None),
**python_native_code.environment_dict,
**(request.extra_env or {}),
}
return Process(
normalized_argv,
description=request.description,
input_digest=input_digest,
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches={"pex_root": pex_root_path},
level=request.level,
cache_scope=request.cache_scope,
)
async def setup_pytest_for_target(
request: TestSetupRequest,
pytest: PyTest,
test_subsystem: TestSubsystem,
python_setup: PythonSetup,
coverage_config: CoverageConfig,
coverage_subsystem: CoverageSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
complete_env: CompleteEnvironment,
) -> TestSetup:
transitive_targets, plugin_setups = await MultiGet(
Get(TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address])),
Get(AllPytestPluginSetups,
AllPytestPluginSetupsRequest(request.field_set.address)),
)
all_targets = transitive_targets.closure
interpreter_constraints = InterpreterConstraints.create_from_targets(
all_targets, python_setup)
requirements_pex_get = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
[request.field_set.address],
internal_only=True,
resolve_and_lockfile=request.field_set.resolve.
resolve_and_lockfile(python_setup),
),
)
pytest_pex_get = Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=pytest.pex_requirements(),
interpreter_constraints=interpreter_constraints,
internal_only=True,
),
)
# Ensure that the empty extra output dir exists.
extra_output_directory_digest_get = Get(
Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)]))
prepared_sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets, include_files=True))
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
field_set_source_files_get = Get(
SourceFiles, SourceFilesRequest([request.field_set.sources]))
(
pytest_pex,
requirements_pex,
prepared_sources,
field_set_source_files,
extra_output_directory_digest,
) = await MultiGet(
pytest_pex_get,
requirements_pex_get,
prepared_sources_get,
field_set_source_files_get,
extra_output_directory_digest_get,
)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[request.field_set.address],
interpreter_constraints=interpreter_constraints,
sources=prepared_sources,
),
)
pytest_runner_pex_get = Get(
VenvPex,
PexRequest(
output_filename="pytest_runner.pex",
interpreter_constraints=interpreter_constraints,
main=pytest.main,
internal_only=True,
pex_path=[pytest_pex, requirements_pex, local_dists.pex],
),
)
config_files_get = Get(
ConfigFiles,
ConfigFilesRequest,
pytest.config_request(field_set_source_files.snapshot.dirs),
)
pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get,
config_files_get)
input_digest = await Get(
Digest,
MergeDigests((
coverage_config.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
config_files.snapshot.digest,
extra_output_directory_digest,
*(plugin_setup.digest for plugin_setup in plugin_setups),
)),
)
add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"]
output_files = []
results_file_name = None
if pytest.options.junit_xml_dir and not request.is_debug:
results_file_name = f"{request.field_set.address.path_safe_spec}.xml"
add_opts.extend((f"--junitxml={results_file_name}", "-o",
f"junit_family={pytest.options.junit_family}"))
output_files.append(results_file_name)
coverage_args = []
if test_subsystem.use_coverage and not request.is_debug:
pytest.validate_pytest_cov_included()
output_files.append(".coverage")
if coverage_subsystem.filter:
cov_args = [f"--cov={morf}" for morf in coverage_subsystem.filter]
else:
# N.B.: Passing `--cov=` or `--cov=.` to communicate "record coverage for all sources"
# fails in certain contexts as detailed in:
# https://github.com/pantsbuild/pants/issues/12390
# Instead we focus coverage on just the directories containing python source files
# materialized to the Process chroot.
cov_args = [
f"--cov={source_root}"
for source_root in prepared_sources.source_roots
]
coverage_args = [
"--cov-report=", # Turn off output.
f"--cov-config={coverage_config.path}",
*cov_args,
]
extra_env = {
"PYTEST_ADDOPTS": " ".join(add_opts),
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
**test_extra_env.env,
# NOTE: `complete_env` intentionally after `test_extra_env` to allow overriding within
# `python_tests`
**complete_env.get_subset(request.field_set.extra_env_vars.value or ()),
}
# Cache test runs only if they are successful, or not at all if `--test-force`.
cache_scope = (ProcessCacheScope.PER_SESSION
if test_subsystem.force else ProcessCacheScope.SUCCESSFUL)
process = await Get(
Process,
VenvPexProcess(
pytest_runner_pex,
argv=(*pytest.options.args, *coverage_args,
*field_set_source_files.files),
extra_env=extra_env,
input_digest=input_digest,
output_directories=(_EXTRA_OUTPUT_DIR, ),
output_files=output_files,
timeout_seconds=request.field_set.timeout.
calculate_from_global_options(pytest),
execution_slot_variable=pytest.options.execution_slot_var,
description=f"Run Pytest for {request.field_set.address}",
level=LogLevel.DEBUG,
cache_scope=cache_scope,
),
)
return TestSetup(process, results_file_name=results_file_name)
async def generate_go_from_protobuf(
request: GenerateGoFromProtobufRequest,
protoc: Protoc,
go_protoc_plugin: _SetupGoProtocPlugin,
) -> GeneratedSources:
output_dir = "_generated_files"
protoc_relpath = "__protoc"
protoc_go_plugin_relpath = "__protoc_gen_go"
downloaded_protoc_binary, empty_output_dir, transitive_targets = await MultiGet(
Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(output_dir)])),
Get(TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address])),
)
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_sources_stripped, target_sources_stripped = await MultiGet(
Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
),
Get(StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]
])),
)
input_digest = await Get(
Digest,
MergeDigests([all_sources_stripped.snapshot.digest, empty_output_dir]))
maybe_grpc_plugin_args = []
if request.protocol_target.get(ProtobufGrpcToggleField).value:
maybe_grpc_plugin_args = [
f"--go-grpc_out={output_dir}",
"--go-grpc_opt=paths=source_relative",
]
result = await Get(
ProcessResult,
Process(
argv=[
os.path.join(protoc_relpath, downloaded_protoc_binary.exe),
f"--plugin=go={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go')}",
f"--plugin=go-grpc={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go-grpc')}",
f"--go_out={output_dir}",
"--go_opt=paths=source_relative",
*maybe_grpc_plugin_args,
*target_sources_stripped.snapshot.files,
],
# Note: Necessary or else --plugin option needs absolute path.
env={"PATH": protoc_go_plugin_relpath},
input_digest=input_digest,
immutable_input_digests={
protoc_relpath: downloaded_protoc_binary.digest,
protoc_go_plugin_relpath: go_protoc_plugin.digest,
},
description=
f"Generating Go sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def pylint_lint_partition(
partition: PylintPartition, pylint: Pylint,
first_party_plugins: PylintFirstPartyPlugins) -> LintResult:
requirements_pex_get = Get(
Pex,
RequirementsPexRequest(
(t.address for t in partition.root_targets),
# NB: These constraints must be identical to the other PEXes. Otherwise, we risk using
# a different version for the requirements than the other two PEXes, which can result
# in a PEX runtime error about missing dependencies.
hardcoded_interpreter_constraints=partition.
interpreter_constraints,
internal_only=True,
),
)
pylint_pex_get = Get(
Pex,
PexRequest,
pylint.to_pex_request(
interpreter_constraints=partition.interpreter_constraints,
extra_requirements=first_party_plugins.requirement_strings,
),
)
prepare_python_sources_get = Get(
PythonSourceFiles, PythonSourceFilesRequest(partition.closure))
field_set_sources_get = Get(
SourceFiles,
SourceFilesRequest(t[PythonSourceField]
for t in partition.root_targets))
# Ensure that the empty report dir exists.
report_directory_digest_get = Get(Digest,
CreateDigest([Directory(REPORT_DIR)]))
(
pylint_pex,
requirements_pex,
prepared_python_sources,
field_set_sources,
report_directory,
) = await MultiGet(
pylint_pex_get,
requirements_pex_get,
prepare_python_sources_get,
field_set_sources_get,
report_directory_digest_get,
)
pylint_runner_pex, config_files = await MultiGet(
Get(
VenvPex,
VenvPexRequest(
PexRequest(
output_filename="pylint_runner.pex",
interpreter_constraints=partition.interpreter_constraints,
main=pylint.main,
internal_only=True,
pex_path=[pylint_pex, requirements_pex],
),
# TODO(John Sirois): Remove this (change to the default of symlinks) when we can
# upgrade to a version of Pylint with https://github.com/PyCQA/pylint/issues/1470
# resolved.
site_packages_copies=True,
),
),
Get(ConfigFiles, ConfigFilesRequest,
pylint.config_request(field_set_sources.snapshot.dirs)),
)
pythonpath = list(prepared_python_sources.source_roots)
if first_party_plugins:
pythonpath.append(first_party_plugins.PREFIX)
input_digest = await Get(
Digest,
MergeDigests((
config_files.snapshot.digest,
first_party_plugins.sources_digest,
prepared_python_sources.source_files.snapshot.digest,
report_directory,
)),
)
result = await Get(
FallibleProcessResult,
VenvPexProcess(
pylint_runner_pex,
argv=generate_argv(field_set_sources, pylint),
input_digest=input_digest,
output_directories=(REPORT_DIR, ),
extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},
concurrency_available=len(partition.root_targets),
description=
f"Run Pylint on {pluralize(len(partition.root_targets), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR))
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def setup_scala_parser_classfiles(jdk: InternalJdk) -> ScalaParserCompiledClassfiles:
dest_dir = "classfiles"
parser_source_content = pkgutil.get_data(
"pants.backend.scala.dependency_inference", "ScalaParser.scala"
)
if not parser_source_content:
raise AssertionError("Unable to find ScalaParser.scala resource.")
parser_source = FileContent("ScalaParser.scala", parser_source_content)
tool_classpath, parser_classpath, source_digest = await MultiGet(
Get(
ToolClasspath,
ToolClasspathRequest(
prefix="__toolcp",
artifact_requirements=ArtifactRequirements.from_coordinates(
[
Coordinate(
group="org.scala-lang",
artifact="scala-compiler",
version=PARSER_SCALA_VERSION,
),
Coordinate(
group="org.scala-lang",
artifact="scala-library",
version=PARSER_SCALA_VERSION,
),
Coordinate(
group="org.scala-lang",
artifact="scala-reflect",
version=PARSER_SCALA_VERSION,
),
]
),
),
),
Get(
ToolClasspath,
ToolClasspathRequest(
prefix="__parsercp", artifact_requirements=SCALA_PARSER_ARTIFACT_REQUIREMENTS
),
),
Get(Digest, CreateDigest([parser_source, Directory(dest_dir)])),
)
merged_digest = await Get(
Digest,
MergeDigests(
(
tool_classpath.digest,
parser_classpath.digest,
source_digest,
)
),
)
process_result = await Get(
ProcessResult,
JvmProcess(
jdk=jdk,
classpath_entries=tool_classpath.classpath_entries(),
argv=[
"scala.tools.nsc.Main",
"-bootclasspath",
":".join(tool_classpath.classpath_entries()),
"-classpath",
":".join(parser_classpath.classpath_entries()),
"-d",
dest_dir,
parser_source.path,
],
input_digest=merged_digest,
output_directories=(dest_dir,),
description="Compile Scala parser for dependency inference with scalac",
level=LogLevel.DEBUG,
# NB: We do not use nailgun for this process, since it is launched exactly once.
use_nailgun=False,
),
)
stripped_classfiles_digest = await Get(
Digest, RemovePrefix(process_result.output_digest, dest_dir)
)
return ScalaParserCompiledClassfiles(digest=stripped_classfiles_digest)
async def setup_pytest_for_target(
request: TestSetupRequest,
pytest: PyTest,
test_subsystem: TestSubsystem,
python_setup: PythonSetup,
coverage_config: CoverageConfig,
coverage_subsystem: CoverageSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
) -> TestSetup:
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address]))
all_targets = transitive_targets.closure
interpreter_constraints = PexInterpreterConstraints.create_from_targets(
all_targets, python_setup)
requirements_pex_get = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([request.field_set.address],
internal_only=True),
)
pytest_pex_get = Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
interpreter_constraints=interpreter_constraints,
internal_only=True,
),
)
extra_output_directory_digest_get = Get(
Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)]))
prepared_sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets, include_files=True))
build_package_dependencies_get = Get(
BuiltPackageDependencies,
BuildPackageDependenciesRequest(
request.field_set.runtime_package_dependencies),
)
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
field_set_source_files_get = Get(
SourceFiles, SourceFilesRequest([request.field_set.sources]))
(
pytest_pex,
requirements_pex,
prepared_sources,
field_set_source_files,
built_package_dependencies,
extra_output_directory_digest,
) = await MultiGet(
pytest_pex_get,
requirements_pex_get,
prepared_sources_get,
field_set_source_files_get,
build_package_dependencies_get,
extra_output_directory_digest_get,
)
pytest_runner_pex_get = Get(
VenvPex,
PexRequest(
output_filename="pytest_runner.pex",
interpreter_constraints=interpreter_constraints,
main=ConsoleScript("pytest"),
internal_only=True,
pex_path=[pytest_pex, requirements_pex],
),
)
config_files_get = Get(
ConfigFiles,
ConfigFilesRequest,
pytest.config_request(field_set_source_files.snapshot.dirs),
)
pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get,
config_files_get)
input_digest = await Get(
Digest,
MergeDigests((
coverage_config.digest,
prepared_sources.source_files.snapshot.digest,
config_files.snapshot.digest,
extra_output_directory_digest,
*(pkg.digest for pkg in built_package_dependencies),
)),
)
add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"]
output_files = []
results_file_name = None
if pytest.options.junit_xml_dir and not request.is_debug:
results_file_name = f"{request.field_set.address.path_safe_spec}.xml"
add_opts.extend((f"--junitxml={results_file_name}", "-o",
f"junit_family={pytest.options.junit_family}"))
output_files.append(results_file_name)
coverage_args = []
if test_subsystem.use_coverage and not request.is_debug:
output_files.append(".coverage")
cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else (
".", )
coverage_args = [
"--cov-report=", # Turn off output.
f"--cov-config={coverage_config.path}",
*itertools.chain.from_iterable(["--cov", cov_path]
for cov_path in cov_paths),
]
extra_env = {
"PYTEST_ADDOPTS": " ".join(add_opts),
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
**test_extra_env.env,
}
# Cache test runs only if they are successful, or not at all if `--test-force`.
cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL
process = await Get(
Process,
VenvPexProcess(
pytest_runner_pex,
argv=(*pytest.options.args, *coverage_args,
*field_set_source_files.files),
extra_env=extra_env,
input_digest=input_digest,
output_directories=(_EXTRA_OUTPUT_DIR, ),
output_files=output_files,
timeout_seconds=request.field_set.timeout.
calculate_from_global_options(pytest),
execution_slot_variable=pytest.options.execution_slot_var,
description=f"Run Pytest for {request.field_set.address}",
level=LogLevel.DEBUG,
cache_scope=cache_scope,
),
)
return TestSetup(process, results_file_name=results_file_name)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest,
protoc: Protoc,
grpc_python_plugin: GrpcPythonPlugin,
python_protobuf_subsystem: PythonProtobufSubsystem,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
# TODO(#10917): Use TransitiveTargets instead of TransitiveTargetsLite.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequestLite([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSources, ),
),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSources]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
# To run the MyPy Protobuf plugin, we first install it with Pex, then extract the wheels and
# point Protoc to the extracted wheels with its `--plugin` argument.
extracted_mypy_wheels = None
if python_protobuf_subsystem.mypy_plugin:
mypy_pex = await Get(
Pex,
PexRequest(
output_filename="mypy_protobuf.pex",
internal_only=True,
requirements=PexRequirements(
[python_protobuf_subsystem.mypy_plugin_version]),
# This is solely to ensure that we use an appropriate interpreter when resolving
# the distribution. We don't actually run the distribution directly with Python,
# as we extract out its binary.
interpreter_constraints=PexInterpreterConstraints(
["CPython>=3.5"]),
),
)
extracted_mypy_wheels = await Get(ExtractedPexDistributions, Pex,
mypy_pex)
downloaded_grpc_plugin = (await Get(
DownloadedExternalTool,
ExternalToolRequest,
grpc_python_plugin.get_request(Platform.current),
) if request.protocol_target.get(ProtobufGrcpToggle).value else None)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
if extracted_mypy_wheels:
unmerged_digests.append(extracted_mypy_wheels.digest)
if downloaded_grpc_plugin:
unmerged_digests.append(downloaded_grpc_plugin.digest)
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--python_out", output_dir]
if extracted_mypy_wheels:
mypy_plugin_path = next(
p for p in extracted_mypy_wheels.wheel_directory_paths
if p.startswith(".deps/mypy_protobuf-"))
argv.extend([
f"--plugin=protoc-gen-mypy={mypy_plugin_path}/bin/protoc-gen-mypy",
"--mypy_out",
output_dir,
])
if downloaded_grpc_plugin:
argv.extend([
f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}",
"--grpc_out", output_dir
])
argv.extend(target_sources_stripped.snapshot.files)
env = {}
if extracted_mypy_wheels:
env["PYTHONPATH"] = ":".join(
extracted_mypy_wheels.wheel_directory_paths)
result = await Get(
ProcessResult,
Process(
argv,
env=env,
input_digest=input_digest,
description=
f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_library's source root.
source_root_request = SourceRootRequest.for_target(
request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def run_shell_command(
request: GenerateFilesFromShellCommandRequest,
shell_setup: ShellSetup,
bash: BashBinary,
) -> GeneratedSources:
shell_command = request.protocol_target
working_directory = shell_command.address.spec_path
command = shell_command[ShellCommandCommandField].value
tools = shell_command[ShellCommandToolsField].value
outputs = shell_command[ShellCommandOutputsField].value or ()
if not command:
raise ValueError(
f"Missing `command` line in `shell_command` target {shell_command.address}."
)
if not tools:
raise ValueError(
f"Must provide any `tools` used by the `shell_command` {shell_command.address}."
)
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_path = shell_setup.executable_search_path(env)
tool_requests = [
BinaryPathRequest(
binary_name=tool,
search_path=search_path,
) for tool in {*tools, *["mkdir", "ln"]}
if tool not in BASH_BUILTIN_COMMANDS
]
tool_paths = await MultiGet(
Get(BinaryPaths, BinaryPathRequest, request)
for request in tool_requests)
command_env = {
"TOOLS":
" ".join(shlex.quote(tool.binary_name) for tool in tool_requests),
}
for binary, tool_request in zip(tool_paths, tool_requests):
if binary.first_path:
command_env[tool_request.binary_name] = binary.first_path.path
else:
raise BinaryNotFoundError.from_request(
tool_request,
rationale=
f"execute experimental_shell_command {shell_command.address}",
)
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([shell_command.address]),
)
sources = await Get(
SourceFiles,
SourceFilesRequest(
sources_fields=[
tgt.get(Sources) for tgt in transitive_targets.dependencies
],
for_sources_types=(
Sources,
FilesSources,
),
enable_codegen=True,
),
)
output_files = [f for f in outputs if not f.endswith("/")]
output_directories = [d for d in outputs if d.endswith("/")]
if working_directory in sources.snapshot.dirs:
input_digest = sources.snapshot.digest
else:
work_dir = await Get(Digest,
CreateDigest([Directory(working_directory)]))
input_digest = await Get(
Digest, MergeDigests([sources.snapshot.digest, work_dir]))
# Setup bin_relpath dir with symlinks to all requested tools, so that we can use PATH.
bin_relpath = ".bin"
setup_tool_symlinks_script = ";".join(
dedent(f"""\
$mkdir -p {bin_relpath}
for tool in $TOOLS; do $ln -s ${{!tool}} {bin_relpath}/; done
export PATH="$PWD/{bin_relpath}"
""").split("\n"))
result = await Get(
ProcessResult,
Process(
argv=(bash.path, "-c", setup_tool_symlinks_script + command),
description=
f"Running experimental_shell_command {shell_command.address}",
env=command_env,
input_digest=input_digest,
output_directories=output_directories,
output_files=output_files,
working_directory=working_directory,
),
)
if shell_command[ShellCommandLogOutputField].value:
if result.stdout:
logger.info(result.stdout.decode())
if result.stderr:
logger.warning(result.stderr.decode())
output = await Get(Snapshot,
AddPrefix(result.output_digest, working_directory))
return GeneratedSources(output)
async def compile_java_source(
bash: BashBinary,
javac: JavacSubsystem,
zip_binary: ZipBinary,
request: CompileJavaSourceRequest,
) -> FallibleClasspathEntry:
# Request the component's direct dependency classpath, and additionally any prerequisite.
optional_prereq_request = [
*((request.prerequisite, ) if request.prerequisite else ())
]
fallibles = await MultiGet(
Get(FallibleClasspathEntries,
ClasspathEntryRequests(optional_prereq_request)),
Get(FallibleClasspathEntries, ClasspathDependenciesRequest(request)),
)
direct_dependency_classpath_entries = FallibleClasspathEntries(
itertools.chain(*fallibles)).if_all_succeeded()
if direct_dependency_classpath_entries is None:
return FallibleClasspathEntry(
description=str(request.component),
result=CompileResult.DEPENDENCY_FAILED,
output=None,
exit_code=1,
)
# Capture just the `ClasspathEntry` objects that are listed as `export` types by source analysis
deps_to_classpath_entries = dict(
zip(request.component.dependencies, direct_dependency_classpath_entries
or ()))
# Re-request inferred dependencies to get a list of export dependency addresses
inferred_dependencies = await MultiGet(
Get(
JavaInferredDependencies,
JavaInferredDependenciesAndExportsRequest(tgt[JavaSourceField]),
) for tgt in request.component.members
if JavaFieldSet.is_applicable(tgt))
flat_exports = {
export
for i in inferred_dependencies for export in i.exports
}
export_classpath_entries = [
classpath_entry for coarsened_target, classpath_entry in
deps_to_classpath_entries.items()
if any(m.address in flat_exports for m in coarsened_target.members)
]
# Then collect the component's sources.
component_members_with_sources = tuple(t for t in request.component.members
if t.has_field(SourcesField))
component_members_and_source_files = zip(
component_members_with_sources,
await MultiGet(
Get(
SourceFiles,
SourceFilesRequest(
(t.get(SourcesField), ),
for_sources_types=(JavaSourceField, ),
enable_codegen=True,
),
) for t in component_members_with_sources),
)
component_members_and_java_source_files = [
(target, sources)
for target, sources in component_members_and_source_files
if sources.snapshot.digest != EMPTY_DIGEST
]
if not component_members_and_java_source_files:
# Is a generator, and so exports all of its direct deps.
exported_digest = await Get(
Digest,
MergeDigests(cpe.digest
for cpe in direct_dependency_classpath_entries))
classpath_entry = ClasspathEntry.merge(
exported_digest, direct_dependency_classpath_entries)
return FallibleClasspathEntry(
description=str(request.component),
result=CompileResult.SUCCEEDED,
output=classpath_entry,
exit_code=0,
)
dest_dir = "classfiles"
dest_dir_digest, jdk = await MultiGet(
Get(
Digest,
CreateDigest([Directory(dest_dir)]),
),
Get(JdkEnvironment, JdkRequest,
JdkRequest.from_target(request.component)),
)
merged_digest = await Get(
Digest,
MergeDigests((
dest_dir_digest,
*(sources.snapshot.digest
for _, sources in component_members_and_java_source_files),
)),
)
usercp = "__cp"
user_classpath = Classpath(direct_dependency_classpath_entries,
request.resolve)
classpath_arg = ":".join(
user_classpath.root_immutable_inputs_args(prefix=usercp))
immutable_input_digests = dict(
user_classpath.root_immutable_inputs(prefix=usercp))
# Compile.
compile_result = await Get(
FallibleProcessResult,
JvmProcess(
jdk=jdk,
classpath_entries=[f"{jdk.java_home}/lib/tools.jar"],
argv=[
"com.sun.tools.javac.Main",
*(("-cp", classpath_arg) if classpath_arg else ()),
*javac.args,
"-d",
dest_dir,
*sorted(
chain.from_iterable(
sources.snapshot.files for _, sources in
component_members_and_java_source_files)),
],
input_digest=merged_digest,
extra_immutable_input_digests=immutable_input_digests,
output_directories=(dest_dir, ),
description=f"Compile {request.component} with javac",
level=LogLevel.DEBUG,
),
)
if compile_result.exit_code != 0:
return FallibleClasspathEntry.from_fallible_process_result(
str(request.component),
compile_result,
None,
)
# Jar.
# NB: We jar up the outputs in a separate process because the nailgun runner cannot support
# invoking via a `bash` wrapper (since the trailing portion of the command is executed by
# the nailgun server). We might be able to resolve this in the future via a Javac wrapper shim.
output_snapshot = await Get(Snapshot, Digest, compile_result.output_digest)
output_file = compute_output_jar_filename(request.component)
output_files: tuple[str, ...] = (output_file, )
if output_snapshot.files:
jar_result = await Get(
ProcessResult,
Process(
argv=[
bash.path,
"-c",
" ".join([
"cd", dest_dir, ";", zip_binary.path, "-r",
f"../{output_file}", "."
]),
],
input_digest=compile_result.output_digest,
output_files=output_files,
description=f"Capture outputs of {request.component} for javac",
level=LogLevel.TRACE,
),
)
jar_output_digest = jar_result.output_digest
else:
# If there was no output, then do not create a jar file. This may occur, for example, when compiling
# a `package-info.java` in a single partition.
output_files = ()
jar_output_digest = EMPTY_DIGEST
output_classpath = ClasspathEntry(jar_output_digest, output_files,
direct_dependency_classpath_entries)
if export_classpath_entries:
merged_export_digest = await Get(
Digest,
MergeDigests((output_classpath.digest,
*(i.digest for i in export_classpath_entries))),
)
merged_classpath = ClasspathEntry.merge(
merged_export_digest,
(output_classpath, *export_classpath_entries))
output_classpath = merged_classpath
return FallibleClasspathEntry.from_fallible_process_result(
str(request.component),
compile_result,
output_classpath,
)
async def setup_full_package_build_request(
request: _SetupGoProtobufPackageBuildRequest,
protoc: Protoc,
go_protoc_plugin: _SetupGoProtocPlugin,
package_mapping: ImportPathToPackages,
go_protobuf_mapping: GoProtobufImportPathMapping,
analyzer: PackageAnalyzerSetup,
) -> FallibleBuildGoPackageRequest:
output_dir = "_generated_files"
protoc_relpath = "__protoc"
protoc_go_plugin_relpath = "__protoc_gen_go"
transitive_targets, downloaded_protoc_binary, empty_output_dir = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)),
Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(output_dir)])),
)
all_sources = await Get(
SourceFiles,
SourceFilesRequest(
sources_fields=(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSourceField, ),
enable_codegen=True,
),
)
source_roots, input_digest = await MultiGet(
Get(SourceRootsResult, SourceRootsRequest,
SourceRootsRequest.for_files(all_sources.files)),
Get(Digest,
MergeDigests([all_sources.snapshot.digest, empty_output_dir])),
)
source_root_paths = sorted(
{sr.path
for sr in source_roots.path_to_root.values()})
pkg_sources = await MultiGet(
Get(SourcesPaths, SourcesPathsRequest(tgt[ProtobufSourceField]))
for tgt in transitive_targets.roots)
pkg_files = sorted({f for ps in pkg_sources for f in ps.files})
maybe_grpc_plugin_args = []
if any(
tgt.get(ProtobufGrpcToggleField).value
for tgt in transitive_targets.roots):
maybe_grpc_plugin_args = [
f"--go-grpc_out={output_dir}",
"--go-grpc_opt=paths=source_relative",
]
gen_result = await Get(
FallibleProcessResult,
Process(
argv=[
os.path.join(protoc_relpath, downloaded_protoc_binary.exe),
f"--plugin=go={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go')}",
f"--plugin=go-grpc={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go-grpc')}",
f"--go_out={output_dir}",
"--go_opt=paths=source_relative",
*(f"--proto_path={source_root}"
for source_root in source_root_paths),
*maybe_grpc_plugin_args,
*pkg_files,
],
# Note: Necessary or else --plugin option needs absolute path.
env={"PATH": protoc_go_plugin_relpath},
input_digest=input_digest,
immutable_input_digests={
protoc_relpath: downloaded_protoc_binary.digest,
protoc_go_plugin_relpath: go_protoc_plugin.digest,
},
description=f"Generating Go sources from {request.import_path}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
if gen_result.exit_code != 0:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=gen_result.exit_code,
stderr=gen_result.stderr.decode(),
)
# Ensure that the generated files are in a single package directory.
gen_sources = await Get(Snapshot, Digest, gen_result.output_digest)
files_by_dir = group_by_dir(gen_sources.files)
if len(files_by_dir) != 1:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=1,
stderr=
("Expected Go files generated from Protobuf sources to be output to a single directory.\n"
f"- import path: {request.import_path}\n"
f"- protobuf files: {', '.join(pkg_files)}"),
)
gen_dir = list(files_by_dir.keys())[0]
# Analyze the generated sources.
input_digest = await Get(
Digest, MergeDigests([gen_sources.digest, analyzer.digest]))
result = await Get(
FallibleProcessResult,
Process(
(analyzer.path, gen_dir),
input_digest=input_digest,
description=
f"Determine metadata for generated Go package for {request.import_path}",
level=LogLevel.DEBUG,
env={"CGO_ENABLED": "0"},
),
)
# Parse the metadata from the analysis.
fallible_analysis = FallibleFirstPartyPkgAnalysis.from_process_result(
result,
dir_path=gen_dir,
import_path=request.import_path,
minimum_go_version="",
description_of_source=
f"Go package generated from protobuf targets `{', '.join(str(addr) for addr in request.addresses)}`",
)
if not fallible_analysis.analysis:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=fallible_analysis.exit_code,
stderr=fallible_analysis.stderr,
)
analysis = fallible_analysis.analysis
# Obtain build requests for third-party dependencies.
# TODO: Consider how to merge this code with existing dependency inference code.
dep_build_request_addrs: list[Address] = []
for dep_import_path in (*analysis.imports, *analysis.test_imports,
*analysis.xtest_imports):
# Infer dependencies on other Go packages.
candidate_addresses = package_mapping.mapping.get(dep_import_path)
if candidate_addresses:
# TODO: Use explicit dependencies to disambiguate? This should never happen with Go backend though.
if len(candidate_addresses) > 1:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=result.exit_code,
stderr=
(f"Multiple addresses match import of `{dep_import_path}`.\n"
f"addresses: {', '.join(str(a) for a in candidate_addresses)}"
),
)
dep_build_request_addrs.extend(candidate_addresses)
# Infer dependencies on other generated Go sources.
go_protobuf_candidate_addresses = go_protobuf_mapping.mapping.get(
dep_import_path)
if go_protobuf_candidate_addresses:
dep_build_request_addrs.extend(go_protobuf_candidate_addresses)
dep_build_requests = await MultiGet(
Get(BuildGoPackageRequest, BuildGoPackageTargetRequest(addr))
for addr in dep_build_request_addrs)
return FallibleBuildGoPackageRequest(
request=BuildGoPackageRequest(
import_path=request.import_path,
digest=gen_sources.digest,
dir_path=analysis.dir_path,
go_file_names=analysis.go_files,
s_file_names=analysis.s_files,
direct_dependencies=dep_build_requests,
minimum_go_version=analysis.minimum_go_version,
),
import_path=request.import_path,
)
def test_pex_working_directory(rule_runner: RuleRunner,
pex_type: type[Pex | VenvPex]) -> None:
named_caches_dir = rule_runner.request(GlobalOptions, []).named_caches_dir
sources = rule_runner.request(
Digest,
[
CreateDigest((FileContent(
path="main.py",
content=textwrap.dedent("""
import os
cwd = os.getcwd()
print(f"CWD: {cwd}")
for path, dirs, _ in os.walk(cwd):
for name in dirs:
print(f"DIR: {os.path.relpath(os.path.join(path, name), cwd)}")
""").encode(),
), )),
],
)
pex_data = create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
main=EntryPoint("main"),
sources=sources,
interpreter_constraints=InterpreterConstraints(["CPython>=3.6"]),
)
pex_process_type = PexProcess if isinstance(pex_data.pex,
Pex) else VenvPexProcess
dirpath = "foo/bar/baz"
runtime_files = rule_runner.request(
Digest, [CreateDigest([Directory(path=dirpath)])])
dirpath_parts = os.path.split(dirpath)
for i in range(0, len(dirpath_parts)):
working_dir = os.path.join(*dirpath_parts[:i]) if i > 0 else None
expected_subdir = os.path.join(
*dirpath_parts[i:]) if i < len(dirpath_parts) else None
process = rule_runner.request(
Process,
[
pex_process_type(
pex_data.pex,
description="Run the pex and check its cwd",
working_directory=working_dir,
input_digest=runtime_files,
# We skip the process cache for this PEX to ensure that it re-runs.
cache_scope=ProcessCacheScope.PER_SESSION,
)
],
)
# For VenvPexes, run the PEX twice while clearing the venv dir in between. This emulates
# situations where a PEX creation hits the process cache, while venv seeding misses the PEX
# cache.
if isinstance(pex_data.pex, VenvPex):
# Request once to ensure that the directory is seeded, and then start a new session so
# that the second run happens as well.
_ = rule_runner.request(ProcessResult, [process])
rule_runner.new_session("re-run-for-venv-pex")
rule_runner.set_options(
["--backend-packages=pants.backend.python"],
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
# Clear the cache.
venv_dir = os.path.join(named_caches_dir, "pex_root",
pex_data.pex.venv_rel_dir)
assert os.path.isdir(venv_dir)
safe_rmtree(venv_dir)
result = rule_runner.request(ProcessResult, [process])
output_str = result.stdout.decode()
mo = re.search(r"CWD: (.*)\n", output_str)
assert mo is not None
reported_cwd = mo.group(1)
if working_dir:
assert reported_cwd.endswith(working_dir)
if expected_subdir:
assert f"DIR: {expected_subdir}" in output_str
async def compile_java_source(
bash: BashBinary,
jdk_setup: JdkSetup,
zip_binary: ZipBinary,
union_membership: UnionMembership,
request: CompileJavaSourceRequest,
) -> FallibleClasspathEntry:
# Request the component's direct dependency classpath.
direct_dependency_classpath_entries = FallibleClasspathEntry.if_all_succeeded(
await MultiGet(
Get(
FallibleClasspathEntry,
ClasspathEntryRequest,
ClasspathEntryRequest.for_targets(union_membership,
component=coarsened_dep,
resolve=request.resolve),
) for coarsened_dep in request.component.dependencies))
if direct_dependency_classpath_entries is None:
return FallibleClasspathEntry(
description=str(request.component),
result=CompileResult.DEPENDENCY_FAILED,
output=None,
exit_code=1,
)
# Then collect the component's sources.
component_members_with_sources = tuple(t for t in request.component.members
if t.has_field(SourcesField))
component_members_and_source_files = zip(
component_members_with_sources,
await MultiGet(
Get(
SourceFiles,
SourceFilesRequest(
(t.get(SourcesField), ),
for_sources_types=(JavaSourceField, ),
enable_codegen=True,
),
) for t in component_members_with_sources),
)
component_members_and_java_source_files = [
(target, sources)
for target, sources in component_members_and_source_files
if sources.snapshot.digest != EMPTY_DIGEST
]
if not component_members_and_java_source_files:
# Is a generator, and so exports all of its direct deps.
exported_digest = await Get(
Digest,
MergeDigests(cpe.digest
for cpe in direct_dependency_classpath_entries))
classpath_entry = ClasspathEntry.merge(
exported_digest, direct_dependency_classpath_entries)
return FallibleClasspathEntry(
description=str(request.component),
result=CompileResult.SUCCEEDED,
output=classpath_entry,
exit_code=0,
)
dest_dir = "classfiles"
(merged_direct_dependency_classpath_digest,
dest_dir_digest) = await MultiGet(
Get(
Digest,
MergeDigests(
classfiles.digest
for classfiles in direct_dependency_classpath_entries),
),
Get(
Digest,
CreateDigest([Directory(dest_dir)]),
),
)
usercp = "__cp"
prefixed_direct_dependency_classpath_digest = await Get(
Digest, AddPrefix(merged_direct_dependency_classpath_digest, usercp))
classpath_arg = ClasspathEntry.arg(direct_dependency_classpath_entries,
prefix=usercp)
merged_digest = await Get(
Digest,
MergeDigests((
prefixed_direct_dependency_classpath_digest,
dest_dir_digest,
jdk_setup.digest,
*(sources.snapshot.digest
for _, sources in component_members_and_java_source_files),
)),
)
# Compile.
compile_result = await Get(
FallibleProcessResult,
Process(
argv=[
*jdk_setup.args(bash,
[f"{jdk_setup.java_home}/lib/tools.jar"]),
"com.sun.tools.javac.Main",
*(("-cp", classpath_arg) if classpath_arg else ()),
"-d",
dest_dir,
*sorted(
chain.from_iterable(
sources.snapshot.files for _, sources in
component_members_and_java_source_files)),
],
input_digest=merged_digest,
use_nailgun=jdk_setup.digest,
append_only_caches=jdk_setup.append_only_caches,
env=jdk_setup.env,
output_directories=(dest_dir, ),
description=f"Compile {request.component} with javac",
level=LogLevel.DEBUG,
),
)
if compile_result.exit_code != 0:
return FallibleClasspathEntry.from_fallible_process_result(
str(request.component),
compile_result,
None,
)
# Jar.
# NB: We jar up the outputs in a separate process because the nailgun runner cannot support
# invoking via a `bash` wrapper (since the trailing portion of the command is executed by
# the nailgun server). We might be able to resolve this in the future via a Javac wrapper shim.
output_snapshot = await Get(Snapshot, Digest, compile_result.output_digest)
output_file = f"{request.component.representative.address.path_safe_spec}.jar"
if output_snapshot.files:
jar_result = await Get(
ProcessResult,
Process(
argv=[
bash.path,
"-c",
" ".join([
"cd", dest_dir, ";", zip_binary.path, "-r",
f"../{output_file}", "."
]),
],
input_digest=compile_result.output_digest,
output_files=(output_file, ),
description=f"Capture outputs of {request.component} for javac",
level=LogLevel.TRACE,
),
)
jar_output_digest = jar_result.output_digest
else:
# If there was no output, then do not create a jar file. This may occur, for example, when compiling
# a `package-info.java` in a single partition.
jar_output_digest = EMPTY_DIGEST
return FallibleClasspathEntry.from_fallible_process_result(
str(request.component),
compile_result,
ClasspathEntry(jar_output_digest, (output_file, ),
direct_dependency_classpath_entries),
)
async def compile_avro_source(
request: CompileAvroSourceRequest,
jdk: InternalJdk,
avro_tools: AvroSubsystem,
) -> CompiledAvroSource:
output_dir = "_generated_files"
toolcp_relpath = "__toolcp"
lockfile_request = await Get(GenerateJvmLockfileFromTool, AvroToolLockfileSentinel())
tool_classpath, subsetted_input_digest, empty_output_dir = await MultiGet(
Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)),
Get(
Digest,
DigestSubset(
request.digest,
PathGlobs(
[request.path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin="the Avro source file name",
),
),
),
Get(Digest, CreateDigest([Directory(output_dir)])),
)
input_digest = await Get(
Digest,
MergeDigests(
[
subsetted_input_digest,
empty_output_dir,
]
),
)
extra_immutable_input_digests = {
toolcp_relpath: tool_classpath.digest,
}
def make_avro_process(
args: Iterable[str],
*,
overridden_input_digest: Digest | None = None,
overridden_output_dir: str | None = None,
) -> JvmProcess:
return JvmProcess(
jdk=jdk,
argv=(
"org.apache.avro.tool.Main",
*args,
),
classpath_entries=tool_classpath.classpath_entries(toolcp_relpath),
input_digest=(
overridden_input_digest if overridden_input_digest is not None else input_digest
),
extra_immutable_input_digests=extra_immutable_input_digests,
extra_nailgun_keys=extra_immutable_input_digests,
description="Generating Java sources from Avro source.",
level=LogLevel.DEBUG,
output_directories=(overridden_output_dir if overridden_output_dir else output_dir,),
)
path = PurePath(request.path)
if path.suffix == ".avsc":
result = await Get(
ProcessResult,
JvmProcess,
make_avro_process(["compile", "schema", request.path, output_dir]),
)
elif path.suffix == ".avpr":
result = await Get(
ProcessResult,
JvmProcess,
make_avro_process(["compile", "protocol", request.path, output_dir]),
)
elif path.suffix == ".avdl":
idl_output_dir = "__idl"
avpr_path = os.path.join(idl_output_dir, str(path.with_suffix(".avpr")))
idl_output_dir_digest = await Get(
Digest, CreateDigest([Directory(os.path.dirname(avpr_path))])
)
idl_input_digest = await Get(Digest, MergeDigests([input_digest, idl_output_dir_digest]))
idl_result = await Get(
ProcessResult,
JvmProcess,
make_avro_process(
["idl", request.path, avpr_path],
overridden_input_digest=idl_input_digest,
overridden_output_dir=idl_output_dir,
),
)
generated_files_dir = await Get(Digest, CreateDigest([Directory(output_dir)]))
protocol_input_digest = await Get(
Digest, MergeDigests([idl_result.output_digest, generated_files_dir])
)
result = await Get(
ProcessResult,
JvmProcess,
make_avro_process(
["compile", "protocol", avpr_path, output_dir],
overridden_input_digest=protocol_input_digest,
),
)
else:
raise AssertionError(
f"Avro backend does not support files with extension `{path.suffix}`: {path}"
)
normalized_digest = await Get(Digest, RemovePrefix(result.output_digest, output_dir))
return CompiledAvroSource(normalized_digest)
