async def strip_source_roots(source_files: SourceFiles) -> StrippedSourceFiles:
"""Removes source roots from a snapshot.
E.g. `src/python/pants/util/strutil.py` -> `pants/util/strutil.py`.
"""
if not source_files.snapshot.files:
return StrippedSourceFiles(source_files.snapshot)
if source_files.unrooted_files:
rooted_files = set(source_files.snapshot.files) - set(
source_files.unrooted_files)
rooted_files_snapshot = await Get(
Snapshot,
DigestSubset(source_files.snapshot.digest,
PathGlobs(rooted_files)))
else:
rooted_files_snapshot = source_files.snapshot
source_roots_result = await Get(
SourceRootsResult,
SourceRootsRequest,
SourceRootsRequest.for_files(rooted_files_snapshot.files),
)
source_roots_to_files = defaultdict(set)
for f, root in source_roots_result.path_to_root.items():
source_roots_to_files[root.path].add(str(f))
if len(source_roots_to_files) == 1:
source_root = next(iter(source_roots_to_files.keys()))
if source_root == ".":
resulting_snapshot = rooted_files_snapshot
else:
resulting_snapshot = await Get(
Snapshot,
RemovePrefix(rooted_files_snapshot.digest, source_root))
else:
digest_subsets = await MultiGet(
Get(Digest,
DigestSubset(rooted_files_snapshot.digest, PathGlobs(files)))
for files in source_roots_to_files.values())
resulting_digests = await MultiGet(
Get(Digest, RemovePrefix(digest, source_root))
for digest, source_root in zip(digest_subsets,
source_roots_to_files.keys()))
resulting_snapshot = await Get(Snapshot,
MergeDigests(resulting_digests))
# Add the unrooted files back in.
if source_files.unrooted_files:
unrooted_files_digest = await Get(
Digest,
DigestSubset(source_files.snapshot.digest,
PathGlobs(source_files.unrooted_files)),
)
resulting_snapshot = await Get(
Snapshot,
MergeDigests((resulting_snapshot.digest, unrooted_files_digest)))
return StrippedSourceFiles(resulting_snapshot)
async def run_python_test(field_set: PythonTestFieldSet,
test_subsystem: TestSubsystem,
pytest: PyTest) -> TestResult:
if field_set.is_conftest_or_type_stub():
return TestResult.skip(field_set.address)
setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=False))
result = await Get(FallibleProcessResult, Process, setup.process)
coverage_data = None
if test_subsystem.use_coverage:
coverage_snapshot = await Get(
Snapshot,
DigestSubset(result.output_digest, PathGlobs([".coverage"])))
if coverage_snapshot.files == (".coverage", ):
coverage_data = PytestCoverageData(field_set.address,
coverage_snapshot.digest)
else:
logger.warning(
f"Failed to generate coverage data for {field_set.address}.")
xml_results_snapshot = None
if setup.results_file_name:
xml_results_snapshot = await Get(
Snapshot,
DigestSubset(result.output_digest,
PathGlobs([setup.results_file_name])))
if xml_results_snapshot.files == (setup.results_file_name, ):
xml_results_snapshot = await Get(
Snapshot,
AddPrefix(xml_results_snapshot.digest,
pytest.options.junit_xml_dir),
)
else:
logger.warning(
f"Failed to generate JUnit XML data for {field_set.address}.")
extra_output_snapshot = await Get(
Snapshot,
DigestSubset(result.output_digest,
PathGlobs([f"{_EXTRA_OUTPUT_DIR}/**"])))
extra_output_snapshot = await Get(
Snapshot, RemovePrefix(extra_output_snapshot.digest,
_EXTRA_OUTPUT_DIR))
return TestResult.from_fallible_process_result(
result,
address=field_set.address,
coverage_data=coverage_data,
xml_results=xml_results_snapshot,
extra_output=extra_output_snapshot,
)
async def find_build_system(request: BuildSystemRequest, setuptools: Setuptools) -> BuildSystem:
digest_contents = await Get(
DigestContents,
DigestSubset(
request.digest,
PathGlobs(
globs=[os.path.join(request.working_directory, "pyproject.toml")],
glob_match_error_behavior=GlobMatchErrorBehavior.ignore,
),
),
)
ret = None
if digest_contents:
file_content = next(iter(digest_contents))
settings: Mapping[str, Any] = toml.loads(file_content.content.decode())
build_system = settings.get("build-system")
if build_system is not None:
build_backend = build_system.get("build-backend")
if build_backend is None:
raise InvalidBuildConfigError(
f"No build-backend found in the [build-system] table in {file_content.path}"
)
requires = build_system.get("requires")
if requires is None:
raise InvalidBuildConfigError(
f"No requires found in the [build-system] table in {file_content.path}"
)
ret = BuildSystem(PexRequirements(requires), build_backend)
# Per PEP 517: "If the pyproject.toml file is absent, or the build-backend key is missing,
# the source tree is not using this specification, and tools should revert to the legacy
# behaviour of running setup.py."
if ret is None:
ret = BuildSystem.legacy(setuptools)
return ret
def test_digest_subset_empty(rule_runner: RuleRunner) -> None:
subset_snapshot = rule_runner.request(
Snapshot, [DigestSubset(generate_original_digest(rule_runner), PathGlobs(()))]
)
assert subset_snapshot.digest == EMPTY_DIGEST
assert subset_snapshot.files == ()
assert subset_snapshot.dirs == ()
async def download_external_helm_plugin(
request: ExternalHelmPluginRequest) -> HelmPlugin:
downloaded_tool = await Get(DownloadedExternalTool, ExternalToolRequest,
request.tool_request)
metadata_file = await Get(
Digest,
DigestSubset(
downloaded_tool.digest,
PathGlobs(
["plugin.yaml"],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=
f"The Helm plugin `{request.plugin_name}`",
),
),
)
metadata_content = await Get(DigestContents, Digest, metadata_file)
if len(metadata_content) == 0:
raise HelmPluginMetadataFileNotFound(request.plugin_name)
metadata = HelmPluginMetadata.from_bytes(metadata_content[0].content)
if not metadata.command and not metadata.platform_command:
raise HelmPluginMissingCommand(request.plugin_name)
return HelmPlugin(metadata=metadata, digest=downloaded_tool.digest)
def test_digest_subset_globs(rule_runner: RuleRunner) -> None:
subset_snapshot = rule_runner.request(
Snapshot,
[
DigestSubset(
generate_original_digest(rule_runner),
PathGlobs(("a.txt", "c.txt", "subdir2/**")),
)
],
)
assert set(subset_snapshot.files) == {
"a.txt",
"c.txt",
"subdir2/a.txt",
"subdir2/nested_subdir/x.txt",
}
assert set(subset_snapshot.dirs) == {"subdir2", "subdir2/nested_subdir"}
expected_files = [
FileContent(path, b"dummy content")
for path in [
"a.txt",
"c.txt",
"subdir2/a.txt",
"subdir2/nested_subdir/x.txt",
]
]
subset_digest = rule_runner.request(Digest, [CreateDigest(expected_files)])
assert subset_snapshot.digest == subset_digest
def test_digest_subset_globs(self) -> None:
subset_snapshot = self.request_product(
Snapshot,
[
DigestSubset(
self.generate_original_digest(),
PathGlobs(("a.txt", "c.txt", "subdir2/**")),
)
],
)
assert set(subset_snapshot.files) == {
"a.txt",
"c.txt",
"subdir2/a.txt",
"subdir2/nested_subdir/x.txt",
}
assert set(
subset_snapshot.dirs) == {"subdir2", "subdir2/nested_subdir"}
content = b"dummy content"
subset_input = CreateDigest((
FileContent(path="a.txt", content=content),
FileContent(path="c.txt", content=content),
FileContent(path="subdir2/a.txt", content=content),
FileContent(path="subdir2/nested_subdir/x.txt", content=content),
))
subset_digest = self.request_product(Digest, [subset_input])
assert subset_snapshot.digest == subset_digest
async def get_sources(request: SetupPySourcesRequest) -> SetupPySources:
python_sources_request = PythonSourceFilesRequest(
targets=request.targets, include_resources=False, include_files=False
)
all_sources_request = PythonSourceFilesRequest(
targets=request.targets, include_resources=True, include_files=True
)
python_sources, all_sources = await MultiGet(
Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, python_sources_request),
Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, all_sources_request),
)
python_files = set(python_sources.stripped_source_files.snapshot.files)
all_files = set(all_sources.stripped_source_files.snapshot.files)
resource_files = all_files - python_files
init_py_digest_contents = await Get(
DigestContents,
DigestSubset(
python_sources.stripped_source_files.snapshot.digest, PathGlobs(["**/__init__.py"])
),
)
packages, namespace_packages, package_data = find_packages(
python_files=python_files,
resource_files=resource_files,
init_py_digest_contents=init_py_digest_contents,
py2=request.py2,
)
return SetupPySources(
digest=all_sources.stripped_source_files.snapshot.digest,
packages=packages,
namespace_packages=namespace_packages,
package_data=package_data,
)
def test_empty_digest_subset(self) -> None:
subset_snapshot = self.request_product(
Snapshot,
[DigestSubset(self.generate_original_digest(), PathGlobs(()))])
assert subset_snapshot.digest == EMPTY_DIGEST
assert subset_snapshot.files == ()
assert subset_snapshot.dirs == ()
async def flake8_lint_partition(
partition: Flake8Partition, flake8: Flake8, lint_subsystem: LintSubsystem
) -> LintResult:
flake8_pex_get = Get(
VenvPex,
PexRequest(
output_filename="flake8.pex",
internal_only=True,
requirements=PexRequirements(flake8.all_requirements),
interpreter_constraints=partition.interpreter_constraints,
main=flake8.main,
),
)
config_files_get = Get(ConfigFiles, ConfigFilesRequest, flake8.config_request)
source_files_get = Get(
SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)
)
flake8_pex, config_files, source_files = await MultiGet(
flake8_pex_get, config_files_get, source_files_get
)
input_digest = await Get(
Digest, MergeDigests((source_files.snapshot.digest, config_files.snapshot.digest))
)
report_file_name = "flake8_report.txt" if lint_subsystem.reports_dir else None
result = await Get(
FallibleProcessResult,
VenvPexProcess(
flake8_pex,
argv=generate_argv(source_files, flake8, report_file_name=report_file_name),
input_digest=input_digest,
output_files=(report_file_name,) if report_file_name else None,
description=f"Run Flake8 on {pluralize(len(partition.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = None
if report_file_name:
report_digest = await Get(
Digest,
DigestSubset(
result.output_digest,
PathGlobs(
[report_file_name],
glob_match_error_behavior=GlobMatchErrorBehavior.warn,
description_of_origin="Flake8 report file",
),
),
)
report = LintReport(report_file_name, report_digest)
return LintResult.from_fallible_process_result(
result,
partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
def _chart_metadata_subset(digest: Digest) -> DigestSubset:
globs = PathGlobs(
[f"**/{filename}" for filename in _HELM_CHART_METADATA_FILENAMES],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.any_match,
description_of_origin="parse_chart_metadata",
)
return DigestSubset(digest, globs)
def test_digest_subset_globs_2(self) -> None:
ds = DigestSubset(
self.generate_original_digest(), PathGlobs(("a.txt", "c.txt", "subdir2/*"))
)
subset_snapshot = self.request_single_product(Snapshot, ds)
assert set(subset_snapshot.files) == {"a.txt", "c.txt", "subdir2/a.txt"}
assert set(subset_snapshot.dirs) == {"subdir2", "subdir2/nested_subdir"}
async def fetch_helm_artifacts(
request: FetchHelmArfifactsRequest) -> FetchedHelmArtifacts:
download_prefix = "__downloads"
empty_download_digest = await Get(
Digest, CreateDigest([Directory(download_prefix)]))
artifacts = await MultiGet(
Get(ResolvedHelmArtifact, HelmArtifact,
HelmArtifact.from_field_set(field_set))
for field_set in request.field_sets)
def create_fetch_process(artifact: ResolvedHelmArtifact) -> HelmProcess:
return HelmProcess(
argv=[
"pull",
artifact.name,
"--repo",
artifact.location_url,
"--version",
artifact.version,
"--destination",
download_prefix,
"--untar",
],
input_digest=empty_download_digest,
description=
f"Pulling Helm Chart '{artifact.name}' with version {artifact.version}",
output_directories=(download_prefix, ),
)
download_results = await MultiGet(
Get(
ProcessResult,
HelmProcess,
create_fetch_process(artifact),
) for artifact in artifacts)
stripped_artifact_digests = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, download_prefix))
for result in download_results)
# Avoid capturing the tarball that has been downloaded by Helm during the pull.
artifact_snapshots = await MultiGet(
Get(
Snapshot,
DigestSubset(digest, PathGlobs([os.path.join(artifact.name, "**")
])))
for artifact, digest in zip(artifacts, stripped_artifact_digests))
fetched_artifacts = [
FetchedHelmArtifact(artifact=artifact, snapshot=snapshot)
for artifact, snapshot in zip(artifacts, artifact_snapshots)
]
logger.debug(
f"Fetched {pluralize(len(fetched_artifacts), 'Helm artifact')} corresponding with:\n"
f"{bullet_list([artifact.address.spec for artifact in fetched_artifacts], max_elements=10)}"
)
return FetchedHelmArtifacts(fetched_artifacts)
async def get_sources(
request: DistBuildChrootRequest, union_membership: UnionMembership
) -> DistBuildSources:
owned_deps, transitive_targets = await MultiGet(
Get(OwnedDependencies, DependencyOwner(request.exported_target)),
Get(
TransitiveTargets,
TransitiveTargetsRequest([request.exported_target.target.address]),
),
)
# files() targets aren't owned by a single exported target - they aren't code, so
# we allow them to be in multiple dists. This is helpful for, e.g., embedding
# a standard license file in a dist.
# TODO: This doesn't actually work, the generated setup.py has no way of referencing
# these, since they aren't in a package, so they won't get included in the built dists.
# There is a separate `license_files()` setup.py kwarg that we should use for this
# special case (see https://setuptools.pypa.io/en/latest/references/keywords.html).
file_targets = targets_with_sources_types(
[FileSourceField], transitive_targets.closure, union_membership
)
targets = Targets(itertools.chain((od.target for od in owned_deps), file_targets))
python_sources_request = PythonSourceFilesRequest(
targets=targets, include_resources=False, include_files=False
)
all_sources_request = PythonSourceFilesRequest(
targets=targets, include_resources=True, include_files=True
)
python_sources, all_sources = await MultiGet(
Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, python_sources_request),
Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, all_sources_request),
)
python_files = set(python_sources.stripped_source_files.snapshot.files)
all_files = set(all_sources.stripped_source_files.snapshot.files)
resource_files = all_files - python_files
init_py_digest_contents = await Get(
DigestContents,
DigestSubset(
python_sources.stripped_source_files.snapshot.digest, PathGlobs(["**/__init__.py"])
),
)
packages, namespace_packages, package_data = find_packages(
python_files=python_files,
resource_files=resource_files,
init_py_digest_contents=init_py_digest_contents,
# Whether to use py2 or py3 package semantics.
py2=request.interpreter_constraints.includes_python2(),
)
return DistBuildSources(
digest=all_sources.stripped_source_files.snapshot.digest,
packages=packages,
namespace_packages=namespace_packages,
package_data=package_data,
)
def test_digest_subset_globs_2(rule_runner: RuleRunner) -> None:
subset_snapshot = rule_runner.request(
Snapshot,
[
DigestSubset(generate_original_digest(rule_runner),
PathGlobs(("a.txt", "c.txt", "subdir2/*")))
],
)
assert set(subset_snapshot.files) == {"a.txt", "c.txt", "subdir2/a.txt"}
assert set(subset_snapshot.dirs) == {"subdir2", "subdir2/nested_subdir"}
def test_digest_subset_nonexistent_filename_globs(
rule_runner: RuleRunner) -> None:
# We behave according to the `GlobMatchErrorBehavior`.
original_digest = generate_original_digest(rule_runner)
globs = ["some_file_not_in_snapshot.txt", "a.txt"]
subset_snapshot = rule_runner.request(
Snapshot, [DigestSubset(original_digest, PathGlobs(globs))])
assert set(subset_snapshot.files) == {"a.txt"}
expected_digest = rule_runner.request(
Digest, [CreateDigest([FileContent("a.txt", b"dummy content")])])
assert subset_snapshot.digest == expected_digest
async def isolate_local_dist_wheels(
dist_field_set: PythonDistributionFieldSet,
bash: BashBinary,
unzip_binary: UnzipBinary,
) -> LocalDistWheels:
dist = await Get(BuiltPackage, PackageFieldSet, dist_field_set)
wheels_snapshot = await Get(Snapshot, DigestSubset(dist.digest, PathGlobs(["**/*.whl"])))
# A given local dist might build a wheel and an sdist (and maybe other artifacts -
# we don't know what setup command was run...)
# As long as there is a wheel, we can ignore the other artifacts.
artifacts = {(a.relpath or "") for a in dist.artifacts}
wheels = [wheel for wheel in wheels_snapshot.files if wheel in artifacts]
if not wheels:
tgt = await Get(
WrappedTarget,
WrappedTargetRequest(dist_field_set.address, description_of_origin="<infallible>"),
)
logger.warning(
softwrap(
f"""
Encountered a dependency on the {tgt.target.alias} target at {dist_field_set.address},
but this target does not produce a Python wheel artifact. Therefore this target's
code will be used directly from sources, without a distribution being built,
and any native extensions in it will not be built.
See {doc_url('python-distributions')} for details on how to set up a
{tgt.target.alias} target to produce a wheel.
"""
)
)
wheels_listing_result = await Get(
ProcessResult,
Process(
argv=[
bash.path,
"-c",
f"""
set -ex
for f in {' '.join(shlex.quote(f) for f in wheels)}; do
{unzip_binary.path} -Z1 "$f"
done
""",
],
input_digest=wheels_snapshot.digest,
description=f"List contents of artifacts produced by {dist_field_set.address}",
),
)
provided_files = set(wheels_listing_result.stdout.decode().splitlines())
return LocalDistWheels(tuple(wheels), wheels_snapshot.digest, frozenset(provided_files))
def test_nonexistent_filename_globs(self) -> None:
# We expect to ignore, rather than error, on files that don't exist in the original snapshot.
ds = DigestSubset(
self.generate_original_digest(), PathGlobs(("some_file_not_in_snapshot.txt", "a.txt")),
)
subset_snapshot = self.request_single_product(Snapshot, ds)
assert set(subset_snapshot.files) == {"a.txt"}
content = b"dummy content"
subset_input = CreateDigest((FileContent(path="a.txt", content=content),))
subset_digest = self.request_single_product(Digest, subset_input)
assert subset_snapshot.digest == subset_digest
def _chart_metadata_subset(
digest: Digest, *, description_of_origin: str, prefix: str | None = None
) -> DigestSubset:
def prefixed_filename(filename: str) -> str:
if not prefix:
return filename
return os.path.join(prefix, filename)
glob_exprs = [prefixed_filename(filename) for filename in HELM_CHART_METADATA_FILENAMES]
globs = PathGlobs(
glob_exprs,
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.any_match,
description_of_origin=description_of_origin,
)
return DigestSubset(digest, globs)
async def run_scalatest_test(
test_subsystem: TestSubsystem,
field_set: ScalatestTestFieldSet,
) -> TestResult:
test_setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=False))
process_result = await Get(FallibleProcessResult, JvmProcess, test_setup.process)
reports_dir_prefix = test_setup.reports_dir_prefix
xml_result_subset = await Get(
Digest, DigestSubset(process_result.output_digest, PathGlobs([f"{reports_dir_prefix}/**"]))
)
xml_results = await Get(Snapshot, RemovePrefix(xml_result_subset, reports_dir_prefix))
return TestResult.from_fallible_process_result(
process_result,
address=field_set.address,
output_setting=test_subsystem.output,
xml_results=xml_results,
)
async def infer_shell_dependencies(
request: InferShellDependencies, shell_mapping: ShellMapping,
shell_setup: ShellSetup) -> InferredDependencies:
if not shell_setup.dependency_inference:
return InferredDependencies([], sibling_dependencies_inferrable=False)
address = request.sources_field.address
wrapped_tgt = await Get(WrappedTarget, Address, address)
explicitly_provided_deps, hydrated_sources = await MultiGet(
Get(ExplicitlyProvidedDependencies,
DependenciesRequest(wrapped_tgt.target[Dependencies])),
Get(HydratedSources, HydrateSourcesRequest(request.sources_field)),
)
per_file_digests = await MultiGet(
Get(Digest,
DigestSubset(hydrated_sources.snapshot.digest, PathGlobs([f])))
for f in hydrated_sources.snapshot.files)
all_detected_imports = await MultiGet(
Get(ParsedShellImports, ParseShellImportsRequest(digest, f))
for digest, f in zip(per_file_digests, hydrated_sources.snapshot.files)
)
result: OrderedSet[Address] = OrderedSet()
for detected_imports in all_detected_imports:
for import_path in detected_imports:
unambiguous = shell_mapping.mapping.get(import_path)
ambiguous = shell_mapping.ambiguous_modules.get(import_path)
if unambiguous:
result.add(unambiguous)
elif ambiguous:
explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference(
ambiguous,
address,
import_reference="file",
context=f"The target {address} sources `{import_path}`",
)
maybe_disambiguated = explicitly_provided_deps.disambiguated_via_ignores(
ambiguous)
if maybe_disambiguated:
result.add(maybe_disambiguated)
return InferredDependencies(sorted(result),
sibling_dependencies_inferrable=True)
async def setup_scalafmt_partition(
request: SetupScalafmtPartition,
jdk: InternalJdk,
) -> Partition:
sources_digest = await Get(
Digest,
DigestSubset(
request.merged_sources_digest,
PathGlobs(
[request.config_file, *request.files],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin=f"the files in scalafmt partition for config file {request.config_file}",
),
),
)
args = [
"org.scalafmt.cli.Cli",
f"--config={request.config_file}",
"--non-interactive",
]
if request.check_only:
args.append("--list")
else:
args.append("--quiet")
args.extend(request.files)
process = JvmProcess(
jdk=jdk,
argv=args,
classpath_entries=request.classpath_entries,
input_digest=sources_digest,
output_files=request.files,
extra_immutable_input_digests=request.extra_immutable_input_digests,
# extra_nailgun_keys=request.extra_immutable_input_digests,
use_nailgun=False,
description=f"Run `scalafmt` on {pluralize(len(request.files), 'file')}.",
level=LogLevel.DEBUG,
)
return Partition(process, f"{pluralize(len(request.files), 'file')} ({request.config_file})")
async def digest_to_file_digest(request: ExtractFileDigest) -> FileDigest:
"""TODO(#11806): This is just a workaround; this extraction should be provided directly by the engine."""
digest = await Get(
Digest, DigestSubset(request.digest, PathGlobs([request.file_path])))
digest_contents = await Get(DigestContents, Digest, digest)
if len(digest_contents) == 0:
raise Exception(
f"ExtractFileDigest: '{request.file_path}' not found in {request.digest}."
)
elif len(digest_contents) > 1:
raise Exception(
f"ExtractFileDigest: Unexpected error: '{request.file_path}' found multiple times in {request.digest}"
)
file_content = digest_contents[0]
hasher = hashlib.sha256()
hasher.update(file_content.content)
return FileDigest(fingerprint=hasher.hexdigest(),
serialized_bytes_length=len(file_content.content))
async def digest_to_file_digest(request: ExtractFileDigest) -> FileDigest:
digest = await Get(
Digest, DigestSubset(request.digest, PathGlobs([request.file_path])))
digest_entries = await Get(DigestEntries, Digest, digest)
if len(digest_entries) == 0:
raise Exception(
f"ExtractFileDigest: '{request.file_path}' not found in {request.digest}."
)
elif len(digest_entries) > 1:
raise Exception(
f"ExtractFileDigest: Unexpected error: '{request.file_path}' found multiple times in {request.digest}"
)
file_info = digest_entries[0]
if not isinstance(file_info, FileEntry):
raise AssertionError(
f"Unexpected error: '{request.file_path}' refers to a directory, not a file."
)
return file_info.file_digest
async def build_local_dists(
request: LocalDistsPexRequest,
) -> LocalDistsPex:
transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses))
applicable_targets = [
tgt for tgt in transitive_targets.closure if PythonDistributionFieldSet.is_applicable(tgt)
]
local_dists_wheels = await MultiGet(
Get(LocalDistWheels, PythonDistributionFieldSet, PythonDistributionFieldSet.create(target))
for target in applicable_targets
)
# The primary use-case of the "local dists" feature is to support consuming native extensions
# as wheels without having to publish them first.
# It doesn't seem very useful to consume locally-built sdists, and it makes it hard to
# reason about possible sys.path collisions between the in-repo sources and whatever the
# sdist will place on the sys.path when it's installed.
# So for now we simply ignore sdists, with a warning if necessary.
provided_files: set[str] = set()
wheels: list[str] = []
wheels_digests = []
for local_dist_wheels in local_dists_wheels:
wheels.extend(local_dist_wheels.wheel_paths)
wheels_digests.append(local_dist_wheels.wheels_digest)
provided_files.update(local_dist_wheels.provided_files)
wheels_digest = await Get(Digest, MergeDigests(wheels_digests))
dists_pex = await Get(
Pex,
PexRequest(
output_filename="local_dists.pex",
requirements=PexRequirements(wheels),
interpreter_constraints=request.interpreter_constraints,
additional_inputs=wheels_digest,
internal_only=request.internal_only,
additional_args=["--intransitive"],
),
)
if not wheels:
# The source calculations below are not (always) cheap, so we skip them if no wheels were
# produced. See https://github.com/pantsbuild/pants/issues/14561 for one possible approach
# to sharing the cost of these calculations.
return LocalDistsPex(dists_pex, request.sources)
# We check source roots in reverse lexicographic order,
# so we'll find the innermost root that matches.
source_roots = sorted(request.sources.source_roots, reverse=True)
remaining_sources = set(request.sources.source_files.files)
unrooted_files_set = set(request.sources.source_files.unrooted_files)
for source in request.sources.source_files.files:
if source not in unrooted_files_set:
for source_root in source_roots:
source_relpath = fast_relpath_optional(source, source_root)
if source_relpath is not None and source_relpath in provided_files:
remaining_sources.remove(source)
remaining_sources_snapshot = await Get(
Snapshot,
DigestSubset(
request.sources.source_files.snapshot.digest, PathGlobs(sorted(remaining_sources))
),
)
subtracted_sources = PythonSourceFiles(
SourceFiles(remaining_sources_snapshot, request.sources.source_files.unrooted_files),
request.sources.source_roots,
)
return LocalDistsPex(dists_pex, subtracted_sources)
async def bandit_lint_partition(partition: BanditPartition, bandit: Bandit,
lint_subsystem: LintSubsystem) -> LintResult:
bandit_pex_request = Get(
Pex,
PexRequest(
output_filename="bandit.pex",
internal_only=True,
requirements=PexRequirements(bandit.all_requirements),
interpreter_constraints=partition.interpreter_constraints,
entry_point=bandit.entry_point,
),
)
config_digest_request = Get(
Digest,
PathGlobs(
globs=[bandit.config] if bandit.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--bandit-config`",
),
)
source_files_request = Get(
SourceFiles,
SourceFilesRequest(field_set.sources
for field_set in partition.field_sets))
bandit_pex, config_digest, source_files = await MultiGet(
bandit_pex_request, config_digest_request, source_files_request)
input_digest = await Get(
Digest,
MergeDigests(
(source_files.snapshot.digest, bandit_pex.digest, config_digest)))
report_file_name = "bandit_report.txt" if lint_subsystem.reports_dir else None
result = await Get(
FallibleProcessResult,
PexProcess(
bandit_pex,
argv=generate_args(source_files=source_files,
bandit=bandit,
report_file_name=report_file_name),
input_digest=input_digest,
description=
f"Run Bandit on {pluralize(len(partition.field_sets), 'file')}.",
output_files=(report_file_name, ) if report_file_name else None,
level=LogLevel.DEBUG,
),
)
report = None
if report_file_name:
report_digest = await Get(
Digest,
DigestSubset(
result.output_digest,
PathGlobs(
[report_file_name],
glob_match_error_behavior=GlobMatchErrorBehavior.warn,
description_of_origin="Bandit report file",
),
),
)
report = LintReport(report_file_name, report_digest)
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def build_local_dists(request: LocalDistsPexRequest, ) -> LocalDistsPex:
transitive_targets = await Get(TransitiveTargets,
TransitiveTargetsRequest(request.addresses))
applicable_targets = [
tgt for tgt in transitive_targets.closure
if PythonDistributionFieldSet.is_applicable(tgt)
]
python_dist_field_sets = [
PythonDistributionFieldSet.create(target)
for target in applicable_targets
]
dists = await MultiGet([
Get(BuiltPackage, PackageFieldSet, field_set)
for field_set in python_dist_field_sets
])
# The primary use-case of the "local dists" feature is to support consuming native extensions
# as wheels without having to publish them first.
# It doesn't seem very useful to consume locally-built sdists, and it makes it hard to
# reason about possible sys.path collisions between the in-repo sources and whatever the
# sdist will place on the sys.path when it's installed.
# So for now we simply ignore sdists, with a warning if necessary.
provided_files = set()
wheels = []
all_contents = await MultiGet(
Get(DigestContents, Digest, dist.digest) for dist in dists)
for dist, contents, tgt in zip(dists, all_contents, applicable_targets):
artifacts = set((a.relpath or "") for a in dist.artifacts)
# A given local dist might build a wheel and an sdist (and maybe other artifacts -
# we don't know what setup command was run...)
# As long as there is a wheel, we can ignore the other artifacts.
wheel = next((art for art in artifacts if art.endswith(".whl")), None)
if wheel:
wheel_content = next(content for content in contents
if content.path == wheel)
wheels.append(wheel)
buf = BytesIO()
buf.write(wheel_content.content)
buf.seek(0)
with zipfile.ZipFile(buf) as zf:
provided_files.update(zf.namelist())
else:
logger.warning(
f"Encountered a dependency on the {tgt.alias} target at {tgt.address.spec}, but "
"this target does not produce a Python wheel artifact. Therefore this target's "
"code will be used directly from sources, without a distribution being built, "
"and therefore any native extensions in it will not be built.\n\n"
f"See {doc_url('python-distributions')} for details on how to set up a {tgt.alias} "
"target to produce a wheel.")
dists_digest = await Get(Digest,
MergeDigests([dist.digest for dist in dists]))
wheels_digest = await Get(
Digest, DigestSubset(dists_digest, PathGlobs(["**/*.whl"])))
dists_pex = await Get(
Pex,
PexRequest(
output_filename="local_dists.pex",
requirements=PexRequirements(wheels),
interpreter_constraints=request.interpreter_constraints,
additional_inputs=wheels_digest,
internal_only=True,
),
)
# We check source roots in reverse lexicographic order,
# so we'll find the innermost root that matches.
source_roots = list(reversed(sorted(request.sources.source_roots)))
remaining_sources = set(request.sources.source_files.files)
unrooted_files_set = set(request.sources.source_files.unrooted_files)
for source in request.sources.source_files.files:
if source not in unrooted_files_set:
for source_root in source_roots:
if (source.startswith(source_root) and os.path.relpath(
source, source_root) in provided_files):
remaining_sources.remove(source)
remaining_sources_snapshot = await Get(
Snapshot,
DigestSubset(request.sources.source_files.snapshot.digest,
PathGlobs(sorted(remaining_sources))),
)
subtracted_sources = PythonSourceFiles(
SourceFiles(remaining_sources_snapshot,
request.sources.source_files.unrooted_files),
request.sources.source_roots,
)
return LocalDistsPex(dists_pex, subtracted_sources)
async def coursier_resolve_lockfile(
bash: BashBinary,
coursier: Coursier,
artifact_requirements: ArtifactRequirements,
) -> CoursierResolvedLockfile:
"""Run `coursier fetch ...` against a list of Maven coordinates and capture the result.
This rule does two things in a single Process invocation:
* Runs `coursier fetch` to let Coursier do the heavy lifting of resolving
dependencies and downloading resolved artifacts (jars, etc).
* Copies the resolved artifacts into the Process output directory, capturing
the artifacts as content-addressed `Digest`s.
It's important that this happens in the same process, since the process isn't
guaranteed to run on the same machine as the rule, nor is a subsequent process
invocation. This guarantees that whatever Coursier resolved, it was fully
captured into Pants' content addressed artifact storage.
Note however that we still get the benefit of Coursier's "global" cache if it
had already been run on the machine where the `coursier fetch` runs, so rerunning
`coursier fetch` tends to be fast in practice.
Finally, this rule bundles up the result into a `CoursierResolvedLockfile`. This
data structure encapsulates everything necessary to either materialize the
resolved dependencies to a classpath for Java invocations, or to write the
lockfile out to the workspace to hermetically freeze the result of the resolve.
"""
if len(artifact_requirements) == 0:
return CoursierResolvedLockfile(entries=())
coursier_report_file_name = "coursier_report.json"
process_result = await Get(
ProcessResult,
Process(
argv=[
bash.path,
coursier.wrapper_script,
coursier.coursier.exe,
coursier_report_file_name,
*(req.to_coord_str() for req in artifact_requirements),
],
input_digest=coursier.digest,
output_directories=("classpath",),
output_files=(coursier_report_file_name,),
description=(
"Running `coursier fetch` against "
f"{pluralize(len(artifact_requirements), 'requirement')}: "
f"{', '.join(req.to_coord_str() for req in artifact_requirements)}"
),
level=LogLevel.DEBUG,
),
)
report_digest = await Get(
Digest, DigestSubset(process_result.output_digest, PathGlobs([coursier_report_file_name]))
)
report_contents = await Get(DigestContents, Digest, report_digest)
report = json.loads(report_contents[0].content)
artifact_file_names = tuple(PurePath(dep["file"]).name for dep in report["dependencies"])
artifact_output_paths = tuple(f"classpath/{file_name}" for file_name in artifact_file_names)
artifact_digests = await MultiGet(
Get(Digest, DigestSubset(process_result.output_digest, PathGlobs([output_path])))
for output_path in artifact_output_paths
)
stripped_artifact_digests = await MultiGet(
Get(Digest, RemovePrefix(artifact_digest, "classpath"))
for artifact_digest in artifact_digests
)
artifact_file_digests = await MultiGet(
Get(FileDigest, ExtractFileDigest(stripped_artifact_digest, file_name))
for stripped_artifact_digest, file_name in zip(
stripped_artifact_digests, artifact_file_names
)
)
return CoursierResolvedLockfile(
entries=tuple(
CoursierLockfileEntry(
coord=Coordinate.from_coord_str(dep["coord"]),
direct_dependencies=Coordinates(
Coordinate.from_coord_str(dd) for dd in dep["directDependencies"]
),
dependencies=Coordinates(Coordinate.from_coord_str(d) for d in dep["dependencies"]),
file_name=file_name,
file_digest=artifact_file_digest,
)
for dep, file_name, artifact_file_digest in zip(
report["dependencies"], artifact_file_names, artifact_file_digests
)
)
)
async def coursier_fetch_one_coord(
bash: BashBinary,
coursier: Coursier,
request: CoursierLockfileEntry,
) -> ResolvedClasspathEntry:
"""Run `coursier fetch --intrasitive` to fetch a single artifact.
This rule exists to permit efficient subsetting of a "global" classpath
in the form of a lockfile. Callers can determine what subset of dependencies
from the lockfile are needed for a given target, then request those
lockfile entries individually.
By fetching only one entry at a time, we maximize our cache efficiency. If instead
we fetched the entire subset that the caller wanted, there would be a different cache
key for every possible subset.
This rule also guarantees exact reproducibility. If all caches have been
removed, `coursier fetch` will re-download the artifact, and this rule will
confirm that what was downloaded matches exactly (by content digest) what
was specified in the lockfile (what Coursier originally downloaded).
"""
coursier_report_file_name = "coursier_report.json"
process_result = await Get(
ProcessResult,
Process(
argv=[
bash.path,
coursier.wrapper_script,
coursier.coursier.exe,
coursier_report_file_name,
"--intransitive",
request.coord.to_coord_str(),
],
input_digest=coursier.digest,
output_directories=("classpath",),
output_files=(coursier_report_file_name,),
description="Run coursier resolve",
level=LogLevel.DEBUG,
),
)
report_digest = await Get(
Digest, DigestSubset(process_result.output_digest, PathGlobs([coursier_report_file_name]))
)
report_contents = await Get(DigestContents, Digest, report_digest)
report = json.loads(report_contents[0].content)
report_deps = report["dependencies"]
if len(report_deps) == 0:
raise CoursierError("Coursier fetch report has no dependencies (i.e. nothing was fetched).")
elif len(report_deps) > 1:
raise CoursierError(
"Coursier fetch report has multiple dependencies, but exactly 1 was expected."
)
dep = report_deps[0]
resolved_coord = Coordinate.from_coord_str(dep["coord"])
if resolved_coord != request.coord:
raise CoursierError(
f'Coursier resolved coord "{resolved_coord.to_coord_str()}" does not match requested coord "{request.coord.to_coord_str()}".'
)
file_path = PurePath(dep["file"])
classpath_dest = f"classpath/{file_path.name}"
resolved_file_digest = await Get(
Digest, DigestSubset(process_result.output_digest, PathGlobs([classpath_dest]))
)
stripped_digest = await Get(Digest, RemovePrefix(resolved_file_digest, "classpath"))
file_digest = await Get(
FileDigest,
ExtractFileDigest(stripped_digest, file_path.name),
)
if file_digest != request.file_digest:
raise CoursierError(
f"Coursier fetch for '{resolved_coord}' succeeded, but fetched artifact {file_digest} did not match the expected artifact: {request.file_digest}."
)
return ResolvedClasspathEntry(
coord=request.coord, file_name=file_path.name, digest=stripped_digest
)
async def run_junit_test(
bash: BashBinary,
jdk_setup: JdkSetup,
junit: JUnit,
test_subsystem: TestSubsystem,
field_set: JavaTestFieldSet,
) -> TestResult:
classpath = await Get(Classpath, Addresses([field_set.address]))
junit_classpath = await Get(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__thirdpartycp",
artifact_requirements=(ArtifactRequirements([
Coordinate(
group="org.junit.platform",
artifact="junit-platform-console",
version="1.7.2",
),
Coordinate(
group="org.junit.jupiter",
artifact="junit-jupiter-engine",
version="5.7.2",
),
Coordinate(
group="org.junit.vintage",
artifact="junit-vintage-engine",
version="5.7.2",
),
]), ),
),
)
merged_digest = await Get(
Digest,
MergeDigests((classpath.content.digest, jdk_setup.digest,
junit_classpath.digest)),
)
reports_dir_prefix = "__reports_dir"
reports_dir = f"{reports_dir_prefix}/{field_set.address.path_safe_spec}"
user_classpath_arg = ":".join(classpath.user_classpath_entries())
process_result = await Get(
FallibleProcessResult,
Process(
argv=[
*jdk_setup.args(bash, [
*classpath.classpath_entries(),
*junit_classpath.classpath_entries()
]),
"org.junit.platform.console.ConsoleLauncher",
*(("--classpath",
user_classpath_arg) if user_classpath_arg else ()),
*(("--scan-class-path",
user_classpath_arg) if user_classpath_arg else ()),
"--reports-dir",
reports_dir,
*junit.options.args,
],
input_digest=merged_digest,
output_directories=(reports_dir, ),
append_only_caches=jdk_setup.append_only_caches,
env=jdk_setup.env,
description=
f"Run JUnit 5 ConsoleLauncher against {field_set.address}",
level=LogLevel.DEBUG,
),
)
xml_result_subset = await Get(
Digest,
DigestSubset(process_result.output_digest,
PathGlobs([f"{reports_dir_prefix}/**"])))
xml_results = await Get(
Snapshot, RemovePrefix(xml_result_subset, reports_dir_prefix))
return TestResult.from_fallible_process_result(
process_result,
address=field_set.address,
output_setting=test_subsystem.output,
xml_results=xml_results,
)