def test_add_prefix(self) -> None:
digest = self.request(
Digest,
[
CreateDigest((
FileContent(path="main.py", content=b'print("from main")'),
FileContent(path="subdir/sub.py",
content=b'print("from sub")'),
))
],
)
# Two components.
output_digest = self.request(
Digest, [AddPrefix(digest, "outer_dir/middle_dir")])
snapshot = self.request(Snapshot, [output_digest])
assert sorted(snapshot.files) == [
"outer_dir/middle_dir/main.py",
"outer_dir/middle_dir/subdir/sub.py",
]
assert sorted(snapshot.dirs) == [
"outer_dir",
"outer_dir/middle_dir",
"outer_dir/middle_dir/subdir",
]
# Empty.
output_digest = self.request(Digest, [AddPrefix(digest, "")])
assert digest == output_digest
# Illegal.
with self.assertRaisesRegex(Exception,
r"The `prefix` must be relative."):
self.request(Digest, [AddPrefix(digest, "../something")])
def test_add_prefix(rule_runner: RuleRunner) -> None:
digest = rule_runner.request(
Digest,
[CreateDigest([FileContent("main.ext", b""), FileContent("subdir/sub.ext", b"")])],
)
# Two components.
output_digest = rule_runner.request(Digest, [AddPrefix(digest, "outer_dir/middle_dir")])
snapshot = rule_runner.request(Snapshot, [output_digest])
assert sorted(snapshot.files) == [
"outer_dir/middle_dir/main.ext",
"outer_dir/middle_dir/subdir/sub.ext",
]
assert sorted(snapshot.dirs) == [
"outer_dir",
"outer_dir/middle_dir",
"outer_dir/middle_dir/subdir",
]
# Empty.
output_digest = rule_runner.request(Digest, [AddPrefix(digest, "")])
assert digest == output_digest
# Illegal.
with pytest.raises(Exception, match=r"The `prefix` must be relative."):
rule_runner.request(Digest, [AddPrefix(digest, "../something")])
async def export(
console: Console,
targets: Targets,
export_subsystem: ExportSubsystem,
workspace: Workspace,
union_membership: UnionMembership,
build_root: BuildRoot,
dist_dir: DistDir,
) -> Export:
request_types = cast("Iterable[type[ExportableDataRequest]]",
union_membership.get(ExportableDataRequest))
requests = tuple(request_type(targets) for request_type in request_types)
exportables = await MultiGet(
Get(ExportableData, ExportableDataRequest, request)
for request in requests)
prefixed_digests = await MultiGet(
Get(Digest, AddPrefix(exp.digest, exp.reldir)) for exp in exportables)
output_dir = os.path.join(str(dist_dir.relpath), "export")
merged_digest = await Get(Digest, MergeDigests(prefixed_digests))
dist_digest = await Get(Digest, AddPrefix(merged_digest, output_dir))
workspace.write_digest(dist_digest)
for exp in exportables:
for symlink in exp.symlinks:
# Note that if symlink.source_path is an abspath, join returns it unchanged.
source_abspath = os.path.join(build_root.path, symlink.source_path)
link_abspath = os.path.abspath(
os.path.join(output_dir, exp.reldir, symlink.link_rel_path))
absolute_symlink(source_abspath, link_abspath)
console.print_stdout(
f"Wrote {exp.description} to {os.path.join(output_dir, exp.reldir)}"
)
return Export(exit_code=0)
def test_add_prefix(self) -> None:
digest = self.request_single_product(
Digest,
CreateDigest(
(
FileContent(path="main.py", content=b'print("from main")'),
FileContent(path="subdir/sub.py", content=b'print("from sub")'),
)
),
)
# Two components.
output_digest = self.request_single_product(
Digest, AddPrefix(digest, "outer_dir/middle_dir")
)
snapshot = self.request_single_product(Snapshot, output_digest)
assert sorted(snapshot.files) == [
"outer_dir/middle_dir/main.py",
"outer_dir/middle_dir/subdir/sub.py",
]
assert sorted(snapshot.dirs) == [
"outer_dir",
"outer_dir/middle_dir",
"outer_dir/middle_dir/subdir",
]
# Empty.
output_digest = self.request_single_product(Digest, AddPrefix(digest, ""))
assert digest == output_digest
# Illegal.
with self.assertRaisesRegex(
Exception, r"Cannot add component .*ParentDir.* of path prefix `../something`."
):
self.request_single_product(Digest, AddPrefix(digest, "../something"))
async def bsp_scala_compile_request(
request: ScalaBSPCompileFieldSet,
classpath_entry_request: ClasspathEntryRequestFactory,
) -> BSPCompileResult:
coarsened_targets = await Get(CoarsenedTargets,
Addresses([request.source.address]))
assert len(coarsened_targets) == 1
coarsened_target = coarsened_targets[0]
resolve = await Get(CoursierResolveKey,
CoarsenedTargets([coarsened_target]))
result = await Get(
FallibleClasspathEntry,
ClasspathEntryRequest,
classpath_entry_request.for_targets(component=coarsened_target,
resolve=resolve),
)
_logger.info(f"scala compile result = {result}")
output_digest = EMPTY_DIGEST
if result.exit_code == 0 and result.output:
entries = await Get(DigestEntries, Digest, result.output.digest)
new_entires = [
dataclasses.replace(entry, path=os.path.basename(entry.path))
for entry in entries
]
flat_digest = await Get(Digest, CreateDigest(new_entires))
output_digest = await Get(
Digest, AddPrefix(flat_digest, f"jvm/resolves/{resolve.name}/lib"))
return BSPCompileResult(
status=StatusCode.ERROR if result.exit_code != 0 else StatusCode.OK,
output_digest=output_digest,
)
async def materialize_scala_runtime_jars(
request: MaterializeScalaRuntimeJarsRequest,
) -> MaterializeScalaRuntimeJarsResult:
tool_classpath = await Get(
ToolClasspath,
ToolClasspathRequest(
artifact_requirements=ArtifactRequirements.from_coordinates([
Coordinate(
group="org.scala-lang",
artifact="scala-compiler",
version=request.scala_version,
),
Coordinate(
group="org.scala-lang",
artifact="scala-library",
version=request.scala_version,
),
]), ),
)
materialized_classpath_digest = await Get(
Digest,
AddPrefix(tool_classpath.content.digest,
f"jvm/scala-runtime/{request.scala_version}"),
)
materialized_classpath = await Get(Snapshot, Digest,
materialized_classpath_digest)
return MaterializeScalaRuntimeJarsResult(materialized_classpath)
async def package_go_binary(field_set: GoBinaryFieldSet) -> BuiltPackage:
main_pkg = await Get(GoBinaryMainPackage, GoBinaryMainPackageRequest(field_set.main))
built_package = await Get(
BuiltGoPackage, BuildGoPackageTargetRequest(main_pkg.address, is_main=True)
)
main_pkg_a_file_path = built_package.import_paths_to_pkg_a_files["main"]
import_config = await Get(
ImportConfig, ImportConfigRequest(built_package.import_paths_to_pkg_a_files)
)
input_digest = await Get(Digest, MergeDigests([built_package.digest, import_config.digest]))
output_filename = PurePath(field_set.output_path.value_or_default(file_ending=None))
binary = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=input_digest,
archives=(main_pkg_a_file_path,),
import_config_path=import_config.CONFIG_PATH,
output_filename=f"./{output_filename.name}",
description=f"Link Go binary for {field_set.address}",
),
)
renamed_output_digest = await Get(Digest, AddPrefix(binary.digest, str(output_filename.parent)))
artifact = BuiltPackageArtifact(relpath=str(output_filename))
return BuiltPackage(renamed_output_digest, (artifact,))
async def analyze_java_source_dependencies(
processor_classfiles: JavaParserCompiledClassfiles,
jdk: InternalJdk,
request: JavaSourceDependencyAnalysisRequest,
) -> FallibleJavaSourceDependencyAnalysisResult:
source_files = request.source_files
if len(source_files.files) > 1:
raise ValueError(
f"parse_java_package expects sources with exactly 1 source file, but found {len(source_files.files)}."
)
elif len(source_files.files) == 0:
raise ValueError(
"parse_java_package expects sources with exactly 1 source file, but found none."
)
source_prefix = "__source_to_analyze"
source_path = os.path.join(source_prefix, source_files.files[0])
processorcp_relpath = "__processorcp"
toolcp_relpath = "__toolcp"
parser_lockfile_request = await Get(GenerateJvmLockfileFromTool,
JavaParserToolLockfileSentinel())
tool_classpath, prefixed_source_files_digest = await MultiGet(
Get(
ToolClasspath,
ToolClasspathRequest(lockfile=parser_lockfile_request),
),
Get(Digest, AddPrefix(source_files.snapshot.digest, source_prefix)),
)
extra_immutable_input_digests = {
toolcp_relpath: tool_classpath.digest,
processorcp_relpath: processor_classfiles.digest,
}
analysis_output_path = "__source_analysis.json"
process_result = await Get(
FallibleProcessResult,
JvmProcess(
jdk=jdk,
classpath_entries=[
*tool_classpath.classpath_entries(toolcp_relpath),
processorcp_relpath,
],
argv=[
"org.pantsbuild.javaparser.PantsJavaParserLauncher",
analysis_output_path,
source_path,
],
input_digest=prefixed_source_files_digest,
extra_immutable_input_digests=extra_immutable_input_digests,
output_files=(analysis_output_path, ),
extra_nailgun_keys=extra_immutable_input_digests,
description=f"Analyzing {source_files.files[0]}",
level=LogLevel.DEBUG,
),
)
return FallibleJavaSourceDependencyAnalysisResult(
process_result=process_result)
async def generate_import_config(
request: GatherImportsRequest, stdlib_imports: GoStdLibImports, goroot: GoRoot
) -> GatheredImports:
import_config_digests: dict[str, tuple[str, Digest]] = {}
for pkg in request.packages:
fp = pkg.object_digest.fingerprint
prefixed_digest = await Get(Digest, AddPrefix(pkg.object_digest, f"__pkgs__/{fp}"))
import_config_digests[pkg.import_path] = (fp, prefixed_digest)
pkg_digests: OrderedSet[Digest] = OrderedSet()
import_config = ["# import config"]
for import_path, (fp, digest) in import_config_digests.items():
pkg_digests.add(digest)
import_config.append(f"packagefile {import_path}=__pkgs__/{fp}/__pkg__.a")
if request.include_stdlib:
pkg_digests.add(goroot.digest)
import_config.extend(
f"packagefile {import_path}={os.path.normpath(static_file_path)}"
for import_path, static_file_path in stdlib_imports.items()
)
import_config_content = "\n".join(import_config).encode("utf-8")
import_config_digest = await Get(
Digest, CreateDigest([FileContent("./importcfg", import_config_content)])
)
pkg_digests.add(import_config_digest)
digest = await Get(Digest, MergeDigests(pkg_digests))
return GatheredImports(digest=digest)
async def generate_python_from_thrift(
request: GeneratePythonFromThriftRequest,
thrift_python: ThriftPythonSubsystem,
) -> GeneratedSources:
result = await Get(
GeneratedThriftSources,
GenerateThriftSourcesRequest(
thrift_source_field=request.protocol_target[ThriftSourceField],
lang_id="py",
lang_options=thrift_python.gen_options,
lang_name="Python",
),
)
# We must add back the source root for Python imports to work properly. Note that the file
# paths will be different depending on whether `namespace py` was used. See the tests for
# examples.
source_root = await Get(
SourceRoot, SourceRootRequest, SourceRootRequest.for_target(request.protocol_target)
)
source_root_restored = (
await Get(Snapshot, AddPrefix(result.snapshot.digest, source_root.path))
if source_root.path != "."
else await Get(Snapshot, Digest, result.snapshot.digest)
)
return GeneratedSources(source_root_restored)
async def relocate_files(
request: RelocateFilesViaCodegenRequest) -> GeneratedSources:
# Unlike normal codegen, we operate the on the sources of the `files_targets` field, not the
# `sources` of the original `relocated_sources` target.
# TODO(#13086): Because we're using `Targets` instead of `UnexpandedTargets`, the
# `files` target generator gets replaced by its generated `file` targets. That replacement is
# necessary because we only hydrate sources for `FileSourcesField`, which is only for the
# `file` target. That's really subtle!
original_file_targets = await Get(
Targets,
UnparsedAddressInputs,
request.protocol_target.get(
RelocatedFilesOriginalTargetsField).to_unparsed_address_inputs(),
)
original_files_sources = await MultiGet(
Get(
HydratedSources,
HydrateSourcesRequest(tgt.get(SourcesField),
for_sources_types=(FileSourceField, )),
) for tgt in original_file_targets)
snapshot = await Get(
Snapshot,
MergeDigests(sources.snapshot.digest
for sources in original_files_sources))
src_val = request.protocol_target.get(RelocatedFilesSrcField).value
dest_val = request.protocol_target.get(RelocatedFilesDestField).value
if src_val:
snapshot = await Get(Snapshot, RemovePrefix(snapshot.digest, src_val))
if dest_val:
snapshot = await Get(Snapshot, AddPrefix(snapshot.digest, dest_val))
return GeneratedSources(snapshot)
async def _jvm_bsp_resources(
request: BSPResourcesRequest,
build_root: BuildRoot,
) -> BSPResourcesResult:
"""Generically handles a BSPResourcesRequest (subclass).
This is a `@rule_helper` rather than a `@rule` for the same reason as `_jvm_bsp_compile`.
"""
coarsened_targets = await Get(
CoarsenedTargets, Addresses([fs.address for fs in request.field_sets]))
source_files = await Get(
StrippedSourceFiles,
SourceFilesRequest(
[tgt.get(SourcesField) for tgt in coarsened_targets.closure()],
for_sources_types=(ResourceSourceField, ),
enable_codegen=True,
),
)
rel_resources_dir = _jvm_resources_directory(
request.bsp_target.bsp_target_id)
output_digest = await Get(
Digest,
AddPrefix(source_files.snapshot.digest, rel_resources_dir),
)
return BSPResourcesResult(
resources=(
# NB: IntelliJ requires that directory URIs end in slashes.
build_root.pathlib_path.joinpath(
".pants.d/bsp", rel_resources_dir).as_uri() + "/", ),
output_digest=output_digest,
)
async def relocate_files(request: RelocateFilesViaCodegenRequest) -> GeneratedSources:
# Unlike normal codegen, we operate the on the sources of the `files_targets` field, not the
# `sources` of the original `relocated_sources` target.
# TODO(#10915): using `await Get(Addresses, UnparsedAddressInputs)` causes a graph failure.
original_files_targets = await MultiGet(
Get(
WrappedTarget,
AddressInput,
AddressInput.parse(v, relative_to=request.protocol_target.address.spec_path),
)
for v in (
request.protocol_target.get(RelocatedFilesOriginalTargets)
.to_unparsed_address_inputs()
.values
)
)
original_files_sources = await MultiGet(
Get(HydratedSources, HydrateSourcesRequest(wrapped_tgt.target.get(Sources)))
for wrapped_tgt in original_files_targets
)
snapshot = await Get(
Snapshot, MergeDigests(sources.snapshot.digest for sources in original_files_sources)
)
src_val = request.protocol_target.get(RelocatedFilesSrcField).value
dest_val = request.protocol_target.get(RelocatedFilesDestField).value
if src_val:
snapshot = await Get(Snapshot, RemovePrefix(snapshot.digest, src_val))
if dest_val:
snapshot = await Get(Snapshot, AddPrefix(snapshot.digest, dest_val))
return GeneratedSources(snapshot)
async def handle_bsp_scalac_options_request(
request: HandleScalacOptionsRequest,
build_root: BuildRoot,
workspace: Workspace,
) -> HandleScalacOptionsResult:
targets = await Get(Targets, BuildTargetIdentifier, request.bsp_target_id)
thirdparty_modules = await Get(
ThirdpartyModules,
ThirdpartyModulesRequest(Addresses(tgt.address for tgt in targets)))
resolve = thirdparty_modules.resolve
resolve_digest = await Get(
Digest,
AddPrefix(thirdparty_modules.merged_digest,
f"jvm/resolves/{resolve.name}/lib"))
workspace.write_digest(resolve_digest, path_prefix=".pants.d/bsp")
classpath = tuple(
build_root.pathlib_path.joinpath(
f".pants.d/bsp/jvm/resolves/{resolve.name}/lib/{filename}").as_uri(
) for cp_entry in thirdparty_modules.entries.values()
for filename in cp_entry.filenames)
return HandleScalacOptionsResult(
ScalacOptionsItem(
target=request.bsp_target_id,
options=(),
classpath=classpath,
class_directory=build_root.pathlib_path.joinpath(
f".pants.d/bsp/{jvm_classes_directory(request.bsp_target_id)}"
).as_uri(),
))
async def materialize_classpath(
request: MaterializedClasspathRequest) -> MaterializedClasspath:
"""Resolve, fetch, and merge various classpath types to a single `Digest` and metadata."""
artifact_requirements_lockfiles = await MultiGet(
Get(CoursierResolvedLockfile, ArtifactRequirements,
artifact_requirements)
for artifact_requirements in request.artifact_requirements)
lockfile_and_requirements_classpath_entries = await MultiGet(
Get(
ResolvedClasspathEntries,
CoursierResolvedLockfile,
lockfile,
)
for lockfile in (*request.lockfiles, *artifact_requirements_lockfiles))
all_classpath_entries = (
*lockfile_and_requirements_classpath_entries,
*request.resolved_classpaths,
)
merged_digest = await Get(
Digest,
MergeDigests(classpath_entry.digest
for classpath_entries in all_classpath_entries
for classpath_entry in classpath_entries),
)
if request.prefix is not None:
merged_digest = await Get(Digest,
AddPrefix(merged_digest, request.prefix))
file_names = tuple(classpath_entry.file_name
for classpath_entries in all_classpath_entries
for classpath_entry in classpath_entries)
return MaterializedClasspath(prefix=request.prefix,
digest=merged_digest,
file_names=file_names)
async def classpath(
coarsened_targets: CoarsenedTargets,
union_membership: UnionMembership,
) -> Classpath:
targets = Targets(t for ct in coarsened_targets.closure()
for t in ct.members)
resolve = await Get(CoursierResolveKey, Targets, targets)
transitive_classpath_entries = await MultiGet(
Get(
ClasspathEntry,
ClasspathEntryRequest,
ClasspathEntryRequest.for_targets(
union_membership, component=t, resolve=resolve),
) for t in coarsened_targets.closure())
merged_transitive_classpath_entries_digest = await Get(
Digest,
MergeDigests(classfiles.digest
for classfiles in transitive_classpath_entries))
return Classpath(await Get(
Snapshot,
AddPrefix(merged_transitive_classpath_entries_digest,
_USERCP_RELPATH)))
async def generate_java_from_wsdl(
request: GenerateJavaFromWsdlRequest) -> GeneratedSources:
sources = await Get(
HydratedSources,
HydrateSourcesRequest(request.protocol_target[WsdlSourceField]))
target_package = request.protocol_target[JavaPackageField].value
compile_results = await MultiGet(
Get(
CompiledWsdlSource,
CompileWsdlSourceRequest(
sources.snapshot.digest,
path=path,
module=request.protocol_target[JavaModuleField].value,
package=target_package,
),
) for path in sources.snapshot.files)
merged_output_digests, source_root = await MultiGet(
Get(Digest, MergeDigests([r.output_digest for r in compile_results])),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(merged_output_digests, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, merged_output_digests))
return GeneratedSources(source_root_restored)
async def materialize_classpath(
request: MaterializedClasspathRequest) -> MaterializedClasspath:
"""Resolve, fetch, and merge various classpath types to a single `Digest` and metadata."""
artifact_requirements_lockfiles = await MultiGet(
Get(CoursierResolvedLockfile, ArtifactRequirements,
artifact_requirements)
for artifact_requirements in request.artifact_requirements)
lockfile_and_requirements_classpath_entries = await MultiGet(
Get(
ResolvedClasspathEntries,
CoursierResolvedLockfile,
lockfile,
)
for lockfile in (*request.lockfiles, *artifact_requirements_lockfiles))
merged_snapshot = await Get(
Snapshot,
MergeDigests(classpath_entry.digest for classpath_entries in
lockfile_and_requirements_classpath_entries
for classpath_entry in classpath_entries),
)
if request.prefix is not None:
merged_snapshot = await Get(
Snapshot, AddPrefix(merged_snapshot.digest, request.prefix))
return MaterializedClasspath(content=merged_snapshot)
async def merge_coverage_data(
data_collection: PytestCoverageDataCollection,
coverage_setup: CoverageSetup) -> MergedCoverageData:
if len(data_collection) == 1:
return MergedCoverageData(data_collection[0].digest)
# We prefix each .coverage file with its corresponding address to avoid collisions.
coverage_digests = await MultiGet(
Get(Digest, AddPrefix(data.digest, prefix=data.address.path_safe_spec))
for data in data_collection)
input_digest = await Get(
Digest, MergeDigests((*coverage_digests, coverage_setup.pex.digest)))
prefixes = sorted(f"{data.address.path_safe_spec}/.coverage"
for data in data_collection)
result = await Get(
ProcessResult,
PexProcess(
coverage_setup.pex,
argv=("combine", *prefixes),
input_digest=input_digest,
output_files=(".coverage", ),
description=f"Merge {len(prefixes)} Pytest coverage reports.",
level=LogLevel.DEBUG,
),
)
return MergedCoverageData(result.output_digest)
async def package_python_dist(
field_set: PythonDistributionFieldSet,
python_setup: PythonSetup,
) -> BuiltPackage:
transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address]))
exported_target = ExportedTarget(transitive_targets.roots[0])
interpreter_constraints = PexInterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup
)
chroot = await Get(
SetupPyChroot,
SetupPyChrootRequest(exported_target, py2=interpreter_constraints.includes_python2()),
)
# If commands were provided, run setup.py with them; Otherwise just dump chroots.
commands = exported_target.target.get(SetupPyCommandsField).value or ()
if commands:
validate_commands(commands)
setup_py_result = await Get(
RunSetupPyResult,
RunSetupPyRequest(exported_target, interpreter_constraints, chroot, commands),
)
dist_snapshot = await Get(Snapshot, Digest, setup_py_result.output)
return BuiltPackage(
setup_py_result.output,
tuple(BuiltPackageArtifact(path) for path in dist_snapshot.files),
)
else:
dirname = f"{chroot.setup_kwargs.name}-{chroot.setup_kwargs.version}"
rel_chroot = await Get(Digest, AddPrefix(chroot.digest, dirname))
return BuiltPackage(rel_chroot, (BuiltPackageArtifact(dirname),))
async def _jvm_bsp_compile(
request: BSPCompileRequest,
classpath_entry_request: ClasspathEntryRequestFactory
) -> BSPCompileResult:
"""Generically handles a BSPCompileRequest (subclass).
This is a `@rule_helper` rather than a `@rule`, because BSP backends like `java` and `scala`
independently declare their `BSPCompileRequest` union members. We can't register a single shared
`BSPCompileRequest` @union member for all JVM because their FieldSets are also declared via
@unions, and we can't forward the implementation of a @union to another the way we might with
an abstract class.
"""
coarsened_targets = await Get(
CoarsenedTargets, Addresses([fs.address for fs in request.field_sets]))
resolve = await Get(CoursierResolveKey, CoarsenedTargets,
coarsened_targets)
# TODO: We include the (non-3rdparty) transitive dependencies here, because each project
# currently only has a single BuildTarget. This has the effect of including `resources` targets,
# which are referenced by BuildTargets (via `buildTarget/resources`), rather than necessarily
# being owned by any particular BuildTarget.
#
# To resolve #15051, this will no longer be transitive, and so `resources` will need to be
# attached-to/referenced-by nearby BuildTarget(s) instead (most likely: direct dependent(s)).
results = await MultiGet(
Get(
FallibleClasspathEntry,
BSPClasspathEntryRequest(
classpath_entry_request.for_targets(component=coarsened_target,
resolve=resolve),
task_id=request.task_id,
),
) for coarsened_target in coarsened_targets.coarsened_closure()
if not any(
JvmArtifactFieldSet.is_applicable(t)
for t in coarsened_target.members))
entries = FallibleClasspathEntry.if_all_succeeded(results)
if entries is None:
return BSPCompileResult(
status=StatusCode.ERROR,
output_digest=EMPTY_DIGEST,
)
loose_classfiles = await MultiGet(
Get(LooseClassfiles, ClasspathEntry, entry) for entry in entries)
merged_loose_classfiles = await Get(
Digest, MergeDigests(lc.digest for lc in loose_classfiles))
output_digest = await Get(
Digest,
AddPrefix(merged_loose_classfiles,
jvm_classes_directory(request.bsp_target.bsp_target_id)),
)
return BSPCompileResult(
status=StatusCode.OK,
output_digest=output_digest,
)
async def analyze_scala_source_dependencies(
jdk: InternalJdk,
processor_classfiles: ScalaParserCompiledClassfiles,
source_files: SourceFiles,
) -> FallibleScalaSourceDependencyAnalysisResult:
if len(source_files.files) > 1:
raise ValueError(
f"analyze_scala_source_dependencies expects sources with exactly 1 source file, but found {len(source_files.snapshot.files)}."
)
elif len(source_files.files) == 0:
raise ValueError(
"analyze_scala_source_dependencies expects sources with exactly 1 source file, but found none."
)
source_prefix = "__source_to_analyze"
source_path = os.path.join(source_prefix, source_files.files[0])
processorcp_relpath = "__processorcp"
toolcp_relpath = "__toolcp"
(tool_classpath, prefixed_source_files_digest,) = await MultiGet(
Get(
ToolClasspath,
ToolClasspathRequest(artifact_requirements=SCALA_PARSER_ARTIFACT_REQUIREMENTS),
),
Get(Digest, AddPrefix(source_files.snapshot.digest, source_prefix)),
)
extra_immutable_input_digests = {
toolcp_relpath: tool_classpath.digest,
processorcp_relpath: processor_classfiles.digest,
}
analysis_output_path = "__source_analysis.json"
process_result = await Get(
FallibleProcessResult,
JvmProcess(
jdk=jdk,
classpath_entries=[
*tool_classpath.classpath_entries(toolcp_relpath),
processorcp_relpath,
],
argv=[
"org.pantsbuild.backend.scala.dependency_inference.ScalaParser",
analysis_output_path,
source_path,
],
input_digest=prefixed_source_files_digest,
extra_immutable_input_digests=extra_immutable_input_digests,
output_files=(analysis_output_path,),
extra_nailgun_keys=extra_immutable_input_digests,
description=f"Analyzing {source_files.files[0]}",
level=LogLevel.DEBUG,
),
)
return FallibleScalaSourceDependencyAnalysisResult(process_result=process_result)
async def package_python_dist(
field_set: PythonDistributionFieldSet,
python_setup: PythonSetup,
) -> BuiltPackage:
transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address]))
exported_target = ExportedTarget(transitive_targets.roots[0])
dist_tgt = exported_target.target
wheel = dist_tgt.get(WheelField).value
sdist = dist_tgt.get(SDistField).value
if not wheel and not sdist:
raise NoDistTypeSelected(
f"In order to package {dist_tgt.address.spec} at least one of {WheelField.alias!r} or "
f"{SDistField.alias!r} must be `True`."
)
wheel_config_settings = dist_tgt.get(WheelConfigSettingsField).value or FrozenDict()
sdist_config_settings = dist_tgt.get(SDistConfigSettingsField).value or FrozenDict()
interpreter_constraints = InterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup
) or InterpreterConstraints(python_setup.interpreter_constraints)
chroot = await Get(
DistBuildChroot,
DistBuildChrootRequest(
exported_target,
py2=interpreter_constraints.includes_python2(),
),
)
# We prefix the entire chroot, and run with this prefix as the cwd, so that we can capture
# any changes setup made within it without also capturing other artifacts of the pex
# process invocation.
chroot_prefix = "chroot"
working_directory = os.path.join(chroot_prefix, chroot.working_directory)
prefixed_chroot = await Get(Digest, AddPrefix(chroot.digest, chroot_prefix))
build_system = await Get(BuildSystem, BuildSystemRequest(prefixed_chroot, working_directory))
setup_py_result = await Get(
DistBuildResult,
DistBuildRequest(
build_system=build_system,
interpreter_constraints=interpreter_constraints,
build_wheel=wheel,
build_sdist=sdist,
input=prefixed_chroot,
working_directory=working_directory,
target_address_spec=exported_target.target.address.spec,
wheel_config_settings=wheel_config_settings,
sdist_config_settings=sdist_config_settings,
),
)
dist_snapshot = await Get(Snapshot, Digest, setup_py_result.output)
return BuiltPackage(
setup_py_result.output,
tuple(BuiltPackageArtifact(path) for path in dist_snapshot.files),
)
async def materialize_classpath_for_tool(
request: ToolClasspathRequest) -> ToolClasspath:
if request.artifact_requirements:
resolution = await Get(CoursierResolvedLockfile, ArtifactRequirements,
request.artifact_requirements)
else:
lockfile_req = request.lockfile
assert lockfile_req is not None
regen_command = f"`{GenerateLockfilesSubsystem.name} --resolve={lockfile_req.resolve_name}`"
if lockfile_req.read_lockfile_dest == DEFAULT_TOOL_LOCKFILE:
lockfile_bytes = importlib.resources.read_binary(
*lockfile_req.default_lockfile_resource)
resolution = CoursierResolvedLockfile.from_serialized(
lockfile_bytes)
else:
lockfile_snapshot = await Get(
Snapshot, PathGlobs([lockfile_req.read_lockfile_dest]))
if not lockfile_snapshot.files:
raise ValueError(
f"No lockfile found at {lockfile_req.read_lockfile_dest}, which is configured "
f"by the option {lockfile_req.lockfile_option_name}."
f"Run {regen_command} to generate it.")
resolution = await Get(
CoursierResolvedLockfile,
CoursierResolveKey(
name=lockfile_req.resolve_name,
path=lockfile_req.read_lockfile_dest,
digest=lockfile_snapshot.digest,
),
)
# Validate that the lockfile is correct.
lockfile_inputs = await Get(
ArtifactRequirements,
GatherJvmCoordinatesRequest(lockfile_req.artifact_inputs,
lockfile_req.artifact_option_name),
)
if resolution.metadata and not resolution.metadata.is_valid_for(
lockfile_inputs, LockfileContext.TOOL):
raise ValueError(
f"The lockfile {lockfile_req.read_lockfile_dest} (configured by the option "
f"{lockfile_req.lockfile_option_name}) was generated with different requirements "
f"than are currently set via {lockfile_req.artifact_option_name}. Run "
f"{regen_command} to regenerate the lockfile.")
classpath_entries = await Get(ResolvedClasspathEntries,
CoursierResolvedLockfile, resolution)
merged_snapshot = await Get(
Snapshot,
MergeDigests(classpath_entry.digest
for classpath_entry in classpath_entries))
if request.prefix is not None:
merged_snapshot = await Get(
Snapshot, AddPrefix(merged_snapshot.digest, request.prefix))
return ToolClasspath(merged_snapshot)
async def materialize_jvm_plugins(
request: MaterializeJvmPluginsRequest, ) -> MaterializedJvmPlugins:
materialized_plugins = await MultiGet(
Get(MaterializedJvmPlugin, MaterializeJvmPluginRequest(plugin))
for plugin in request.plugins)
plugin_digests = await MultiGet(
Get(Digest, AddPrefix(p.classpath.digest, p.name))
for p in materialized_plugins)
merged_plugins_digest = await Get(Digest, MergeDigests(plugin_digests))
return MaterializedJvmPlugins(merged_plugins_digest, materialized_plugins)
async def export(
console: Console,
targets: Targets,
workspace: Workspace,
union_membership: UnionMembership,
build_root: BuildRoot,
dist_dir: DistDir,
) -> Export:
request_types = cast("Iterable[type[ExportRequest]]",
union_membership.get(ExportRequest))
requests = tuple(request_type(targets) for request_type in request_types)
all_results = await MultiGet(
Get(ExportResults, ExportRequest, request) for request in requests)
flattened_results = [res for results in all_results for res in results]
prefixed_digests = await MultiGet(
Get(Digest, AddPrefix(result.digest, result.reldir))
for result in flattened_results)
output_dir = os.path.join(str(dist_dir.relpath), "export")
merged_digest = await Get(Digest, MergeDigests(prefixed_digests))
dist_digest = await Get(Digest, AddPrefix(merged_digest, output_dir))
workspace.write_digest(dist_digest)
environment = await Get(Environment, EnvironmentRequest(["PATH"]))
for result in flattened_results:
digest_root = os.path.join(build_root.path, output_dir, result.reldir)
for cmd in result.post_processing_cmds:
argv = tuple(
arg.format(digest_root=digest_root) for arg in cmd.argv)
ip = InteractiveProcess(
argv=argv,
env={
"PATH": environment.get("PATH", ""),
**cmd.extra_env
},
run_in_workspace=True,
)
await Effect(InteractiveProcessResult, InteractiveProcess, ip)
console.print_stdout(
f"Wrote {result.description} to {os.path.join(output_dir, result.reldir)}"
)
return Export(exit_code=0)
async def run_python_test(
field_set: PythonTestFieldSet,
test_setup: TestTargetSetup,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
global_options: GlobalOptions,
test_options: TestOptions,
) -> TestResult:
"""Runs pytest for one target."""
add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"]
if test_setup.xml_dir:
test_results_file = f"{field_set.address.path_safe_spec}.xml"
add_opts.extend(
(f"--junitxml={test_results_file}", f"-o junit_family={test_setup.junit_family}",)
)
env = {"PYTEST_ADDOPTS": " ".join(add_opts)}
use_coverage = test_options.values.use_coverage
output_dirs = [".coverage"] if use_coverage else []
if test_setup.xml_dir:
output_dirs.append(test_results_file)
process = test_setup.test_runner_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./{test_setup.test_runner_pex.output_filename}",
pex_args=test_setup.args,
input_digest=test_setup.input_digest,
output_directories=tuple(output_dirs) if output_dirs else None,
description=f"Run Pytest for {field_set.address.reference()}",
timeout_seconds=test_setup.timeout_seconds,
env=env,
)
result = await Get[FallibleProcessResult](Process, process)
coverage_data = None
if use_coverage:
coverage_snapshot = await Get[Snapshot](
SnapshotSubset(result.output_digest, PathGlobs([".coverage"]))
)
coverage_data = PytestCoverageData(field_set.address, coverage_snapshot.digest)
xml_results_digest = None
if test_setup.xml_dir:
xml_results_snapshot = await Get[Snapshot](
SnapshotSubset(result.output_digest, PathGlobs([test_results_file]))
)
xml_results_digest = await Get[Digest](
AddPrefix(xml_results_snapshot.digest, test_setup.xml_dir)
)
return TestResult.from_fallible_process_result(
result, coverage_data=coverage_data, xml_results=xml_results_digest
)
def test_add_prefix(self) -> None:
input_files_content = InputFilesContent((
FileContent(path="main.py", content=b'print("from main")'),
FileContent(path="subdir/sub.py", content=b'print("from sub")'),
))
digest = self.request_single_product(Digest, input_files_content)
output_digest = self.request_single_product(
Digest, AddPrefix(digest, "outer_dir"))
snapshot = self.request_single_product(Snapshot, output_digest)
assert sorted(snapshot.files) == [
"outer_dir/main.py", "outer_dir/subdir/sub.py"
]
assert sorted(snapshot.dirs) == ["outer_dir", "outer_dir/subdir"]
async def run_python_test(field_set: PythonTestFieldSet,
test_subsystem: TestSubsystem,
pytest: PyTest) -> TestResult:
if field_set.is_conftest_or_type_stub():
return TestResult.skip(field_set.address)
setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=False))
result = await Get(FallibleProcessResult, Process, setup.process)
coverage_data = None
if test_subsystem.use_coverage:
coverage_snapshot = await Get(
Snapshot,
DigestSubset(result.output_digest, PathGlobs([".coverage"])))
if coverage_snapshot.files == (".coverage", ):
coverage_data = PytestCoverageData(field_set.address,
coverage_snapshot.digest)
else:
logger.warning(
f"Failed to generate coverage data for {field_set.address}.")
xml_results_snapshot = None
if setup.results_file_name:
xml_results_snapshot = await Get(
Snapshot,
DigestSubset(result.output_digest,
PathGlobs([setup.results_file_name])))
if xml_results_snapshot.files == (setup.results_file_name, ):
xml_results_snapshot = await Get(
Snapshot,
AddPrefix(xml_results_snapshot.digest,
pytest.options.junit_xml_dir),
)
else:
logger.warning(
f"Failed to generate JUnit XML data for {field_set.address}.")
extra_output_snapshot = await Get(
Snapshot,
DigestSubset(result.output_digest,
PathGlobs([f"{_EXTRA_OUTPUT_DIR}/**"])))
extra_output_snapshot = await Get(
Snapshot, RemovePrefix(extra_output_snapshot.digest,
_EXTRA_OUTPUT_DIR))
return TestResult.from_fallible_process_result(
result,
address=field_set.address,
coverage_data=coverage_data,
xml_results=xml_results_snapshot,
extra_output=extra_output_snapshot,
)
async def run_shell_command(
request: GenerateFilesFromShellCommandRequest,
) -> GeneratedSources:
shell_command = request.protocol_target
result = await Get(ProcessResult, ShellCommandProcessRequest(shell_command))
if shell_command[ShellCommandLogOutputField].value:
if result.stdout:
logger.info(result.stdout.decode())
if result.stderr:
logger.warning(result.stderr.decode())
working_directory = shell_command.address.spec_path
output = await Get(Snapshot, AddPrefix(result.output_digest, working_directory))
return GeneratedSources(output)