async def classpath(
coarsened_targets: CoarsenedTargets,
union_membership: UnionMembership,
) -> Classpath:
targets = Targets(t for ct in coarsened_targets.closure()
for t in ct.members)
resolve = await Get(CoursierResolveKey, Targets, targets)
transitive_classpath_entries = await MultiGet(
Get(
ClasspathEntry,
ClasspathEntryRequest,
ClasspathEntryRequest.for_targets(
union_membership, component=t, resolve=resolve),
) for t in coarsened_targets.closure())
merged_transitive_classpath_entries_digest = await Get(
Digest,
MergeDigests(classfiles.digest
for classfiles in transitive_classpath_entries))
return Classpath(await Get(
Snapshot,
AddPrefix(merged_transitive_classpath_entries_digest,
_USERCP_RELPATH)))
async def kotlinc_check(
request: KotlincCheckRequest,
classpath_entry_request: ClasspathEntryRequestFactory,
) -> CheckResults:
coarsened_targets = await Get(
CoarsenedTargets, Addresses(field_set.address for field_set in request.field_sets)
)
# NB: Each root can have an independent resolve, because there is no inherent relation
# between them other than that they were on the commandline together.
resolves = await MultiGet(
Get(CoursierResolveKey, CoarsenedTargets([t])) for t in coarsened_targets
)
results = await MultiGet(
Get(
FallibleClasspathEntry,
ClasspathEntryRequest,
classpath_entry_request.for_targets(component=target, resolve=resolve),
)
for target, resolve in zip(coarsened_targets, resolves)
)
# NB: We don't pass stdout/stderr as it will have already been rendered as streaming.
exit_code = next((result.exit_code for result in results if result.exit_code != 0), 0)
return CheckResults([CheckResult(exit_code, "", "")], checker_name=request.name)
async def coarsened_targets(addresses: Addresses) -> CoarsenedTargets:
dependency_mapping = await Get(
_DependencyMapping,
_DependencyMappingRequest(
# NB: We set include_special_cased_deps=True because although computing CoarsenedTargets
# requires a transitive graph walk (to ensure that all cycles are actually detected),
# the resulting CoarsenedTargets instance is not itself transitive: everything not directly
# involved in a cycle with one of the input Addresses is discarded in the output.
TransitiveTargetsRequest(addresses,
include_special_cased_deps=True),
expanded_targets=False,
),
)
components = native_engine.strongly_connected_components(
list(dependency_mapping.mapping.items()))
addresses_set = set(addresses)
addresses_to_targets = {
t.address: t
for t in
[*dependency_mapping.visited, *dependency_mapping.roots_as_targets]
}
targets = []
for component in components:
if not any(component_address in addresses_set
for component_address in component):
continue
component_set = set(component)
members = tuple(addresses_to_targets[a] for a in component)
dependencies = FrozenOrderedSet([
d for a in component for d in dependency_mapping.mapping[a]
if d not in component_set
])
targets.append(CoarsenedTarget(members, dependencies))
return CoarsenedTargets(targets)
async def bsp_scala_compile_request(
request: ScalaBSPCompileFieldSet,
classpath_entry_request: ClasspathEntryRequestFactory,
) -> BSPCompileResult:
coarsened_targets = await Get(CoarsenedTargets,
Addresses([request.source.address]))
assert len(coarsened_targets) == 1
coarsened_target = coarsened_targets[0]
resolve = await Get(CoursierResolveKey,
CoarsenedTargets([coarsened_target]))
result = await Get(
FallibleClasspathEntry,
ClasspathEntryRequest,
classpath_entry_request.for_targets(component=coarsened_target,
resolve=resolve),
)
_logger.info(f"scala compile result = {result}")
output_digest = EMPTY_DIGEST
if result.exit_code == 0 and result.output:
entries = await Get(DigestEntries, Digest, result.output.digest)
new_entires = [
dataclasses.replace(entry, path=os.path.basename(entry.path))
for entry in entries
]
flat_digest = await Get(Digest, CreateDigest(new_entires))
output_digest = await Get(
Digest, AddPrefix(flat_digest, f"jvm/resolves/{resolve.name}/lib"))
return BSPCompileResult(
status=StatusCode.ERROR if result.exit_code != 0 else StatusCode.OK,
output_digest=output_digest,
)
async def coarsened_targets(addresses: Addresses) -> CoarsenedTargets:
dependency_mapping = await Get(
_DependencyMapping,
_DependencyMappingRequest(
# NB: We set include_special_cased_deps=True because although computing CoarsenedTargets
# requires a transitive graph walk (to ensure that all cycles are actually detected),
# the resulting CoarsenedTargets instance is not itself transitive: everything not directly
# involved in a cycle with one of the input Addresses is discarded in the output.
TransitiveTargetsRequest(addresses,
include_special_cased_deps=True),
expanded_targets=False,
),
)
addresses_to_targets = {
t.address: t
for t in
[*dependency_mapping.visited, *dependency_mapping.roots_as_targets]
}
# Because this is Tarjan's SCC (TODO: update signature to guarantee), components are returned
# in reverse topological order. We can thus assume when building the structure shared
# `CoarsenedTarget` instances that each instance will already have had its dependencies
# constructed.
components = native_engine.strongly_connected_components(
list(dependency_mapping.mapping.items()))
coarsened_targets: dict[Address, CoarsenedTarget] = {}
root_coarsened_targets = []
root_addresses_set = set(addresses)
for component in components:
component = sorted(component)
component_set = set(component)
# For each member of the component, include the CoarsenedTarget for each of its external
# dependencies.
coarsened_target = CoarsenedTarget(
(addresses_to_targets[a] for a in component),
(coarsened_targets[d] for a in component
for d in dependency_mapping.mapping[a] if d not in component_set),
)
# Add to the coarsened_targets mapping under each of the component's Addresses.
for address in component:
coarsened_targets[address] = coarsened_target
# If any of the input Addresses was a member of this component, it is a root.
if component_set & root_addresses_set:
root_coarsened_targets.append(coarsened_target)
return CoarsenedTargets(tuple(root_coarsened_targets))
async def coarsened_targets(
request: CoarsenedTargetsRequest) -> CoarsenedTargets:
dependency_mapping = await Get(
_DependencyMapping,
_DependencyMappingRequest(
TransitiveTargetsRequest(
request.roots,
include_special_cased_deps=request.include_special_cased_deps),
expanded_targets=request.expanded_targets,
),
)
addresses_to_targets = {
t.address: t
for t in
[*dependency_mapping.visited, *dependency_mapping.roots_as_targets]
}
# Because this is Tarjan's SCC (TODO: update signature to guarantee), components are returned
# in reverse topological order. We can thus assume when building the structure shared
# `CoarsenedTarget` instances that each instance will already have had its dependencies
# constructed.
components = native_engine.strongly_connected_components(
list(dependency_mapping.mapping.items()))
coarsened_targets: dict[Address, CoarsenedTarget] = {}
root_coarsened_targets = []
root_addresses_set = set(request.roots)
for component in components:
component = sorted(component)
component_set = set(component)
# For each member of the component, include the CoarsenedTarget for each of its external
# dependencies.
coarsened_target = CoarsenedTarget(
(addresses_to_targets[a] for a in component),
(coarsened_targets[d] for a in component
for d in dependency_mapping.mapping[a] if d not in component_set),
)
# Add to the coarsened_targets mapping under each of the component's Addresses.
for address in component:
coarsened_targets[address] = coarsened_target
# If any of the input Addresses was a member of this component, it is a root.
if component_set & root_addresses_set:
root_coarsened_targets.append(coarsened_target)
return CoarsenedTargets(tuple(root_coarsened_targets))
async def pylint_determine_partitions(
request: PylintRequest, python_setup: PythonSetup,
first_party_plugins: PylintFirstPartyPlugins) -> PylintPartitions:
resolve_and_interpreter_constraints_to_coarsened_targets = (
await partition._by_interpreter_constraints_and_resolve(
request.field_sets, python_setup))
first_party_ics = InterpreterConstraints.create_from_compatibility_fields(
first_party_plugins.interpreter_constraints_fields, python_setup)
return PylintPartitions(
PylintPartition(
FrozenOrderedSet(roots),
FrozenOrderedSet(CoarsenedTargets(root_cts).closure()),
resolve if len(python_setup.resolves) > 1 else None,
InterpreterConstraints.merge((interpreter_constraints,
first_party_ics)),
) for (resolve, interpreter_constraints), (roots, root_cts) in sorted(
resolve_and_interpreter_constraints_to_coarsened_targets.items()))
async def mypy_determine_partitions(
request: MyPyRequest, mypy: MyPy, python_setup: PythonSetup
) -> MyPyPartitions:
resolve_and_interpreter_constraints_to_coarsened_targets = (
await partition._by_interpreter_constraints_and_resolve(request.field_sets, python_setup)
)
return MyPyPartitions(
MyPyPartition(
FrozenOrderedSet(roots),
FrozenOrderedSet(CoarsenedTargets(root_cts).closure()),
resolve if len(python_setup.resolves) > 1 else None,
interpreter_constraints or mypy.interpreter_constraints,
)
for (resolve, interpreter_constraints), (roots, root_cts) in sorted(
resolve_and_interpreter_constraints_to_coarsened_targets.items()
)
)
async def select_coursier_resolve_for_targets(
coarsened_targets: CoarsenedTargets,
jvm: JvmSubsystem) -> CoursierResolveKey:
"""Selects and validates (transitively) a single resolve for a set of roots in a compile graph.
In most cases, a `CoursierResolveKey` should be requested for a single `CoarsenedTarget` root,
which avoids coupling un-related roots unnecessarily. But in other cases, a single compatible
resolve is required for multiple roots (such as when running a `repl` over unrelated code), and
in that case there might be multiple CoarsenedTargets.
"""
targets = list(coarsened_targets.closure())
# Find a single resolve that is compatible with all targets in the closure.
compatible_resolve: str | None = None
all_compatible = True
for tgt in targets:
if not tgt.has_field(JvmResolveField):
continue
resolve = tgt[JvmResolveField].normalized_value(jvm)
if compatible_resolve is None:
compatible_resolve = resolve
elif resolve != compatible_resolve:
all_compatible = False
if not all_compatible:
raise NoCompatibleResolve(
jvm, "The selected targets did not have a resolve in common",
targets)
resolve = compatible_resolve or jvm.default_resolve
# Load the resolve.
resolve_path = jvm.resolves[resolve]
lockfile_source = PathGlobs(
[resolve_path],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=f"The resolve `{resolve}` from `[jvm].resolves`",
)
resolve_digest = await Get(Digest, PathGlobs, lockfile_source)
return CoursierResolveKey(resolve, resolve_path, resolve_digest)