async def classpath( coarsened_targets: CoarsenedTargets, union_membership: UnionMembership, ) -> Classpath: targets = Targets(t for ct in coarsened_targets.closure() for t in ct.members) resolve = await Get(CoursierResolveKey, Targets, targets) transitive_classpath_entries = await MultiGet( Get( ClasspathEntry, ClasspathEntryRequest, ClasspathEntryRequest.for_targets( union_membership, component=t, resolve=resolve), ) for t in coarsened_targets.closure()) merged_transitive_classpath_entries_digest = await Get( Digest, MergeDigests(classfiles.digest for classfiles in transitive_classpath_entries)) return Classpath(await Get( Snapshot, AddPrefix(merged_transitive_classpath_entries_digest, _USERCP_RELPATH)))
async def kotlinc_check( request: KotlincCheckRequest, classpath_entry_request: ClasspathEntryRequestFactory, ) -> CheckResults: coarsened_targets = await Get( CoarsenedTargets, Addresses(field_set.address for field_set in request.field_sets) ) # NB: Each root can have an independent resolve, because there is no inherent relation # between them other than that they were on the commandline together. resolves = await MultiGet( Get(CoursierResolveKey, CoarsenedTargets([t])) for t in coarsened_targets ) results = await MultiGet( Get( FallibleClasspathEntry, ClasspathEntryRequest, classpath_entry_request.for_targets(component=target, resolve=resolve), ) for target, resolve in zip(coarsened_targets, resolves) ) # NB: We don't pass stdout/stderr as it will have already been rendered as streaming. exit_code = next((result.exit_code for result in results if result.exit_code != 0), 0) return CheckResults([CheckResult(exit_code, "", "")], checker_name=request.name)
async def coarsened_targets(addresses: Addresses) -> CoarsenedTargets: dependency_mapping = await Get( _DependencyMapping, _DependencyMappingRequest( # NB: We set include_special_cased_deps=True because although computing CoarsenedTargets # requires a transitive graph walk (to ensure that all cycles are actually detected), # the resulting CoarsenedTargets instance is not itself transitive: everything not directly # involved in a cycle with one of the input Addresses is discarded in the output. TransitiveTargetsRequest(addresses, include_special_cased_deps=True), expanded_targets=False, ), ) components = native_engine.strongly_connected_components( list(dependency_mapping.mapping.items())) addresses_set = set(addresses) addresses_to_targets = { t.address: t for t in [*dependency_mapping.visited, *dependency_mapping.roots_as_targets] } targets = [] for component in components: if not any(component_address in addresses_set for component_address in component): continue component_set = set(component) members = tuple(addresses_to_targets[a] for a in component) dependencies = FrozenOrderedSet([ d for a in component for d in dependency_mapping.mapping[a] if d not in component_set ]) targets.append(CoarsenedTarget(members, dependencies)) return CoarsenedTargets(targets)
async def bsp_scala_compile_request( request: ScalaBSPCompileFieldSet, classpath_entry_request: ClasspathEntryRequestFactory, ) -> BSPCompileResult: coarsened_targets = await Get(CoarsenedTargets, Addresses([request.source.address])) assert len(coarsened_targets) == 1 coarsened_target = coarsened_targets[0] resolve = await Get(CoursierResolveKey, CoarsenedTargets([coarsened_target])) result = await Get( FallibleClasspathEntry, ClasspathEntryRequest, classpath_entry_request.for_targets(component=coarsened_target, resolve=resolve), ) _logger.info(f"scala compile result = {result}") output_digest = EMPTY_DIGEST if result.exit_code == 0 and result.output: entries = await Get(DigestEntries, Digest, result.output.digest) new_entires = [ dataclasses.replace(entry, path=os.path.basename(entry.path)) for entry in entries ] flat_digest = await Get(Digest, CreateDigest(new_entires)) output_digest = await Get( Digest, AddPrefix(flat_digest, f"jvm/resolves/{resolve.name}/lib")) return BSPCompileResult( status=StatusCode.ERROR if result.exit_code != 0 else StatusCode.OK, output_digest=output_digest, )
async def coarsened_targets(addresses: Addresses) -> CoarsenedTargets: dependency_mapping = await Get( _DependencyMapping, _DependencyMappingRequest( # NB: We set include_special_cased_deps=True because although computing CoarsenedTargets # requires a transitive graph walk (to ensure that all cycles are actually detected), # the resulting CoarsenedTargets instance is not itself transitive: everything not directly # involved in a cycle with one of the input Addresses is discarded in the output. TransitiveTargetsRequest(addresses, include_special_cased_deps=True), expanded_targets=False, ), ) addresses_to_targets = { t.address: t for t in [*dependency_mapping.visited, *dependency_mapping.roots_as_targets] } # Because this is Tarjan's SCC (TODO: update signature to guarantee), components are returned # in reverse topological order. We can thus assume when building the structure shared # `CoarsenedTarget` instances that each instance will already have had its dependencies # constructed. components = native_engine.strongly_connected_components( list(dependency_mapping.mapping.items())) coarsened_targets: dict[Address, CoarsenedTarget] = {} root_coarsened_targets = [] root_addresses_set = set(addresses) for component in components: component = sorted(component) component_set = set(component) # For each member of the component, include the CoarsenedTarget for each of its external # dependencies. coarsened_target = CoarsenedTarget( (addresses_to_targets[a] for a in component), (coarsened_targets[d] for a in component for d in dependency_mapping.mapping[a] if d not in component_set), ) # Add to the coarsened_targets mapping under each of the component's Addresses. for address in component: coarsened_targets[address] = coarsened_target # If any of the input Addresses was a member of this component, it is a root. if component_set & root_addresses_set: root_coarsened_targets.append(coarsened_target) return CoarsenedTargets(tuple(root_coarsened_targets))
async def coarsened_targets( request: CoarsenedTargetsRequest) -> CoarsenedTargets: dependency_mapping = await Get( _DependencyMapping, _DependencyMappingRequest( TransitiveTargetsRequest( request.roots, include_special_cased_deps=request.include_special_cased_deps), expanded_targets=request.expanded_targets, ), ) addresses_to_targets = { t.address: t for t in [*dependency_mapping.visited, *dependency_mapping.roots_as_targets] } # Because this is Tarjan's SCC (TODO: update signature to guarantee), components are returned # in reverse topological order. We can thus assume when building the structure shared # `CoarsenedTarget` instances that each instance will already have had its dependencies # constructed. components = native_engine.strongly_connected_components( list(dependency_mapping.mapping.items())) coarsened_targets: dict[Address, CoarsenedTarget] = {} root_coarsened_targets = [] root_addresses_set = set(request.roots) for component in components: component = sorted(component) component_set = set(component) # For each member of the component, include the CoarsenedTarget for each of its external # dependencies. coarsened_target = CoarsenedTarget( (addresses_to_targets[a] for a in component), (coarsened_targets[d] for a in component for d in dependency_mapping.mapping[a] if d not in component_set), ) # Add to the coarsened_targets mapping under each of the component's Addresses. for address in component: coarsened_targets[address] = coarsened_target # If any of the input Addresses was a member of this component, it is a root. if component_set & root_addresses_set: root_coarsened_targets.append(coarsened_target) return CoarsenedTargets(tuple(root_coarsened_targets))
async def pylint_determine_partitions( request: PylintRequest, python_setup: PythonSetup, first_party_plugins: PylintFirstPartyPlugins) -> PylintPartitions: resolve_and_interpreter_constraints_to_coarsened_targets = ( await partition._by_interpreter_constraints_and_resolve( request.field_sets, python_setup)) first_party_ics = InterpreterConstraints.create_from_compatibility_fields( first_party_plugins.interpreter_constraints_fields, python_setup) return PylintPartitions( PylintPartition( FrozenOrderedSet(roots), FrozenOrderedSet(CoarsenedTargets(root_cts).closure()), resolve if len(python_setup.resolves) > 1 else None, InterpreterConstraints.merge((interpreter_constraints, first_party_ics)), ) for (resolve, interpreter_constraints), (roots, root_cts) in sorted( resolve_and_interpreter_constraints_to_coarsened_targets.items()))
async def mypy_determine_partitions( request: MyPyRequest, mypy: MyPy, python_setup: PythonSetup ) -> MyPyPartitions: resolve_and_interpreter_constraints_to_coarsened_targets = ( await partition._by_interpreter_constraints_and_resolve(request.field_sets, python_setup) ) return MyPyPartitions( MyPyPartition( FrozenOrderedSet(roots), FrozenOrderedSet(CoarsenedTargets(root_cts).closure()), resolve if len(python_setup.resolves) > 1 else None, interpreter_constraints or mypy.interpreter_constraints, ) for (resolve, interpreter_constraints), (roots, root_cts) in sorted( resolve_and_interpreter_constraints_to_coarsened_targets.items() ) )
async def select_coursier_resolve_for_targets( coarsened_targets: CoarsenedTargets, jvm: JvmSubsystem) -> CoursierResolveKey: """Selects and validates (transitively) a single resolve for a set of roots in a compile graph. In most cases, a `CoursierResolveKey` should be requested for a single `CoarsenedTarget` root, which avoids coupling un-related roots unnecessarily. But in other cases, a single compatible resolve is required for multiple roots (such as when running a `repl` over unrelated code), and in that case there might be multiple CoarsenedTargets. """ targets = list(coarsened_targets.closure()) # Find a single resolve that is compatible with all targets in the closure. compatible_resolve: str | None = None all_compatible = True for tgt in targets: if not tgt.has_field(JvmResolveField): continue resolve = tgt[JvmResolveField].normalized_value(jvm) if compatible_resolve is None: compatible_resolve = resolve elif resolve != compatible_resolve: all_compatible = False if not all_compatible: raise NoCompatibleResolve( jvm, "The selected targets did not have a resolve in common", targets) resolve = compatible_resolve or jvm.default_resolve # Load the resolve. resolve_path = jvm.resolves[resolve] lockfile_source = PathGlobs( [resolve_path], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin=f"The resolve `{resolve}` from `[jvm].resolves`", ) resolve_digest = await Get(Digest, PathGlobs, lockfile_source) return CoursierResolveKey(resolve, resolve_path, resolve_digest)