async def fetch_with_coursier( coursier: Coursier, request: CoursierFetchRequest, ) -> FallibleClasspathEntry: # TODO: Loading this per JvmArtifact. lockfile = await Get(CoursierResolvedLockfile, CoursierResolveKey, request.resolve) # All of the transitive dependencies are exported. # TODO: Expose an option to control whether this exports only the root, direct dependencies, # transitive dependencies, etc. assert len(request.component.members ) == 1, "JvmArtifact does not have dependencies." root_entry, transitive_entries = lockfile.dependencies( request.resolve, Coordinate.from_jvm_artifact_target(request.component.representative)) classpath_entries = await MultiGet( Get(ClasspathEntry, CoursierLockfileEntry, entry) for entry in (root_entry, *transitive_entries)) exported_digest = await Get( Digest, MergeDigests(cpe.digest for cpe in classpath_entries)) return FallibleClasspathEntry( description=str(request.component), result=CompileResult.SUCCEEDED, output=ClasspathEntry.merge(exported_digest, classpath_entries), exit_code=0, )
async def fetch_kotlinc_plugins( request: KotlincPluginsRequest) -> KotlincPlugins: # Fetch all the artifacts coarsened_targets = await Get( CoarsenedTargets, Addresses(target.address for target in request.artifacts)) fallible_artifacts = await MultiGet( Get( FallibleClasspathEntry, CoursierFetchRequest(ct, resolve=request.resolve), ) for ct in coarsened_targets) artifacts = FallibleClasspathEntry.if_all_succeeded(fallible_artifacts) if artifacts is None: failed = [i for i in fallible_artifacts if i.exit_code != 0] raise Exception(f"Fetching local kotlinc plugins failed: {failed}") entries = list(ClasspathEntry.closure(artifacts)) merged_classpath_digest = await Get( Digest, MergeDigests(entry.digest for entry in entries)) merged = ClasspathEntry.merge(merged_classpath_digest, entries) ids = tuple(_plugin_id(target) for target in request.plugins) plugin_args = FrozenDict({ _plugin_id(plugin): tuple(plugin[KotlincPluginArgsField].value or []) for plugin in request.plugins }) return KotlincPlugins(ids=ids, classpath=merged, plugin_args=plugin_args)
async def fetch_with_coursier( request: CoursierFetchRequest) -> FallibleClasspathEntry: # TODO: Loading this per JvmArtifact. lockfile = await Get(CoursierResolvedLockfile, CoursierResolveKey, request.resolve) requirement = ArtifactRequirement.from_jvm_artifact_target( request.component.representative) if lockfile.metadata and not lockfile.metadata.is_valid_for( [requirement], LockfileContext.USER): raise ValueError( f"Requirement `{requirement.to_coord_arg_str()}` has changed since the lockfile " f"for {request.resolve.path} was generated. Run `{bin_name()} generate-lockfiles` to update your " "lockfile based on the new requirements.") # All of the transitive dependencies are exported. # TODO: Expose an option to control whether this exports only the root, direct dependencies, # transitive dependencies, etc. assert len(request.component.members ) == 1, "JvmArtifact does not have dependencies." root_entry, transitive_entries = lockfile.dependencies( request.resolve, requirement.coordinate, ) classpath_entries = await MultiGet( Get(ClasspathEntry, CoursierLockfileEntry, entry) for entry in (root_entry, *transitive_entries)) exported_digest = await Get( Digest, MergeDigests(cpe.digest for cpe in classpath_entries)) return FallibleClasspathEntry( description=str(request.component), result=CompileResult.SUCCEEDED, output=ClasspathEntry.merge(exported_digest, classpath_entries), exit_code=0, )
async def fetch_plugins(request: ScalaPluginsRequest) -> ScalaPlugins: # Fetch all the artifacts coarsened_targets = await Get( CoarsenedTargets, Addresses(target.address for target in request.artifacts)) fallible_artifacts = await MultiGet( Get( FallibleClasspathEntry, CoursierFetchRequest(ct, resolve=request.resolve), ) for ct in coarsened_targets) artifacts = FallibleClasspathEntry.if_all_succeeded(fallible_artifacts) if artifacts is None: failed = [i for i in fallible_artifacts if i.exit_code != 0] raise Exception(f"Fetching local scala plugins failed: {failed}") merged_classpath_digest = await Get( Digest, MergeDigests(i.digest for i in artifacts)) merged = ClasspathEntry.merge(merged_classpath_digest, artifacts) names = tuple(_plugin_name(target) for target in request.plugins) return ScalaPlugins(names=names, classpath=merged)
async def compile_java_source( bash: BashBinary, javac: JavacSubsystem, zip_binary: ZipBinary, request: CompileJavaSourceRequest, ) -> FallibleClasspathEntry: # Request the component's direct dependency classpath, and additionally any prerequisite. optional_prereq_request = [ *((request.prerequisite, ) if request.prerequisite else ()) ] fallibles = await MultiGet( Get(FallibleClasspathEntries, ClasspathEntryRequests(optional_prereq_request)), Get(FallibleClasspathEntries, ClasspathDependenciesRequest(request)), ) direct_dependency_classpath_entries = FallibleClasspathEntries( itertools.chain(*fallibles)).if_all_succeeded() if direct_dependency_classpath_entries is None: return FallibleClasspathEntry( description=str(request.component), result=CompileResult.DEPENDENCY_FAILED, output=None, exit_code=1, ) # Capture just the `ClasspathEntry` objects that are listed as `export` types by source analysis deps_to_classpath_entries = dict( zip(request.component.dependencies, direct_dependency_classpath_entries or ())) # Re-request inferred dependencies to get a list of export dependency addresses inferred_dependencies = await MultiGet( Get( JavaInferredDependencies, JavaInferredDependenciesAndExportsRequest(tgt[JavaSourceField]), ) for tgt in request.component.members if JavaFieldSet.is_applicable(tgt)) flat_exports = { export for i in inferred_dependencies for export in i.exports } export_classpath_entries = [ classpath_entry for coarsened_target, classpath_entry in deps_to_classpath_entries.items() if any(m.address in flat_exports for m in coarsened_target.members) ] # Then collect the component's sources. component_members_with_sources = tuple(t for t in request.component.members if t.has_field(SourcesField)) component_members_and_source_files = zip( component_members_with_sources, await MultiGet( Get( SourceFiles, SourceFilesRequest( (t.get(SourcesField), ), for_sources_types=(JavaSourceField, ), enable_codegen=True, ), ) for t in component_members_with_sources), ) component_members_and_java_source_files = [ (target, sources) for target, sources in component_members_and_source_files if sources.snapshot.digest != EMPTY_DIGEST ] if not component_members_and_java_source_files: # Is a generator, and so exports all of its direct deps. exported_digest = await Get( Digest, MergeDigests(cpe.digest for cpe in direct_dependency_classpath_entries)) classpath_entry = ClasspathEntry.merge( exported_digest, direct_dependency_classpath_entries) return FallibleClasspathEntry( description=str(request.component), result=CompileResult.SUCCEEDED, output=classpath_entry, exit_code=0, ) dest_dir = "classfiles" dest_dir_digest, jdk = await MultiGet( Get( Digest, CreateDigest([Directory(dest_dir)]), ), Get(JdkEnvironment, JdkRequest, JdkRequest.from_target(request.component)), ) merged_digest = await Get( Digest, MergeDigests(( dest_dir_digest, *(sources.snapshot.digest for _, sources in component_members_and_java_source_files), )), ) usercp = "__cp" user_classpath = Classpath(direct_dependency_classpath_entries, request.resolve) classpath_arg = ":".join( user_classpath.root_immutable_inputs_args(prefix=usercp)) immutable_input_digests = dict( user_classpath.root_immutable_inputs(prefix=usercp)) # Compile. compile_result = await Get( FallibleProcessResult, JvmProcess( jdk=jdk, classpath_entries=[f"{jdk.java_home}/lib/tools.jar"], argv=[ "com.sun.tools.javac.Main", *(("-cp", classpath_arg) if classpath_arg else ()), *javac.args, "-d", dest_dir, *sorted( chain.from_iterable( sources.snapshot.files for _, sources in component_members_and_java_source_files)), ], input_digest=merged_digest, extra_immutable_input_digests=immutable_input_digests, output_directories=(dest_dir, ), description=f"Compile {request.component} with javac", level=LogLevel.DEBUG, ), ) if compile_result.exit_code != 0: return FallibleClasspathEntry.from_fallible_process_result( str(request.component), compile_result, None, ) # Jar. # NB: We jar up the outputs in a separate process because the nailgun runner cannot support # invoking via a `bash` wrapper (since the trailing portion of the command is executed by # the nailgun server). We might be able to resolve this in the future via a Javac wrapper shim. output_snapshot = await Get(Snapshot, Digest, compile_result.output_digest) output_file = compute_output_jar_filename(request.component) output_files: tuple[str, ...] = (output_file, ) if output_snapshot.files: jar_result = await Get( ProcessResult, Process( argv=[ bash.path, "-c", " ".join([ "cd", dest_dir, ";", zip_binary.path, "-r", f"../{output_file}", "." ]), ], input_digest=compile_result.output_digest, output_files=output_files, description=f"Capture outputs of {request.component} for javac", level=LogLevel.TRACE, ), ) jar_output_digest = jar_result.output_digest else: # If there was no output, then do not create a jar file. This may occur, for example, when compiling # a `package-info.java` in a single partition. output_files = () jar_output_digest = EMPTY_DIGEST output_classpath = ClasspathEntry(jar_output_digest, output_files, direct_dependency_classpath_entries) if export_classpath_entries: merged_export_digest = await Get( Digest, MergeDigests((output_classpath.digest, *(i.digest for i in export_classpath_entries))), ) merged_classpath = ClasspathEntry.merge( merged_export_digest, (output_classpath, *export_classpath_entries)) output_classpath = merged_classpath return FallibleClasspathEntry.from_fallible_process_result( str(request.component), compile_result, output_classpath, )
async def compile_java_source( bash: BashBinary, jdk_setup: JdkSetup, zip_binary: ZipBinary, union_membership: UnionMembership, request: CompileJavaSourceRequest, ) -> FallibleClasspathEntry: # Request the component's direct dependency classpath. direct_dependency_classpath_entries = FallibleClasspathEntry.if_all_succeeded( await MultiGet( Get( FallibleClasspathEntry, ClasspathEntryRequest, ClasspathEntryRequest.for_targets(union_membership, component=coarsened_dep, resolve=request.resolve), ) for coarsened_dep in request.component.dependencies)) if direct_dependency_classpath_entries is None: return FallibleClasspathEntry( description=str(request.component), result=CompileResult.DEPENDENCY_FAILED, output=None, exit_code=1, ) # Then collect the component's sources. component_members_with_sources = tuple(t for t in request.component.members if t.has_field(SourcesField)) component_members_and_source_files = zip( component_members_with_sources, await MultiGet( Get( SourceFiles, SourceFilesRequest( (t.get(SourcesField), ), for_sources_types=(JavaSourceField, ), enable_codegen=True, ), ) for t in component_members_with_sources), ) component_members_and_java_source_files = [ (target, sources) for target, sources in component_members_and_source_files if sources.snapshot.digest != EMPTY_DIGEST ] if not component_members_and_java_source_files: # Is a generator, and so exports all of its direct deps. exported_digest = await Get( Digest, MergeDigests(cpe.digest for cpe in direct_dependency_classpath_entries)) classpath_entry = ClasspathEntry.merge( exported_digest, direct_dependency_classpath_entries) return FallibleClasspathEntry( description=str(request.component), result=CompileResult.SUCCEEDED, output=classpath_entry, exit_code=0, ) dest_dir = "classfiles" (merged_direct_dependency_classpath_digest, dest_dir_digest) = await MultiGet( Get( Digest, MergeDigests( classfiles.digest for classfiles in direct_dependency_classpath_entries), ), Get( Digest, CreateDigest([Directory(dest_dir)]), ), ) usercp = "__cp" prefixed_direct_dependency_classpath_digest = await Get( Digest, AddPrefix(merged_direct_dependency_classpath_digest, usercp)) classpath_arg = ClasspathEntry.arg(direct_dependency_classpath_entries, prefix=usercp) merged_digest = await Get( Digest, MergeDigests(( prefixed_direct_dependency_classpath_digest, dest_dir_digest, jdk_setup.digest, *(sources.snapshot.digest for _, sources in component_members_and_java_source_files), )), ) # Compile. compile_result = await Get( FallibleProcessResult, Process( argv=[ *jdk_setup.args(bash, [f"{jdk_setup.java_home}/lib/tools.jar"]), "com.sun.tools.javac.Main", *(("-cp", classpath_arg) if classpath_arg else ()), "-d", dest_dir, *sorted( chain.from_iterable( sources.snapshot.files for _, sources in component_members_and_java_source_files)), ], input_digest=merged_digest, use_nailgun=jdk_setup.digest, append_only_caches=jdk_setup.append_only_caches, env=jdk_setup.env, output_directories=(dest_dir, ), description=f"Compile {request.component} with javac", level=LogLevel.DEBUG, ), ) if compile_result.exit_code != 0: return FallibleClasspathEntry.from_fallible_process_result( str(request.component), compile_result, None, ) # Jar. # NB: We jar up the outputs in a separate process because the nailgun runner cannot support # invoking via a `bash` wrapper (since the trailing portion of the command is executed by # the nailgun server). We might be able to resolve this in the future via a Javac wrapper shim. output_snapshot = await Get(Snapshot, Digest, compile_result.output_digest) output_file = f"{request.component.representative.address.path_safe_spec}.jar" if output_snapshot.files: jar_result = await Get( ProcessResult, Process( argv=[ bash.path, "-c", " ".join([ "cd", dest_dir, ";", zip_binary.path, "-r", f"../{output_file}", "." ]), ], input_digest=compile_result.output_digest, output_files=(output_file, ), description=f"Capture outputs of {request.component} for javac", level=LogLevel.TRACE, ), ) jar_output_digest = jar_result.output_digest else: # If there was no output, then do not create a jar file. This may occur, for example, when compiling # a `package-info.java` in a single partition. jar_output_digest = EMPTY_DIGEST return FallibleClasspathEntry.from_fallible_process_result( str(request.component), compile_result, ClasspathEntry(jar_output_digest, (output_file, ), direct_dependency_classpath_entries), )
async def assemble_resources_jar( zip: ZipBinary, request: JvmResourcesRequest, ) -> FallibleClasspathEntry: # Request the component's direct dependency classpath, and additionally any prerequisite. # Filter out any dependencies that are generated by our current target so that each resource # only appears in a single input JAR. # NOTE: Generated dependencies will have the same dependencies as the current target, so we # don't need to inspect those dependencies. optional_prereq_request = [ *((request.prerequisite, ) if request.prerequisite else ()) ] fallibles = await MultiGet( Get(FallibleClasspathEntries, ClasspathEntryRequests(optional_prereq_request)), Get(FallibleClasspathEntries, ClasspathDependenciesRequest(request, ignore_generated=True)), ) direct_dependency_classpath_entries = FallibleClasspathEntries( itertools.chain(*fallibles)).if_all_succeeded() if direct_dependency_classpath_entries is None: return FallibleClasspathEntry( description=str(request.component), result=CompileResult.DEPENDENCY_FAILED, output=None, exit_code=1, ) source_files = await Get( StrippedSourceFiles, SourceFilesRequest( [tgt.get(SourcesField) for tgt in request.component.members]), ) output_filename = f"{request.component.representative.address.path_safe_spec}.jar" output_files = [output_filename] resources_jar_input_digest = source_files.snapshot.digest resources_jar_result = await Get( ProcessResult, Process( argv=[ zip.path, output_filename, *source_files.snapshot.files, ], description="Build partial JAR containing resources files", input_digest=resources_jar_input_digest, output_files=output_files, ), ) cpe = ClasspathEntry(resources_jar_result.output_digest, output_files, []) merged_cpe_digest = await Get( Digest, MergeDigests( chain((cpe.digest, ), (i.digest for i in direct_dependency_classpath_entries))), ) merged_cpe = ClasspathEntry.merge( digest=merged_cpe_digest, entries=[cpe, *direct_dependency_classpath_entries]) return FallibleClasspathEntry(output_filename, CompileResult.SUCCEEDED, merged_cpe, 0)
async def compile_scala_source( bash: BashBinary, coursier: Coursier, jdk_setup: JdkSetup, scala: ScalaSubsystem, union_membership: UnionMembership, request: CompileScalaSourceRequest, ) -> FallibleClasspathEntry: # Request classpath entries for our direct dependencies. direct_dependency_classpath_entries = FallibleClasspathEntry.if_all_succeeded( await MultiGet( Get( FallibleClasspathEntry, ClasspathEntryRequest, ClasspathEntryRequest.for_targets(union_membership, component=coarsened_dep, resolve=request.resolve), ) for coarsened_dep in request.component.dependencies)) if direct_dependency_classpath_entries is None: return FallibleClasspathEntry( description=str(request.component), result=CompileResult.DEPENDENCY_FAILED, output=None, exit_code=1, ) component_members_with_sources = tuple(t for t in request.component.members if t.has_field(SourcesField)) component_members_and_source_files = zip( component_members_with_sources, await MultiGet( Get( SourceFiles, SourceFilesRequest( (t.get(SourcesField), ), for_sources_types=(ScalaSourceField, ), enable_codegen=True, ), ) for t in component_members_with_sources), ) component_members_and_scala_source_files = [ (target, sources) for target, sources in component_members_and_source_files if sources.snapshot.digest != EMPTY_DIGEST ] if not component_members_and_scala_source_files: # Is a generator, and so exports all of its direct deps. exported_digest = await Get( Digest, MergeDigests(cpe.digest for cpe in direct_dependency_classpath_entries)) classpath_entry = ClasspathEntry.merge( exported_digest, direct_dependency_classpath_entries) return FallibleClasspathEntry( description=str(request.component), result=CompileResult.SUCCEEDED, output=classpath_entry, exit_code=0, ) ( tool_classpath, merged_transitive_dependency_classpath_entries_digest, ) = await MultiGet( Get( MaterializedClasspath, MaterializedClasspathRequest( prefix="__toolcp", artifact_requirements=(ArtifactRequirements([ Coordinate( group="org.scala-lang", artifact="scala-compiler", version=scala.version, ), Coordinate( group="org.scala-lang", artifact="scala-library", version=scala.version, ), ]), ), ), ), Get( Digest, # Flatten the entire transitive classpath. MergeDigests(classfiles.digest for classfiles in ClasspathEntry.closure( direct_dependency_classpath_entries)), ), ) usercp = "__cp" prefixed_transitive_dependency_classpath_digest = await Get( Digest, AddPrefix(merged_transitive_dependency_classpath_entries_digest, usercp)) merged_digest = await Get( Digest, MergeDigests(( prefixed_transitive_dependency_classpath_digest, tool_classpath.digest, jdk_setup.digest, *(sources.snapshot.digest for _, sources in component_members_and_scala_source_files), )), ) classpath_arg = ClasspathEntry.arg( ClasspathEntry.closure(direct_dependency_classpath_entries), prefix=usercp) output_file = f"{request.component.representative.address.path_safe_spec}.jar" process_result = await Get( FallibleProcessResult, Process( argv=[ *jdk_setup.args(bash, tool_classpath.classpath_entries()), "scala.tools.nsc.Main", "-bootclasspath", ":".join(tool_classpath.classpath_entries()), *(("-classpath", classpath_arg) if classpath_arg else ()), "-d", output_file, *sorted( chain.from_iterable( sources.snapshot.files for _, sources in component_members_and_scala_source_files)), ], input_digest=merged_digest, use_nailgun=jdk_setup.digest, output_files=(output_file, ), description=f"Compile {request.component} with scalac", level=LogLevel.DEBUG, append_only_caches=jdk_setup.append_only_caches, env=jdk_setup.env, ), ) output: ClasspathEntry | None = None if process_result.exit_code == 0: output = ClasspathEntry(process_result.output_digest, (output_file, ), direct_dependency_classpath_entries) return FallibleClasspathEntry.from_fallible_process_result( str(request.component), process_result, output, )
async def compile_kotlin_source( kotlin: KotlinSubsystem, kotlinc: KotlincSubsystem, request: CompileKotlinSourceRequest, ) -> FallibleClasspathEntry: # Request classpath entries for our direct dependencies. dependency_cpers = await Get(FallibleClasspathEntries, ClasspathDependenciesRequest(request)) direct_dependency_classpath_entries = dependency_cpers.if_all_succeeded() if direct_dependency_classpath_entries is None: return FallibleClasspathEntry( description=str(request.component), result=CompileResult.DEPENDENCY_FAILED, output=None, exit_code=1, ) kotlin_version = kotlin.version_for_resolve(request.resolve.name) component_members_with_sources = tuple( t for t in request.component.members if t.has_field(SourcesField) ) component_members_and_source_files = zip( component_members_with_sources, await MultiGet( Get( SourceFiles, SourceFilesRequest( (t.get(SourcesField),), for_sources_types=(KotlinSourceField,), enable_codegen=True, ), ) for t in component_members_with_sources ), ) plugins_ = await MultiGet( Get( KotlincPluginTargetsForTarget, KotlincPluginsForTargetRequest(target, request.resolve.name), ) for target in request.component.members ) plugins_request = KotlincPluginsRequest.from_target_plugins(plugins_, request.resolve) local_plugins = await Get(KotlincPlugins, KotlincPluginsRequest, plugins_request) component_members_and_kotlin_source_files = [ (target, sources) for target, sources in component_members_and_source_files if sources.snapshot.digest != EMPTY_DIGEST ] if not component_members_and_kotlin_source_files: # Is a generator, and so exports all of its direct deps. exported_digest = await Get( Digest, MergeDigests(cpe.digest for cpe in direct_dependency_classpath_entries) ) classpath_entry = ClasspathEntry.merge(exported_digest, direct_dependency_classpath_entries) return FallibleClasspathEntry( description=str(request.component), result=CompileResult.SUCCEEDED, output=classpath_entry, exit_code=0, ) toolcp_relpath = "__toolcp" local_kotlinc_plugins_relpath = "__localplugincp" usercp = "__cp" user_classpath = Classpath(direct_dependency_classpath_entries, request.resolve) tool_classpath, sources_digest, jdk = await MultiGet( Get( ToolClasspath, ToolClasspathRequest( artifact_requirements=ArtifactRequirements.from_coordinates( [ Coordinate( group="org.jetbrains.kotlin", artifact="kotlin-compiler-embeddable", version=kotlin_version, ), Coordinate( group="org.jetbrains.kotlin", artifact="kotlin-scripting-compiler-embeddable", version=kotlin_version, ), ] ), ), ), Get( Digest, MergeDigests( ( sources.snapshot.digest for _, sources in component_members_and_kotlin_source_files ) ), ), Get(JdkEnvironment, JdkRequest, JdkRequest.from_target(request.component)), ) extra_immutable_input_digests = { toolcp_relpath: tool_classpath.digest, local_kotlinc_plugins_relpath: local_plugins.classpath.digest, } extra_nailgun_keys = tuple(extra_immutable_input_digests) extra_immutable_input_digests.update(user_classpath.immutable_inputs(prefix=usercp)) classpath_arg = ":".join(user_classpath.immutable_inputs_args(prefix=usercp)) output_file = compute_output_jar_filename(request.component) process_result = await Get( FallibleProcessResult, JvmProcess( jdk=jdk, classpath_entries=tool_classpath.classpath_entries(toolcp_relpath), argv=[ "org.jetbrains.kotlin.cli.jvm.K2JVMCompiler", *(("-classpath", classpath_arg) if classpath_arg else ()), "-d", output_file, *(local_plugins.args(local_kotlinc_plugins_relpath)), *kotlinc.args, *sorted( itertools.chain.from_iterable( sources.snapshot.files for _, sources in component_members_and_kotlin_source_files ) ), ], input_digest=sources_digest, extra_immutable_input_digests=extra_immutable_input_digests, extra_nailgun_keys=extra_nailgun_keys, output_files=(output_file,), description=f"Compile {request.component} with kotlinc", level=LogLevel.DEBUG, ), ) output: ClasspathEntry | None = None if process_result.exit_code == 0: output = ClasspathEntry( process_result.output_digest, (output_file,), direct_dependency_classpath_entries ) return FallibleClasspathEntry.from_fallible_process_result( str(request.component), process_result, output, )