async def build_processors( bash: BashBinary, jdk_setup: JdkSetup) -> JavaParserCompiledClassfiles: dest_dir = "classfiles" materialized_classpath, source_digest = await MultiGet( Get( MaterializedClasspath, MaterializedClasspathRequest( prefix="__toolcp", artifact_requirements=(java_parser_artifact_requirements(), ), ), ), Get( Digest, CreateDigest([ FileContent( path=_LAUNCHER_BASENAME, content=_load_javaparser_launcher_source(), ), Directory(dest_dir), ]), ), ) merged_digest = await Get( Digest, MergeDigests(( materialized_classpath.digest, jdk_setup.digest, source_digest, )), ) process_result = await Get( ProcessResult, Process( argv=[ *jdk_setup.args(bash, [f"{jdk_setup.java_home}/lib/tools.jar"]), "com.sun.tools.javac.Main", "-cp", materialized_classpath.classpath_arg(), "-d", dest_dir, _LAUNCHER_BASENAME, ], input_digest=merged_digest, use_nailgun=jdk_setup.digest, output_directories=(dest_dir, ), description= f"Compile {_LAUNCHER_BASENAME} import processors with javac", level=LogLevel.DEBUG, ), ) stripped_classfiles_digest = await Get( Digest, RemovePrefix(process_result.output_digest, dest_dir)) return JavaParserCompiledClassfiles(digest=stripped_classfiles_digest)
async def run_junit_test( bash: BashBinary, jdk_setup: JdkSetup, junit: JUnit, test_subsystem: TestSubsystem, field_set: JavaTestFieldSet, ) -> TestResult: transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])) coarsened_targets = await Get( CoarsenedTargets, Addresses(t.address for t in transitive_targets.closure) ) lockfile = await Get( CoursierResolvedLockfile, CoursierLockfileForTargetRequest(Targets(transitive_targets.closure)), ) materialized_classpath = await Get( MaterializedClasspath, MaterializedClasspathRequest( prefix="__thirdpartycp", lockfiles=(lockfile,), artifact_requirements=( ArtifactRequirements( [ Coordinate( group="org.junit.platform", artifact="junit-platform-console", version="1.7.2", ), Coordinate( group="org.junit.jupiter", artifact="junit-jupiter-engine", version="5.7.2", ), Coordinate( group="org.junit.vintage", artifact="junit-vintage-engine", version="5.7.2", ), ] ), ), ), ) transitive_user_classfiles = await MultiGet( Get(CompiledClassfiles, CompileJavaSourceRequest(component=t)) for t in coarsened_targets ) merged_transitive_user_classfiles_digest = await Get( Digest, MergeDigests(classfiles.digest for classfiles in transitive_user_classfiles) ) usercp_relpath = "__usercp" prefixed_transitive_user_classfiles_digest = await Get( Digest, AddPrefix(merged_transitive_user_classfiles_digest, usercp_relpath) ) merged_digest = await Get( Digest, MergeDigests( ( prefixed_transitive_user_classfiles_digest, materialized_classpath.digest, jdk_setup.digest, ) ), ) proc = Process( argv=[ *jdk_setup.args(bash, [materialized_classpath.classpath_arg()]), "org.junit.platform.console.ConsoleLauncher", "--classpath", usercp_relpath, "--scan-class-path", usercp_relpath, *junit.options.args, ], input_digest=merged_digest, description=f"Run JUnit 5 ConsoleLauncher against {field_set.address}", level=LogLevel.DEBUG, ) process_result = await Get( FallibleProcessResult, Process, proc, ) return TestResult.from_fallible_process_result( process_result, address=field_set.address, output_setting=test_subsystem.output, )
async def run_junit_test( bash: BashBinary, jdk_setup: JdkSetup, junit: JUnit, test_subsystem: TestSubsystem, field_set: JavaTestFieldSet, ) -> TestResult: classpath = await Get(Classpath, Addresses([field_set.address])) junit_classpath = await Get( MaterializedClasspath, MaterializedClasspathRequest( prefix="__thirdpartycp", artifact_requirements=(ArtifactRequirements([ Coordinate( group="org.junit.platform", artifact="junit-platform-console", version="1.7.2", ), Coordinate( group="org.junit.jupiter", artifact="junit-jupiter-engine", version="5.7.2", ), Coordinate( group="org.junit.vintage", artifact="junit-vintage-engine", version="5.7.2", ), ]), ), ), ) merged_digest = await Get( Digest, MergeDigests((classpath.content.digest, jdk_setup.digest, junit_classpath.digest)), ) reports_dir_prefix = "__reports_dir" reports_dir = f"{reports_dir_prefix}/{field_set.address.path_safe_spec}" user_classpath_arg = ":".join(classpath.user_classpath_entries()) process_result = await Get( FallibleProcessResult, Process( argv=[ *jdk_setup.args(bash, [ *classpath.classpath_entries(), *junit_classpath.classpath_entries() ]), "org.junit.platform.console.ConsoleLauncher", *(("--classpath", user_classpath_arg) if user_classpath_arg else ()), *(("--scan-class-path", user_classpath_arg) if user_classpath_arg else ()), "--reports-dir", reports_dir, *junit.options.args, ], input_digest=merged_digest, output_directories=(reports_dir, ), append_only_caches=jdk_setup.append_only_caches, env=jdk_setup.env, description= f"Run JUnit 5 ConsoleLauncher against {field_set.address}", level=LogLevel.DEBUG, ), ) xml_result_subset = await Get( Digest, DigestSubset(process_result.output_digest, PathGlobs([f"{reports_dir_prefix}/**"]))) xml_results = await Get( Snapshot, RemovePrefix(xml_result_subset, reports_dir_prefix)) return TestResult.from_fallible_process_result( process_result, address=field_set.address, output_setting=test_subsystem.output, xml_results=xml_results, )
async def compile_scala_source( bash: BashBinary, coursier: Coursier, jdk_setup: JdkSetup, scala: ScalaSubsystem, union_membership: UnionMembership, request: CompileScalaSourceRequest, ) -> FallibleClasspathEntry: # Request classpath entries for our direct dependencies. direct_dependency_classpath_entries = FallibleClasspathEntry.if_all_succeeded( await MultiGet( Get( FallibleClasspathEntry, ClasspathEntryRequest, ClasspathEntryRequest.for_targets(union_membership, component=coarsened_dep, resolve=request.resolve), ) for coarsened_dep in request.component.dependencies)) if direct_dependency_classpath_entries is None: return FallibleClasspathEntry( description=str(request.component), result=CompileResult.DEPENDENCY_FAILED, output=None, exit_code=1, ) component_members_with_sources = tuple(t for t in request.component.members if t.has_field(SourcesField)) component_members_and_source_files = zip( component_members_with_sources, await MultiGet( Get( SourceFiles, SourceFilesRequest( (t.get(SourcesField), ), for_sources_types=(ScalaSourceField, ), enable_codegen=True, ), ) for t in component_members_with_sources), ) component_members_and_scala_source_files = [ (target, sources) for target, sources in component_members_and_source_files if sources.snapshot.digest != EMPTY_DIGEST ] if not component_members_and_scala_source_files: # Is a generator, and so exports all of its direct deps. exported_digest = await Get( Digest, MergeDigests(cpe.digest for cpe in direct_dependency_classpath_entries)) classpath_entry = ClasspathEntry.merge( exported_digest, direct_dependency_classpath_entries) return FallibleClasspathEntry( description=str(request.component), result=CompileResult.SUCCEEDED, output=classpath_entry, exit_code=0, ) ( tool_classpath, merged_transitive_dependency_classpath_entries_digest, ) = await MultiGet( Get( MaterializedClasspath, MaterializedClasspathRequest( prefix="__toolcp", artifact_requirements=(ArtifactRequirements([ Coordinate( group="org.scala-lang", artifact="scala-compiler", version=scala.version, ), Coordinate( group="org.scala-lang", artifact="scala-library", version=scala.version, ), ]), ), ), ), Get( Digest, # Flatten the entire transitive classpath. MergeDigests(classfiles.digest for classfiles in ClasspathEntry.closure( direct_dependency_classpath_entries)), ), ) usercp = "__cp" prefixed_transitive_dependency_classpath_digest = await Get( Digest, AddPrefix(merged_transitive_dependency_classpath_entries_digest, usercp)) merged_digest = await Get( Digest, MergeDigests(( prefixed_transitive_dependency_classpath_digest, tool_classpath.digest, jdk_setup.digest, *(sources.snapshot.digest for _, sources in component_members_and_scala_source_files), )), ) classpath_arg = ClasspathEntry.arg( ClasspathEntry.closure(direct_dependency_classpath_entries), prefix=usercp) output_file = f"{request.component.representative.address.path_safe_spec}.jar" process_result = await Get( FallibleProcessResult, Process( argv=[ *jdk_setup.args(bash, tool_classpath.classpath_entries()), "scala.tools.nsc.Main", "-bootclasspath", ":".join(tool_classpath.classpath_entries()), *(("-classpath", classpath_arg) if classpath_arg else ()), "-d", output_file, *sorted( chain.from_iterable( sources.snapshot.files for _, sources in component_members_and_scala_source_files)), ], input_digest=merged_digest, use_nailgun=jdk_setup.digest, output_files=(output_file, ), description=f"Compile {request.component} with scalac", level=LogLevel.DEBUG, append_only_caches=jdk_setup.append_only_caches, env=jdk_setup.env, ), ) output: ClasspathEntry | None = None if process_result.exit_code == 0: output = ClasspathEntry(process_result.output_digest, (output_file, ), direct_dependency_classpath_entries) return FallibleClasspathEntry.from_fallible_process_result( str(request.component), process_result, output, )
async def analyze_java_source_dependencies( bash: BashBinary, jdk_setup: JdkSetup, processor_classfiles: JavaParserCompiledClassfiles, source_files: SourceFiles, ) -> FallibleJavaSourceDependencyAnalysisResult: if len(source_files.files) > 1: raise ValueError( f"parse_java_package expects sources with exactly 1 source file, but found {len(source_files.snapshot.files)}." ) elif len(source_files.files) == 0: raise ValueError( "parse_java_package expects sources with exactly 1 source file, but found none." ) source_prefix = "__source_to_analyze" source_path = os.path.join(source_prefix, source_files.files[0]) processorcp_relpath = "__processorcp" ( tool_classpath, prefixed_processor_classfiles_digest, prefixed_source_files_digest, ) = await MultiGet( Get( MaterializedClasspath, MaterializedClasspathRequest( prefix="__toolcp", artifact_requirements=(java_parser_artifact_requirements(), ), ), ), Get(Digest, AddPrefix(processor_classfiles.digest, processorcp_relpath)), Get(Digest, AddPrefix(source_files.snapshot.digest, source_prefix)), ) tool_digest = await Get( Digest, MergeDigests(( prefixed_processor_classfiles_digest, tool_classpath.digest, jdk_setup.digest, )), ) merged_digest = await Get( Digest, MergeDigests(( tool_digest, prefixed_source_files_digest, )), ) analysis_output_path = "__source_analysis.json" process_result = await Get( FallibleProcessResult, Process( argv=[ *jdk_setup.args(bash, [ *tool_classpath.classpath_entries(), processorcp_relpath ]), "org.pantsbuild.javaparser.PantsJavaParserLauncher", analysis_output_path, source_path, ], input_digest=merged_digest, output_files=(analysis_output_path, ), use_nailgun=tool_digest, append_only_caches=jdk_setup.append_only_caches, env=jdk_setup.env, description=f"Analyzing {source_files.files[0]}", level=LogLevel.DEBUG, ), ) return FallibleJavaSourceDependencyAnalysisResult( process_result=process_result)
async def setup_scala_parser_classfiles( bash: BashBinary, jdk_setup: JdkSetup ) -> ScalaParserCompiledClassfiles: dest_dir = "classfiles" parser_source_content = pkgutil.get_data( "pants.backend.scala.dependency_inference", "ScalaParser.scala" ) if not parser_source_content: raise AssertionError("Unable to find ScalaParser.scala resource.") parser_source = FileContent("ScalaParser.scala", parser_source_content) tool_classpath, parser_classpath, source_digest = await MultiGet( Get( MaterializedClasspath, MaterializedClasspathRequest( prefix="__toolcp", artifact_requirements=( ArtifactRequirements( [ Coordinate( group="org.scala-lang", artifact="scala-compiler", version=PARSER_SCALA_VERSION, ), Coordinate( group="org.scala-lang", artifact="scala-library", version=PARSER_SCALA_VERSION, ), Coordinate( group="org.scala-lang", artifact="scala-reflect", version=PARSER_SCALA_VERSION, ), ] ), ), ), ), Get( MaterializedClasspath, MaterializedClasspathRequest( prefix="__parsercp", artifact_requirements=(SCALA_PARSER_ARTIFACT_REQUIREMENTS,) ), ), Get( Digest, CreateDigest( [ parser_source, Directory(dest_dir), ] ), ), ) merged_digest = await Get( Digest, MergeDigests( ( tool_classpath.digest, parser_classpath.digest, jdk_setup.digest, source_digest, ) ), ) # NB: We do not use nailgun for this process, since it is launched exactly once. process_result = await Get( ProcessResult, Process( argv=[ *jdk_setup.args(bash, tool_classpath.classpath_entries()), "scala.tools.nsc.Main", "-bootclasspath", ":".join(tool_classpath.classpath_entries()), "-classpath", ":".join(parser_classpath.classpath_entries()), "-d", dest_dir, parser_source.path, ], input_digest=merged_digest, append_only_caches=jdk_setup.append_only_caches, env=jdk_setup.env, output_directories=(dest_dir,), description="Compile Scala parser for dependency inference with scalac", level=LogLevel.DEBUG, ), ) stripped_classfiles_digest = await Get( Digest, RemovePrefix(process_result.output_digest, dest_dir) ) return ScalaParserCompiledClassfiles(digest=stripped_classfiles_digest)
async def compile_java_source( bash: BashBinary, coursier: Coursier, jdk_setup: JdkSetup, request: CompileJavaSourceRequest, ) -> FallibleCompiledClassfiles: component_members_with_sources = tuple(t for t in request.component.members if t.has_field(Sources)) component_members_and_source_files = zip( component_members_with_sources, await MultiGet( Get( SourceFiles, SourceFilesRequest( (t.get(Sources), ), for_sources_types=(JavaSourceField, ), enable_codegen=True, ), ) for t in component_members_with_sources), ) component_members_and_java_source_files = [ (target, sources) for target, sources in component_members_and_source_files if sources.snapshot.digest != EMPTY_DIGEST ] if not component_members_and_java_source_files: return FallibleCompiledClassfiles( description=str(request.component), result=CompileResult.SUCCEEDED, output=CompiledClassfiles(digest=EMPTY_DIGEST), exit_code=0, ) # Target coarsening currently doesn't perform dep expansion, which matters for targets # with multiple sources that expand to individual source subtargets. # We expand the dependencies explicitly here before coarsening, but ideally this could # be done somehow during coarsening. # TODO: Should component dependencies be filtered out here if they were only brought in by component members which were # filtered out above (due to having no JavaSources to contribute)? If so, that will likely required extending # the CoarsenedTargets API to include more complete dependency information, or to support such filtering directly. expanded_direct_deps = await Get(Targets, Addresses(request.component.dependencies)) coarsened_direct_deps = await Get( CoarsenedTargets, Addresses(t.address for t in expanded_direct_deps)) lockfile = await Get( CoursierResolvedLockfile, CoursierLockfileForTargetRequest(Targets(request.component.members)), ) direct_dependency_classfiles_fallible = await MultiGet( Get(FallibleCompiledClassfiles, CompileJavaSourceRequest(component=coarsened_dep)) for coarsened_dep in coarsened_direct_deps) direct_dependency_classfiles = [ fcc.output for fcc in direct_dependency_classfiles_fallible if fcc.output ] if len(direct_dependency_classfiles) != len( direct_dependency_classfiles_fallible): return FallibleCompiledClassfiles( description=str(request.component), result=CompileResult.DEPENDENCY_FAILED, output=None, exit_code=1, ) dest_dir = "classfiles" ( materialized_classpath, merged_direct_dependency_classfiles_digest, dest_dir_digest, ) = await MultiGet( Get( MaterializedClasspath, MaterializedClasspathRequest( prefix="__thirdpartycp", lockfiles=(lockfile, ), ), ), Get( Digest, MergeDigests(classfiles.digest for classfiles in direct_dependency_classfiles)), Get( Digest, CreateDigest([Directory(dest_dir)]), ), ) usercp_relpath = "__usercp" prefixed_direct_dependency_classfiles_digest = await Get( Digest, AddPrefix(merged_direct_dependency_classfiles_digest, usercp_relpath)) classpath_arg = usercp_relpath third_party_classpath_arg = materialized_classpath.classpath_arg() if third_party_classpath_arg: classpath_arg = ":".join([classpath_arg, third_party_classpath_arg]) merged_digest = await Get( Digest, MergeDigests(( prefixed_direct_dependency_classfiles_digest, materialized_classpath.digest, dest_dir_digest, jdk_setup.digest, *(sources.snapshot.digest for _, sources in component_members_and_java_source_files), )), ) process_result = await Get( FallibleProcessResult, Process( argv=[ *jdk_setup.args(bash, [f"{jdk_setup.java_home}/lib/tools.jar"]), "com.sun.tools.javac.Main", "-cp", classpath_arg, "-d", dest_dir, *sorted( chain.from_iterable( sources.snapshot.files for _, sources in component_members_and_java_source_files)), ], input_digest=merged_digest, use_nailgun=jdk_setup.digest, output_directories=(dest_dir, ), description=f"Compile {request.component.members} with javac", level=LogLevel.DEBUG, ), ) output: CompiledClassfiles | None = None if process_result.exit_code == 0: stripped_classfiles_digest = await Get( Digest, RemovePrefix(process_result.output_digest, dest_dir)) output = CompiledClassfiles(stripped_classfiles_digest) return FallibleCompiledClassfiles.from_fallible_process_result( str(request.component), process_result, output, )