async def build_processors(
bash: BashBinary, jdk_setup: JdkSetup) -> JavaParserCompiledClassfiles:
dest_dir = "classfiles"
materialized_classpath, source_digest = await MultiGet(
Get(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__toolcp",
artifact_requirements=(java_parser_artifact_requirements(), ),
),
),
Get(
Digest,
CreateDigest([
FileContent(
path=_LAUNCHER_BASENAME,
content=_load_javaparser_launcher_source(),
),
Directory(dest_dir),
]),
),
)
merged_digest = await Get(
Digest,
MergeDigests((
materialized_classpath.digest,
jdk_setup.digest,
source_digest,
)),
)
process_result = await Get(
ProcessResult,
Process(
argv=[
*jdk_setup.args(bash,
[f"{jdk_setup.java_home}/lib/tools.jar"]),
"com.sun.tools.javac.Main",
"-cp",
materialized_classpath.classpath_arg(),
"-d",
dest_dir,
_LAUNCHER_BASENAME,
],
input_digest=merged_digest,
use_nailgun=jdk_setup.digest,
output_directories=(dest_dir, ),
description=
f"Compile {_LAUNCHER_BASENAME} import processors with javac",
level=LogLevel.DEBUG,
),
)
stripped_classfiles_digest = await Get(
Digest, RemovePrefix(process_result.output_digest, dest_dir))
return JavaParserCompiledClassfiles(digest=stripped_classfiles_digest)
async def run_junit_test(
bash: BashBinary,
jdk_setup: JdkSetup,
junit: JUnit,
test_subsystem: TestSubsystem,
field_set: JavaTestFieldSet,
) -> TestResult:
transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address]))
coarsened_targets = await Get(
CoarsenedTargets, Addresses(t.address for t in transitive_targets.closure)
)
lockfile = await Get(
CoursierResolvedLockfile,
CoursierLockfileForTargetRequest(Targets(transitive_targets.closure)),
)
materialized_classpath = await Get(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__thirdpartycp",
lockfiles=(lockfile,),
artifact_requirements=(
ArtifactRequirements(
[
Coordinate(
group="org.junit.platform",
artifact="junit-platform-console",
version="1.7.2",
),
Coordinate(
group="org.junit.jupiter",
artifact="junit-jupiter-engine",
version="5.7.2",
),
Coordinate(
group="org.junit.vintage",
artifact="junit-vintage-engine",
version="5.7.2",
),
]
),
),
),
)
transitive_user_classfiles = await MultiGet(
Get(CompiledClassfiles, CompileJavaSourceRequest(component=t)) for t in coarsened_targets
)
merged_transitive_user_classfiles_digest = await Get(
Digest, MergeDigests(classfiles.digest for classfiles in transitive_user_classfiles)
)
usercp_relpath = "__usercp"
prefixed_transitive_user_classfiles_digest = await Get(
Digest, AddPrefix(merged_transitive_user_classfiles_digest, usercp_relpath)
)
merged_digest = await Get(
Digest,
MergeDigests(
(
prefixed_transitive_user_classfiles_digest,
materialized_classpath.digest,
jdk_setup.digest,
)
),
)
proc = Process(
argv=[
*jdk_setup.args(bash, [materialized_classpath.classpath_arg()]),
"org.junit.platform.console.ConsoleLauncher",
"--classpath",
usercp_relpath,
"--scan-class-path",
usercp_relpath,
*junit.options.args,
],
input_digest=merged_digest,
description=f"Run JUnit 5 ConsoleLauncher against {field_set.address}",
level=LogLevel.DEBUG,
)
process_result = await Get(
FallibleProcessResult,
Process,
proc,
)
return TestResult.from_fallible_process_result(
process_result,
address=field_set.address,
output_setting=test_subsystem.output,
)
async def run_junit_test(
bash: BashBinary,
jdk_setup: JdkSetup,
junit: JUnit,
test_subsystem: TestSubsystem,
field_set: JavaTestFieldSet,
) -> TestResult:
classpath = await Get(Classpath, Addresses([field_set.address]))
junit_classpath = await Get(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__thirdpartycp",
artifact_requirements=(ArtifactRequirements([
Coordinate(
group="org.junit.platform",
artifact="junit-platform-console",
version="1.7.2",
),
Coordinate(
group="org.junit.jupiter",
artifact="junit-jupiter-engine",
version="5.7.2",
),
Coordinate(
group="org.junit.vintage",
artifact="junit-vintage-engine",
version="5.7.2",
),
]), ),
),
)
merged_digest = await Get(
Digest,
MergeDigests((classpath.content.digest, jdk_setup.digest,
junit_classpath.digest)),
)
reports_dir_prefix = "__reports_dir"
reports_dir = f"{reports_dir_prefix}/{field_set.address.path_safe_spec}"
user_classpath_arg = ":".join(classpath.user_classpath_entries())
process_result = await Get(
FallibleProcessResult,
Process(
argv=[
*jdk_setup.args(bash, [
*classpath.classpath_entries(),
*junit_classpath.classpath_entries()
]),
"org.junit.platform.console.ConsoleLauncher",
*(("--classpath",
user_classpath_arg) if user_classpath_arg else ()),
*(("--scan-class-path",
user_classpath_arg) if user_classpath_arg else ()),
"--reports-dir",
reports_dir,
*junit.options.args,
],
input_digest=merged_digest,
output_directories=(reports_dir, ),
append_only_caches=jdk_setup.append_only_caches,
env=jdk_setup.env,
description=
f"Run JUnit 5 ConsoleLauncher against {field_set.address}",
level=LogLevel.DEBUG,
),
)
xml_result_subset = await Get(
Digest,
DigestSubset(process_result.output_digest,
PathGlobs([f"{reports_dir_prefix}/**"])))
xml_results = await Get(
Snapshot, RemovePrefix(xml_result_subset, reports_dir_prefix))
return TestResult.from_fallible_process_result(
process_result,
address=field_set.address,
output_setting=test_subsystem.output,
xml_results=xml_results,
)
async def compile_scala_source(
bash: BashBinary,
coursier: Coursier,
jdk_setup: JdkSetup,
scala: ScalaSubsystem,
union_membership: UnionMembership,
request: CompileScalaSourceRequest,
) -> FallibleClasspathEntry:
# Request classpath entries for our direct dependencies.
direct_dependency_classpath_entries = FallibleClasspathEntry.if_all_succeeded(
await MultiGet(
Get(
FallibleClasspathEntry,
ClasspathEntryRequest,
ClasspathEntryRequest.for_targets(union_membership,
component=coarsened_dep,
resolve=request.resolve),
) for coarsened_dep in request.component.dependencies))
if direct_dependency_classpath_entries is None:
return FallibleClasspathEntry(
description=str(request.component),
result=CompileResult.DEPENDENCY_FAILED,
output=None,
exit_code=1,
)
component_members_with_sources = tuple(t for t in request.component.members
if t.has_field(SourcesField))
component_members_and_source_files = zip(
component_members_with_sources,
await MultiGet(
Get(
SourceFiles,
SourceFilesRequest(
(t.get(SourcesField), ),
for_sources_types=(ScalaSourceField, ),
enable_codegen=True,
),
) for t in component_members_with_sources),
)
component_members_and_scala_source_files = [
(target, sources)
for target, sources in component_members_and_source_files
if sources.snapshot.digest != EMPTY_DIGEST
]
if not component_members_and_scala_source_files:
# Is a generator, and so exports all of its direct deps.
exported_digest = await Get(
Digest,
MergeDigests(cpe.digest
for cpe in direct_dependency_classpath_entries))
classpath_entry = ClasspathEntry.merge(
exported_digest, direct_dependency_classpath_entries)
return FallibleClasspathEntry(
description=str(request.component),
result=CompileResult.SUCCEEDED,
output=classpath_entry,
exit_code=0,
)
(
tool_classpath,
merged_transitive_dependency_classpath_entries_digest,
) = await MultiGet(
Get(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__toolcp",
artifact_requirements=(ArtifactRequirements([
Coordinate(
group="org.scala-lang",
artifact="scala-compiler",
version=scala.version,
),
Coordinate(
group="org.scala-lang",
artifact="scala-library",
version=scala.version,
),
]), ),
),
),
Get(
Digest,
# Flatten the entire transitive classpath.
MergeDigests(classfiles.digest
for classfiles in ClasspathEntry.closure(
direct_dependency_classpath_entries)),
),
)
usercp = "__cp"
prefixed_transitive_dependency_classpath_digest = await Get(
Digest,
AddPrefix(merged_transitive_dependency_classpath_entries_digest,
usercp))
merged_digest = await Get(
Digest,
MergeDigests((
prefixed_transitive_dependency_classpath_digest,
tool_classpath.digest,
jdk_setup.digest,
*(sources.snapshot.digest
for _, sources in component_members_and_scala_source_files),
)),
)
classpath_arg = ClasspathEntry.arg(
ClasspathEntry.closure(direct_dependency_classpath_entries),
prefix=usercp)
output_file = f"{request.component.representative.address.path_safe_spec}.jar"
process_result = await Get(
FallibleProcessResult,
Process(
argv=[
*jdk_setup.args(bash, tool_classpath.classpath_entries()),
"scala.tools.nsc.Main",
"-bootclasspath",
":".join(tool_classpath.classpath_entries()),
*(("-classpath", classpath_arg) if classpath_arg else ()),
"-d",
output_file,
*sorted(
chain.from_iterable(
sources.snapshot.files for _, sources in
component_members_and_scala_source_files)),
],
input_digest=merged_digest,
use_nailgun=jdk_setup.digest,
output_files=(output_file, ),
description=f"Compile {request.component} with scalac",
level=LogLevel.DEBUG,
append_only_caches=jdk_setup.append_only_caches,
env=jdk_setup.env,
),
)
output: ClasspathEntry | None = None
if process_result.exit_code == 0:
output = ClasspathEntry(process_result.output_digest, (output_file, ),
direct_dependency_classpath_entries)
return FallibleClasspathEntry.from_fallible_process_result(
str(request.component),
process_result,
output,
)
async def analyze_java_source_dependencies(
bash: BashBinary,
jdk_setup: JdkSetup,
processor_classfiles: JavaParserCompiledClassfiles,
source_files: SourceFiles,
) -> FallibleJavaSourceDependencyAnalysisResult:
if len(source_files.files) > 1:
raise ValueError(
f"parse_java_package expects sources with exactly 1 source file, but found {len(source_files.snapshot.files)}."
)
elif len(source_files.files) == 0:
raise ValueError(
"parse_java_package expects sources with exactly 1 source file, but found none."
)
source_prefix = "__source_to_analyze"
source_path = os.path.join(source_prefix, source_files.files[0])
processorcp_relpath = "__processorcp"
(
tool_classpath,
prefixed_processor_classfiles_digest,
prefixed_source_files_digest,
) = await MultiGet(
Get(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__toolcp",
artifact_requirements=(java_parser_artifact_requirements(), ),
),
),
Get(Digest, AddPrefix(processor_classfiles.digest,
processorcp_relpath)),
Get(Digest, AddPrefix(source_files.snapshot.digest, source_prefix)),
)
tool_digest = await Get(
Digest,
MergeDigests((
prefixed_processor_classfiles_digest,
tool_classpath.digest,
jdk_setup.digest,
)),
)
merged_digest = await Get(
Digest,
MergeDigests((
tool_digest,
prefixed_source_files_digest,
)),
)
analysis_output_path = "__source_analysis.json"
process_result = await Get(
FallibleProcessResult,
Process(
argv=[
*jdk_setup.args(bash, [
*tool_classpath.classpath_entries(), processorcp_relpath
]),
"org.pantsbuild.javaparser.PantsJavaParserLauncher",
analysis_output_path,
source_path,
],
input_digest=merged_digest,
output_files=(analysis_output_path, ),
use_nailgun=tool_digest,
append_only_caches=jdk_setup.append_only_caches,
env=jdk_setup.env,
description=f"Analyzing {source_files.files[0]}",
level=LogLevel.DEBUG,
),
)
return FallibleJavaSourceDependencyAnalysisResult(
process_result=process_result)
async def setup_scala_parser_classfiles(
bash: BashBinary, jdk_setup: JdkSetup
) -> ScalaParserCompiledClassfiles:
dest_dir = "classfiles"
parser_source_content = pkgutil.get_data(
"pants.backend.scala.dependency_inference", "ScalaParser.scala"
)
if not parser_source_content:
raise AssertionError("Unable to find ScalaParser.scala resource.")
parser_source = FileContent("ScalaParser.scala", parser_source_content)
tool_classpath, parser_classpath, source_digest = await MultiGet(
Get(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__toolcp",
artifact_requirements=(
ArtifactRequirements(
[
Coordinate(
group="org.scala-lang",
artifact="scala-compiler",
version=PARSER_SCALA_VERSION,
),
Coordinate(
group="org.scala-lang",
artifact="scala-library",
version=PARSER_SCALA_VERSION,
),
Coordinate(
group="org.scala-lang",
artifact="scala-reflect",
version=PARSER_SCALA_VERSION,
),
]
),
),
),
),
Get(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__parsercp", artifact_requirements=(SCALA_PARSER_ARTIFACT_REQUIREMENTS,)
),
),
Get(
Digest,
CreateDigest(
[
parser_source,
Directory(dest_dir),
]
),
),
)
merged_digest = await Get(
Digest,
MergeDigests(
(
tool_classpath.digest,
parser_classpath.digest,
jdk_setup.digest,
source_digest,
)
),
)
# NB: We do not use nailgun for this process, since it is launched exactly once.
process_result = await Get(
ProcessResult,
Process(
argv=[
*jdk_setup.args(bash, tool_classpath.classpath_entries()),
"scala.tools.nsc.Main",
"-bootclasspath",
":".join(tool_classpath.classpath_entries()),
"-classpath",
":".join(parser_classpath.classpath_entries()),
"-d",
dest_dir,
parser_source.path,
],
input_digest=merged_digest,
append_only_caches=jdk_setup.append_only_caches,
env=jdk_setup.env,
output_directories=(dest_dir,),
description="Compile Scala parser for dependency inference with scalac",
level=LogLevel.DEBUG,
),
)
stripped_classfiles_digest = await Get(
Digest, RemovePrefix(process_result.output_digest, dest_dir)
)
return ScalaParserCompiledClassfiles(digest=stripped_classfiles_digest)
async def compile_java_source(
bash: BashBinary,
coursier: Coursier,
jdk_setup: JdkSetup,
request: CompileJavaSourceRequest,
) -> FallibleCompiledClassfiles:
component_members_with_sources = tuple(t for t in request.component.members
if t.has_field(Sources))
component_members_and_source_files = zip(
component_members_with_sources,
await MultiGet(
Get(
SourceFiles,
SourceFilesRequest(
(t.get(Sources), ),
for_sources_types=(JavaSourceField, ),
enable_codegen=True,
),
) for t in component_members_with_sources),
)
component_members_and_java_source_files = [
(target, sources)
for target, sources in component_members_and_source_files
if sources.snapshot.digest != EMPTY_DIGEST
]
if not component_members_and_java_source_files:
return FallibleCompiledClassfiles(
description=str(request.component),
result=CompileResult.SUCCEEDED,
output=CompiledClassfiles(digest=EMPTY_DIGEST),
exit_code=0,
)
# Target coarsening currently doesn't perform dep expansion, which matters for targets
# with multiple sources that expand to individual source subtargets.
# We expand the dependencies explicitly here before coarsening, but ideally this could
# be done somehow during coarsening.
# TODO: Should component dependencies be filtered out here if they were only brought in by component members which were
# filtered out above (due to having no JavaSources to contribute)? If so, that will likely required extending
# the CoarsenedTargets API to include more complete dependency information, or to support such filtering directly.
expanded_direct_deps = await Get(Targets,
Addresses(request.component.dependencies))
coarsened_direct_deps = await Get(
CoarsenedTargets, Addresses(t.address for t in expanded_direct_deps))
lockfile = await Get(
CoursierResolvedLockfile,
CoursierLockfileForTargetRequest(Targets(request.component.members)),
)
direct_dependency_classfiles_fallible = await MultiGet(
Get(FallibleCompiledClassfiles,
CompileJavaSourceRequest(component=coarsened_dep))
for coarsened_dep in coarsened_direct_deps)
direct_dependency_classfiles = [
fcc.output for fcc in direct_dependency_classfiles_fallible
if fcc.output
]
if len(direct_dependency_classfiles) != len(
direct_dependency_classfiles_fallible):
return FallibleCompiledClassfiles(
description=str(request.component),
result=CompileResult.DEPENDENCY_FAILED,
output=None,
exit_code=1,
)
dest_dir = "classfiles"
(
materialized_classpath,
merged_direct_dependency_classfiles_digest,
dest_dir_digest,
) = await MultiGet(
Get(
MaterializedClasspath,
MaterializedClasspathRequest(
prefix="__thirdpartycp",
lockfiles=(lockfile, ),
),
),
Get(
Digest,
MergeDigests(classfiles.digest
for classfiles in direct_dependency_classfiles)),
Get(
Digest,
CreateDigest([Directory(dest_dir)]),
),
)
usercp_relpath = "__usercp"
prefixed_direct_dependency_classfiles_digest = await Get(
Digest,
AddPrefix(merged_direct_dependency_classfiles_digest, usercp_relpath))
classpath_arg = usercp_relpath
third_party_classpath_arg = materialized_classpath.classpath_arg()
if third_party_classpath_arg:
classpath_arg = ":".join([classpath_arg, third_party_classpath_arg])
merged_digest = await Get(
Digest,
MergeDigests((
prefixed_direct_dependency_classfiles_digest,
materialized_classpath.digest,
dest_dir_digest,
jdk_setup.digest,
*(sources.snapshot.digest
for _, sources in component_members_and_java_source_files),
)),
)
process_result = await Get(
FallibleProcessResult,
Process(
argv=[
*jdk_setup.args(bash,
[f"{jdk_setup.java_home}/lib/tools.jar"]),
"com.sun.tools.javac.Main",
"-cp",
classpath_arg,
"-d",
dest_dir,
*sorted(
chain.from_iterable(
sources.snapshot.files for _, sources in
component_members_and_java_source_files)),
],
input_digest=merged_digest,
use_nailgun=jdk_setup.digest,
output_directories=(dest_dir, ),
description=f"Compile {request.component.members} with javac",
level=LogLevel.DEBUG,
),
)
output: CompiledClassfiles | None = None
if process_result.exit_code == 0:
stripped_classfiles_digest = await Get(
Digest, RemovePrefix(process_result.output_digest, dest_dir))
output = CompiledClassfiles(stripped_classfiles_digest)
return FallibleCompiledClassfiles.from_fallible_process_result(
str(request.component),
process_result,
output,
)