async def package_go_binary(field_set: GoBinaryFieldSet) -> BuiltPackage:
main_pkg = await Get(GoBinaryMainPackage, GoBinaryMainPackageRequest(field_set.main))
built_package = await Get(
BuiltGoPackage, BuildGoPackageTargetRequest(main_pkg.address, is_main=True)
)
main_pkg_a_file_path = built_package.import_paths_to_pkg_a_files["main"]
import_config = await Get(
ImportConfig, ImportConfigRequest(built_package.import_paths_to_pkg_a_files)
)
input_digest = await Get(Digest, MergeDigests([built_package.digest, import_config.digest]))
output_filename = PurePath(field_set.output_path.value_or_default(file_ending=None))
binary = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=input_digest,
archives=(main_pkg_a_file_path,),
import_config_path=import_config.CONFIG_PATH,
output_filename=f"./{output_filename.name}",
description=f"Link Go binary for {field_set.address}",
),
)
renamed_output_digest = await Get(Digest, AddPrefix(binary.digest, str(output_filename.parent)))
artifact = BuiltPackageArtifact(relpath=str(output_filename))
return BuiltPackage(renamed_output_digest, (artifact,))
def create_config(stdlib: bool) -> str:
config = rule_runner.request(
ImportConfig, [ImportConfigRequest(mapping, include_stdlib=stdlib)]
)
digest_contents = rule_runner.request(DigestContents, [config.digest])
assert len(digest_contents) == 1
file_content = digest_contents[0]
assert file_content.path == os.path.normpath(ImportConfig.CONFIG_PATH)
return file_content.content.decode()
async def setup_analyzer() -> PackageAnalyzerSetup:
def get_file(filename: str) -> bytes:
content = pkgutil.get_data("pants.backend.go.util_rules", filename)
if not content:
raise AssertionError(f"Unable to find resource for `{filename}`.")
return content
analyer_sources_content = [
FileContent(filename, get_file(filename))
for filename in ("analyze_package.go", "read.go")
]
source_digest, import_config = await MultiGet(
Get(Digest, CreateDigest(analyer_sources_content)),
Get(ImportConfig, ImportConfigRequest,
ImportConfigRequest.stdlib_only()),
)
built_analyzer_pkg = await Get(
BuiltGoPackage,
BuildGoPackageRequest(
import_path="main",
subpath="",
digest=source_digest,
go_file_names=tuple(fc.path for fc in analyer_sources_content),
s_file_names=(),
direct_dependencies=(),
minimum_go_version=None,
),
)
main_pkg_a_file_path = built_analyzer_pkg.import_paths_to_pkg_a_files[
"main"]
input_digest = await Get(
Digest, MergeDigests([built_analyzer_pkg.digest,
import_config.digest]))
analyzer = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=input_digest,
archives=(main_pkg_a_file_path, ),
import_config_path=import_config.CONFIG_PATH,
output_filename=PackageAnalyzerSetup.PATH,
description="Link Go package analyzer",
),
)
return PackageAnalyzerSetup(analyzer.digest)
async def setup_analyzer() -> AnalyzerSetup:
source_entry_content = pkgutil.get_data("pants.backend.go.util_rules", "generate_testmain.go")
if not source_entry_content:
raise AssertionError("Unable to find resource for `generate_testmain.go`.")
source_entry = FileContent("generate_testmain.go", source_entry_content)
source_digest, import_config = await MultiGet(
Get(Digest, CreateDigest([source_entry])),
Get(ImportConfig, ImportConfigRequest, ImportConfigRequest.stdlib_only()),
)
built_analyzer_pkg = await Get(
BuiltGoPackage,
BuildGoPackageRequest(
import_path="main",
subpath="",
digest=source_digest,
go_file_names=(source_entry.path,),
s_file_names=(),
direct_dependencies=(),
minimum_go_version=None,
),
)
main_pkg_a_file_path = built_analyzer_pkg.import_paths_to_pkg_a_files["main"]
input_digest = await Get(
Digest, MergeDigests([built_analyzer_pkg.digest, import_config.digest])
)
analyzer = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=input_digest,
archives=(main_pkg_a_file_path,),
import_config_path=import_config.CONFIG_PATH,
output_filename=AnalyzerSetup.PATH,
description="Link Go test sources analyzer",
),
)
return AnalyzerSetup(analyzer.digest)
async def setup_go_binary(request: LoadedGoBinaryRequest) -> LoadedGoBinary:
file_contents = setup_files(request.dir_name, request.file_names)
source_digest, import_config = await MultiGet(
Get(Digest, CreateDigest(file_contents)),
Get(ImportConfig, ImportConfigRequest,
ImportConfigRequest.stdlib_only()),
)
built_pkg = await Get(
BuiltGoPackage,
BuildGoPackageRequest(
import_path="main",
dir_path="",
digest=source_digest,
go_file_names=tuple(fc.path for fc in file_contents),
s_file_names=(),
direct_dependencies=(),
minimum_go_version=None,
),
)
main_pkg_a_file_path = built_pkg.import_paths_to_pkg_a_files["main"]
input_digest = await Get(
Digest, MergeDigests([built_pkg.digest, import_config.digest]))
binary = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=input_digest,
archives=(main_pkg_a_file_path, ),
import_config_path=import_config.CONFIG_PATH,
output_filename=request.output_name,
description="Link Go package analyzer",
),
)
return LoadedGoBinary(binary.digest)
async def run_go_tests(
field_set: GoTestFieldSet, test_subsystem: TestSubsystem, go_test_subsystem: GoTestSubsystem
) -> TestResult:
maybe_pkg_info, wrapped_target = await MultiGet(
Get(FallibleFirstPartyPkgInfo, FirstPartyPkgInfoRequest(field_set.address)),
Get(WrappedTarget, Address, field_set.address),
)
if maybe_pkg_info.info is None:
assert maybe_pkg_info.stderr is not None
return TestResult(
exit_code=maybe_pkg_info.exit_code,
stdout="",
stderr=maybe_pkg_info.stderr,
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=field_set.address,
output_setting=test_subsystem.output,
)
pkg_info = maybe_pkg_info.info
target = wrapped_target.target
import_path = target[GoImportPathField].value
testmain = await Get(
GeneratedTestMain,
GenerateTestMainRequest(
pkg_info.digest,
FrozenOrderedSet(
os.path.join(".", pkg_info.subpath, name) for name in pkg_info.test_files
),
FrozenOrderedSet(
os.path.join(".", pkg_info.subpath, name) for name in pkg_info.xtest_files
),
import_path=import_path,
),
)
if not testmain.has_tests and not testmain.has_xtests:
# Nothing to do so return an empty result.
# TODO: There should really be a "skipped entirely" mechanism for `TestResult`.
return TestResult(
exit_code=0,
stdout="",
stderr="",
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=field_set.address,
output_setting=test_subsystem.output,
)
# Construct the build request for the package under test.
maybe_test_pkg_build_request = await Get(
FallibleBuildGoPackageRequest,
BuildGoPackageTargetRequest(field_set.address, for_tests=True),
)
if maybe_test_pkg_build_request.request is None:
assert maybe_test_pkg_build_request.stderr is not None
return TestResult(
exit_code=maybe_test_pkg_build_request.exit_code,
stdout="",
stderr=maybe_test_pkg_build_request.stderr,
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=field_set.address,
output_setting=test_subsystem.output,
)
test_pkg_build_request = maybe_test_pkg_build_request.request
main_direct_deps = [test_pkg_build_request]
if testmain.has_xtests:
# Build a synthetic package for xtests where the import path is the same as the package under test
# but with "_test" appended.
#
# Subset the direct dependencies to only the dependencies used by the xtest code. (Dependency
# inference will have included all of the regular, test, and xtest dependencies of the package in
# the build graph.) Moreover, ensure that any import of the package under test is on the _test_
# version of the package that was just built.
dep_by_import_path = {
dep.import_path: dep for dep in test_pkg_build_request.direct_dependencies
}
direct_dependencies: OrderedSet[BuildGoPackageRequest] = OrderedSet()
for xtest_import in pkg_info.xtest_imports:
if xtest_import == pkg_info.import_path:
direct_dependencies.add(test_pkg_build_request)
elif xtest_import in dep_by_import_path:
direct_dependencies.add(dep_by_import_path[xtest_import])
xtest_pkg_build_request = BuildGoPackageRequest(
import_path=f"{import_path}_test",
digest=pkg_info.digest,
subpath=pkg_info.subpath,
go_file_names=pkg_info.xtest_files,
s_file_names=(), # TODO: Are there .s files for xtest?
direct_dependencies=tuple(direct_dependencies),
minimum_go_version=pkg_info.minimum_go_version,
)
main_direct_deps.append(xtest_pkg_build_request)
# Generate the synthetic main package which imports the test and/or xtest packages.
maybe_built_main_pkg = await Get(
FallibleBuiltGoPackage,
BuildGoPackageRequest(
import_path="main",
digest=testmain.digest,
subpath="",
go_file_names=(GeneratedTestMain.TEST_MAIN_FILE,),
s_file_names=(),
direct_dependencies=tuple(main_direct_deps),
minimum_go_version=pkg_info.minimum_go_version,
),
)
if maybe_built_main_pkg.output is None:
assert maybe_built_main_pkg.stderr is not None
return TestResult(
exit_code=maybe_built_main_pkg.exit_code,
stdout="",
stderr=maybe_built_main_pkg.stderr,
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=field_set.address,
output_setting=test_subsystem.output,
)
built_main_pkg = maybe_built_main_pkg.output
main_pkg_a_file_path = built_main_pkg.import_paths_to_pkg_a_files["main"]
import_config = await Get(
ImportConfig, ImportConfigRequest(built_main_pkg.import_paths_to_pkg_a_files)
)
input_digest = await Get(Digest, MergeDigests([built_main_pkg.digest, import_config.digest]))
binary = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=input_digest,
archives=(main_pkg_a_file_path,),
import_config_path=import_config.CONFIG_PATH,
output_filename="./test_runner", # TODO: Name test binary the way that `go` does?
description=f"Link Go test binary for {field_set.address}",
),
)
cache_scope = (
ProcessCacheScope.PER_SESSION if test_subsystem.force else ProcessCacheScope.SUCCESSFUL
)
result = await Get(
FallibleProcessResult,
Process(
["./test_runner", *transform_test_args(go_test_subsystem.args)],
input_digest=binary.digest,
description=f"Run Go tests: {field_set.address}",
cache_scope=cache_scope,
level=LogLevel.INFO,
),
)
return TestResult.from_fallible_process_result(result, field_set.address, test_subsystem.output)
async def run_go_tests(field_set: GoTestFieldSet,
test_subsystem: TestSubsystem,
go_test_subsystem: GoTestSubsystem) -> TestResult:
maybe_pkg_analysis, maybe_pkg_digest, dependencies = await MultiGet(
Get(FallibleFirstPartyPkgAnalysis,
FirstPartyPkgAnalysisRequest(field_set.address)),
Get(FallibleFirstPartyPkgDigest,
FirstPartyPkgDigestRequest(field_set.address)),
Get(Targets, DependenciesRequest(field_set.dependencies)),
)
def compilation_failure(exit_code: int, stdout: str | None,
stderr: str | None) -> TestResult:
return TestResult(
exit_code=exit_code,
stdout=stdout or "",
stderr=stderr or "",
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=field_set.address,
output_setting=test_subsystem.output,
result_metadata=None,
)
if maybe_pkg_analysis.analysis is None:
assert maybe_pkg_analysis.stderr is not None
return compilation_failure(maybe_pkg_analysis.exit_code, None,
maybe_pkg_analysis.stderr)
if maybe_pkg_digest.pkg_digest is None:
assert maybe_pkg_digest.stderr is not None
return compilation_failure(maybe_pkg_digest.exit_code, None,
maybe_pkg_digest.stderr)
pkg_analysis = maybe_pkg_analysis.analysis
pkg_digest = maybe_pkg_digest.pkg_digest
import_path = pkg_analysis.import_path
testmain = await Get(
GeneratedTestMain,
GenerateTestMainRequest(
pkg_digest.digest,
FrozenOrderedSet(
os.path.join(".", pkg_analysis.dir_path, name)
for name in pkg_analysis.test_go_files),
FrozenOrderedSet(
os.path.join(".", pkg_analysis.dir_path, name)
for name in pkg_analysis.xtest_go_files),
import_path,
field_set.address,
),
)
if testmain.failed_exit_code_and_stderr is not None:
_exit_code, _stderr = testmain.failed_exit_code_and_stderr
return compilation_failure(_exit_code, None, _stderr)
if not testmain.has_tests and not testmain.has_xtests:
return TestResult.skip(field_set.address,
output_setting=test_subsystem.output)
# Construct the build request for the package under test.
maybe_test_pkg_build_request = await Get(
FallibleBuildGoPackageRequest,
BuildGoPackageTargetRequest(field_set.address, for_tests=True),
)
if maybe_test_pkg_build_request.request is None:
assert maybe_test_pkg_build_request.stderr is not None
return compilation_failure(maybe_test_pkg_build_request.exit_code,
None, maybe_test_pkg_build_request.stderr)
test_pkg_build_request = maybe_test_pkg_build_request.request
main_direct_deps = [test_pkg_build_request]
if testmain.has_xtests:
# Build a synthetic package for xtests where the import path is the same as the package under test
# but with "_test" appended.
#
# Subset the direct dependencies to only the dependencies used by the xtest code. (Dependency
# inference will have included all of the regular, test, and xtest dependencies of the package in
# the build graph.) Moreover, ensure that any import of the package under test is on the _test_
# version of the package that was just built.
dep_by_import_path = {
dep.import_path: dep
for dep in test_pkg_build_request.direct_dependencies
}
direct_dependencies: OrderedSet[BuildGoPackageRequest] = OrderedSet()
for xtest_import in pkg_analysis.xtest_imports:
if xtest_import == pkg_analysis.import_path:
direct_dependencies.add(test_pkg_build_request)
elif xtest_import in dep_by_import_path:
direct_dependencies.add(dep_by_import_path[xtest_import])
xtest_pkg_build_request = BuildGoPackageRequest(
import_path=f"{import_path}_test",
digest=pkg_digest.digest,
dir_path=pkg_analysis.dir_path,
go_file_names=pkg_analysis.xtest_go_files,
s_file_names=(), # TODO: Are there .s files for xtest?
direct_dependencies=tuple(direct_dependencies),
minimum_go_version=pkg_analysis.minimum_go_version,
embed_config=pkg_digest.xtest_embed_config,
)
main_direct_deps.append(xtest_pkg_build_request)
# Generate the synthetic main package which imports the test and/or xtest packages.
maybe_built_main_pkg = await Get(
FallibleBuiltGoPackage,
BuildGoPackageRequest(
import_path="main",
digest=testmain.digest,
dir_path="",
go_file_names=(GeneratedTestMain.TEST_MAIN_FILE, ),
s_file_names=(),
direct_dependencies=tuple(main_direct_deps),
minimum_go_version=pkg_analysis.minimum_go_version,
),
)
if maybe_built_main_pkg.output is None:
assert maybe_built_main_pkg.stderr is not None
return compilation_failure(maybe_built_main_pkg.exit_code,
maybe_built_main_pkg.stdout,
maybe_built_main_pkg.stderr)
built_main_pkg = maybe_built_main_pkg.output
main_pkg_a_file_path = built_main_pkg.import_paths_to_pkg_a_files["main"]
import_config = await Get(
ImportConfig,
ImportConfigRequest(built_main_pkg.import_paths_to_pkg_a_files))
linker_input_digest = await Get(
Digest, MergeDigests([built_main_pkg.digest, import_config.digest]))
binary = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=linker_input_digest,
archives=(main_pkg_a_file_path, ),
import_config_path=import_config.CONFIG_PATH,
output_filename=
"./test_runner", # TODO: Name test binary the way that `go` does?
description=f"Link Go test binary for {field_set.address}",
),
)
# To emulate Go's test runner, we set the working directory to the path of the `go_package`.
# This allows tests to open dependencies on `file` targets regardless of where they are
# located. See https://dave.cheney.net/2016/05/10/test-fixtures-in-go.
working_dir = field_set.address.spec_path
binary_with_prefix, files_sources = await MultiGet(
Get(Digest, AddPrefix(binary.digest, working_dir)),
Get(
SourceFiles,
SourceFilesRequest(
(dep.get(SourcesField) for dep in dependencies),
for_sources_types=(FileSourceField, ),
enable_codegen=True,
),
),
)
test_input_digest = await Get(
Digest,
MergeDigests((binary_with_prefix, files_sources.snapshot.digest)))
cache_scope = (ProcessCacheScope.PER_SESSION
if test_subsystem.force else ProcessCacheScope.SUCCESSFUL)
result = await Get(
FallibleProcessResult,
Process(
[
"./test_runner",
*transform_test_args(go_test_subsystem.args,
field_set.timeout.value),
],
input_digest=test_input_digest,
description=f"Run Go tests: {field_set.address}",
cache_scope=cache_scope,
working_directory=working_dir,
level=LogLevel.DEBUG,
),
)
return TestResult.from_fallible_process_result(result, field_set.address,
test_subsystem.output)
async def build_go_package(request: BuildGoPackageRequest) -> FallibleBuiltGoPackage:
maybe_built_deps = await MultiGet(
Get(FallibleBuiltGoPackage, BuildGoPackageRequest, build_request)
for build_request in request.direct_dependencies
)
import_paths_to_pkg_a_files: dict[str, str] = {}
dep_digests = []
for maybe_dep in maybe_built_deps:
if maybe_dep.output is None:
return dataclasses.replace(
maybe_dep, import_path=request.import_path, dependency_failed=True
)
dep = maybe_dep.output
import_paths_to_pkg_a_files.update(dep.import_paths_to_pkg_a_files)
dep_digests.append(dep.digest)
merged_deps_digest, import_config, embedcfg = await MultiGet(
Get(Digest, MergeDigests(dep_digests)),
Get(ImportConfig, ImportConfigRequest(FrozenDict(import_paths_to_pkg_a_files))),
Get(RenderedEmbedConfig, RenderEmbedConfigRequest(request.embed_config)),
)
input_digest = await Get(
Digest,
MergeDigests([merged_deps_digest, import_config.digest, embedcfg.digest, request.digest]),
)
assembly_digests = None
symabis_path = None
if request.s_file_names:
assembly_setup = await Get(
FallibleAssemblyPreCompilation,
AssemblyPreCompilationRequest(input_digest, request.s_file_names, request.subpath),
)
if assembly_setup.result is None:
return FallibleBuiltGoPackage(
None,
request.import_path,
assembly_setup.exit_code,
stdout=assembly_setup.stdout,
stderr=assembly_setup.stderr,
)
input_digest = assembly_setup.result.merged_compilation_input_digest
assembly_digests = assembly_setup.result.assembly_digests
symabis_path = "./symabis"
compile_args = [
"tool",
"compile",
"-o",
"__pkg__.a",
"-pack",
"-p",
request.import_path,
"-importcfg",
import_config.CONFIG_PATH,
# See https://github.com/golang/go/blob/f229e7031a6efb2f23241b5da000c3b3203081d6/src/cmd/go/internal/work/gc.go#L79-L100
# for why Go sets the default to 1.16.
f"-lang=go{request.minimum_go_version or '1.16'}",
]
if symabis_path:
compile_args.extend(["-symabis", symabis_path])
if embedcfg.digest != EMPTY_DIGEST:
compile_args.extend(["-embedcfg", RenderedEmbedConfig.PATH])
if not request.s_file_names:
# If there are no non-Go sources, then pass -complete flag which tells the compiler that the provided
# Go files are the entire package.
compile_args.append("-complete")
relativized_sources = (
f"./{request.subpath}/{name}" if request.subpath else f"./{name}"
for name in request.go_file_names
)
compile_args.extend(["--", *relativized_sources])
compile_result = await Get(
FallibleProcessResult,
GoSdkProcess(
input_digest=input_digest,
command=tuple(compile_args),
description=f"Compile Go package: {request.import_path}",
output_files=("__pkg__.a",),
),
)
if compile_result.exit_code != 0:
return FallibleBuiltGoPackage(
None,
request.import_path,
compile_result.exit_code,
stdout=compile_result.stdout.decode("utf-8"),
stderr=compile_result.stderr.decode("utf-8"),
)
compilation_digest = compile_result.output_digest
if assembly_digests:
assembly_result = await Get(
AssemblyPostCompilation,
AssemblyPostCompilationRequest(
compilation_digest,
assembly_digests,
request.s_file_names,
request.subpath,
),
)
if assembly_result.result.exit_code != 0:
return FallibleBuiltGoPackage(
None,
request.import_path,
assembly_result.result.exit_code,
stdout=assembly_result.result.stdout.decode("utf-8"),
stderr=assembly_result.result.stderr.decode("utf-8"),
)
assert assembly_result.merged_output_digest
compilation_digest = assembly_result.merged_output_digest
path_prefix = os.path.join("__pkgs__", path_safe(request.import_path))
import_paths_to_pkg_a_files[request.import_path] = os.path.join(path_prefix, "__pkg__.a")
output_digest = await Get(Digest, AddPrefix(compilation_digest, path_prefix))
merged_result_digest = await Get(Digest, MergeDigests([*dep_digests, output_digest]))
output = BuiltGoPackage(merged_result_digest, FrozenDict(import_paths_to_pkg_a_files))
return FallibleBuiltGoPackage(output, request.import_path)