async def package_go_binary(field_set: GoBinaryFieldSet) -> BuiltPackage:
main_pkg = await Get(GoBinaryMainPackage, GoBinaryMainPackageRequest(field_set.main))
built_package = await Get(
BuiltGoPackage, BuildGoPackageTargetRequest(main_pkg.address, is_main=True)
)
main_pkg_a_file_path = built_package.import_paths_to_pkg_a_files["main"]
import_config = await Get(
ImportConfig, ImportConfigRequest(built_package.import_paths_to_pkg_a_files)
)
input_digest = await Get(Digest, MergeDigests([built_package.digest, import_config.digest]))
output_filename = PurePath(field_set.output_path.value_or_default(file_ending=None))
binary = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=input_digest,
archives=(main_pkg_a_file_path,),
import_config_path=import_config.CONFIG_PATH,
output_filename=f"./{output_filename.name}",
description=f"Link Go binary for {field_set.address}",
),
)
renamed_output_digest = await Get(Digest, AddPrefix(binary.digest, str(output_filename.parent)))
artifact = BuiltPackageArtifact(relpath=str(output_filename))
return BuiltPackage(renamed_output_digest, (artifact,))
async def generate_from_file(request: GoCodegenBuildFilesRequest) -> FallibleBuildGoPackageRequest:
content = dedent(
"""\
package gen
import "fmt"
import "github.com/google/uuid"
func Quote(s string) string {
uuid.SetClockSequence(-1) // A trivial line to use uuid.
return fmt.Sprintf(">> %s <<", s)
}
"""
)
digest = await Get(Digest, CreateDigest([FileContent("codegen/f.go", content.encode())]))
deps = await Get(Addresses, DependenciesRequest(request.target[Dependencies]))
assert len(deps) == 1
assert deps[0].generated_name == "github.com/google/uuid"
thirdparty_dep = await Get(FallibleBuildGoPackageRequest, BuildGoPackageTargetRequest(deps[0]))
assert thirdparty_dep.request is not None
return FallibleBuildGoPackageRequest(
request=BuildGoPackageRequest(
import_path="codegen.com/gen",
digest=digest,
dir_path="codegen",
go_file_names=("f.go",),
s_file_names=(),
direct_dependencies=(thirdparty_dep.request,),
minimum_go_version=None,
),
import_path="codegen.com/gen",
)
async def check_go(request: GoCheckRequest) -> CheckResults:
build_requests = await MultiGet(
Get(FallibleBuildGoPackageRequest,
BuildGoPackageTargetRequest(field_set.address))
for field_set in request.field_sets)
invalid_requests = []
valid_requests = []
for fallible_request in build_requests:
if fallible_request.request is None:
invalid_requests.append(fallible_request)
else:
valid_requests.append(fallible_request.request)
build_results = await MultiGet(
Get(FallibleBuiltGoPackage, BuildGoPackageRequest, request)
for request in valid_requests)
# NB: We don't pass stdout/stderr as it will have already been rendered as streaming.
exit_code = next(
(
result.exit_code # type: ignore[attr-defined]
for result in (*build_results, *invalid_requests)
if result.exit_code != 0 # type: ignore[attr-defined]
),
0,
)
return CheckResults([CheckResult(exit_code, "", "")],
checker_name=request.name)
def assert_pkg_target_built(
rule_runner: RuleRunner,
addr: Address,
*,
expected_import_path: str,
expected_dir_path: str,
expected_direct_dependency_import_paths: list[str],
expected_transitive_dependency_import_paths: list[str],
expected_go_file_names: list[str],
) -> None:
build_request = rule_runner.request(BuildGoPackageRequest, [BuildGoPackageTargetRequest(addr)])
assert build_request.import_path == expected_import_path
assert build_request.dir_path == expected_dir_path
assert build_request.go_file_names == tuple(expected_go_file_names)
assert not build_request.s_file_names
assert [
dep.import_path for dep in build_request.direct_dependencies
] == expected_direct_dependency_import_paths
assert_built(
rule_runner,
build_request,
expected_import_paths=[
expected_import_path,
*expected_direct_dependency_import_paths,
*expected_transitive_dependency_import_paths,
],
)
def test_build_invalid_target(rule_runner: RuleRunner) -> None:
rule_runner.write_files(
{
"go.mod": dedent(
"""\
module example.com/greeter
go 1.17
"""
),
"BUILD": "go_mod(name='mod')",
"direct/f.go": "invalid!!!",
"direct/BUILD": "go_package()",
"dep/f.go": "invalid!!!",
"dep/BUILD": "go_package()",
"uses_dep/f.go": dedent(
"""\
package uses_dep
import "example.com/greeter/dep"
func Hello() {
dep.Foo("Hello world!")
}
"""
),
"uses_dep/BUILD": "go_package()",
}
)
direct_build_request = rule_runner.request(
FallibleBuildGoPackageRequest, [BuildGoPackageTargetRequest(Address("direct"))]
)
assert direct_build_request.request is None
assert direct_build_request.exit_code == 1
assert "direct/f.go:1:1: expected 'package', found invalid\n" in (
direct_build_request.stderr or ""
)
dep_build_request = rule_runner.request(
FallibleBuildGoPackageRequest, [BuildGoPackageTargetRequest(Address("uses_dep"))]
)
assert dep_build_request.request is None
assert dep_build_request.exit_code == 1
assert "dep/f.go:1:1: expected 'package', found invalid\n" in (dep_build_request.stderr or "")
async def run_go_tests(
field_set: GoTestFieldSet, test_subsystem: TestSubsystem, go_test_subsystem: GoTestSubsystem
) -> TestResult:
maybe_pkg_info, wrapped_target = await MultiGet(
Get(FallibleFirstPartyPkgInfo, FirstPartyPkgInfoRequest(field_set.address)),
Get(WrappedTarget, Address, field_set.address),
)
if maybe_pkg_info.info is None:
assert maybe_pkg_info.stderr is not None
return TestResult(
exit_code=maybe_pkg_info.exit_code,
stdout="",
stderr=maybe_pkg_info.stderr,
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=field_set.address,
output_setting=test_subsystem.output,
)
pkg_info = maybe_pkg_info.info
target = wrapped_target.target
import_path = target[GoImportPathField].value
testmain = await Get(
GeneratedTestMain,
GenerateTestMainRequest(
pkg_info.digest,
FrozenOrderedSet(
os.path.join(".", pkg_info.subpath, name) for name in pkg_info.test_files
),
FrozenOrderedSet(
os.path.join(".", pkg_info.subpath, name) for name in pkg_info.xtest_files
),
import_path=import_path,
),
)
if not testmain.has_tests and not testmain.has_xtests:
# Nothing to do so return an empty result.
# TODO: There should really be a "skipped entirely" mechanism for `TestResult`.
return TestResult(
exit_code=0,
stdout="",
stderr="",
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=field_set.address,
output_setting=test_subsystem.output,
)
# Construct the build request for the package under test.
maybe_test_pkg_build_request = await Get(
FallibleBuildGoPackageRequest,
BuildGoPackageTargetRequest(field_set.address, for_tests=True),
)
if maybe_test_pkg_build_request.request is None:
assert maybe_test_pkg_build_request.stderr is not None
return TestResult(
exit_code=maybe_test_pkg_build_request.exit_code,
stdout="",
stderr=maybe_test_pkg_build_request.stderr,
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=field_set.address,
output_setting=test_subsystem.output,
)
test_pkg_build_request = maybe_test_pkg_build_request.request
main_direct_deps = [test_pkg_build_request]
if testmain.has_xtests:
# Build a synthetic package for xtests where the import path is the same as the package under test
# but with "_test" appended.
#
# Subset the direct dependencies to only the dependencies used by the xtest code. (Dependency
# inference will have included all of the regular, test, and xtest dependencies of the package in
# the build graph.) Moreover, ensure that any import of the package under test is on the _test_
# version of the package that was just built.
dep_by_import_path = {
dep.import_path: dep for dep in test_pkg_build_request.direct_dependencies
}
direct_dependencies: OrderedSet[BuildGoPackageRequest] = OrderedSet()
for xtest_import in pkg_info.xtest_imports:
if xtest_import == pkg_info.import_path:
direct_dependencies.add(test_pkg_build_request)
elif xtest_import in dep_by_import_path:
direct_dependencies.add(dep_by_import_path[xtest_import])
xtest_pkg_build_request = BuildGoPackageRequest(
import_path=f"{import_path}_test",
digest=pkg_info.digest,
subpath=pkg_info.subpath,
go_file_names=pkg_info.xtest_files,
s_file_names=(), # TODO: Are there .s files for xtest?
direct_dependencies=tuple(direct_dependencies),
minimum_go_version=pkg_info.minimum_go_version,
)
main_direct_deps.append(xtest_pkg_build_request)
# Generate the synthetic main package which imports the test and/or xtest packages.
maybe_built_main_pkg = await Get(
FallibleBuiltGoPackage,
BuildGoPackageRequest(
import_path="main",
digest=testmain.digest,
subpath="",
go_file_names=(GeneratedTestMain.TEST_MAIN_FILE,),
s_file_names=(),
direct_dependencies=tuple(main_direct_deps),
minimum_go_version=pkg_info.minimum_go_version,
),
)
if maybe_built_main_pkg.output is None:
assert maybe_built_main_pkg.stderr is not None
return TestResult(
exit_code=maybe_built_main_pkg.exit_code,
stdout="",
stderr=maybe_built_main_pkg.stderr,
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=field_set.address,
output_setting=test_subsystem.output,
)
built_main_pkg = maybe_built_main_pkg.output
main_pkg_a_file_path = built_main_pkg.import_paths_to_pkg_a_files["main"]
import_config = await Get(
ImportConfig, ImportConfigRequest(built_main_pkg.import_paths_to_pkg_a_files)
)
input_digest = await Get(Digest, MergeDigests([built_main_pkg.digest, import_config.digest]))
binary = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=input_digest,
archives=(main_pkg_a_file_path,),
import_config_path=import_config.CONFIG_PATH,
output_filename="./test_runner", # TODO: Name test binary the way that `go` does?
description=f"Link Go test binary for {field_set.address}",
),
)
cache_scope = (
ProcessCacheScope.PER_SESSION if test_subsystem.force else ProcessCacheScope.SUCCESSFUL
)
result = await Get(
FallibleProcessResult,
Process(
["./test_runner", *transform_test_args(go_test_subsystem.args)],
input_digest=binary.digest,
description=f"Run Go tests: {field_set.address}",
cache_scope=cache_scope,
level=LogLevel.INFO,
),
)
return TestResult.from_fallible_process_result(result, field_set.address, test_subsystem.output)
async def setup_full_package_build_request(
request: _SetupGoProtobufPackageBuildRequest,
protoc: Protoc,
go_protoc_plugin: _SetupGoProtocPlugin,
package_mapping: ImportPathToPackages,
go_protobuf_mapping: GoProtobufImportPathMapping,
analyzer: PackageAnalyzerSetup,
) -> FallibleBuildGoPackageRequest:
output_dir = "_generated_files"
protoc_relpath = "__protoc"
protoc_go_plugin_relpath = "__protoc_gen_go"
transitive_targets, downloaded_protoc_binary, empty_output_dir = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)),
Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(output_dir)])),
)
all_sources = await Get(
SourceFiles,
SourceFilesRequest(
sources_fields=(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSourceField, ),
enable_codegen=True,
),
)
source_roots, input_digest = await MultiGet(
Get(SourceRootsResult, SourceRootsRequest,
SourceRootsRequest.for_files(all_sources.files)),
Get(Digest,
MergeDigests([all_sources.snapshot.digest, empty_output_dir])),
)
source_root_paths = sorted(
{sr.path
for sr in source_roots.path_to_root.values()})
pkg_sources = await MultiGet(
Get(SourcesPaths, SourcesPathsRequest(tgt[ProtobufSourceField]))
for tgt in transitive_targets.roots)
pkg_files = sorted({f for ps in pkg_sources for f in ps.files})
maybe_grpc_plugin_args = []
if any(
tgt.get(ProtobufGrpcToggleField).value
for tgt in transitive_targets.roots):
maybe_grpc_plugin_args = [
f"--go-grpc_out={output_dir}",
"--go-grpc_opt=paths=source_relative",
]
gen_result = await Get(
FallibleProcessResult,
Process(
argv=[
os.path.join(protoc_relpath, downloaded_protoc_binary.exe),
f"--plugin=go={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go')}",
f"--plugin=go-grpc={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go-grpc')}",
f"--go_out={output_dir}",
"--go_opt=paths=source_relative",
*(f"--proto_path={source_root}"
for source_root in source_root_paths),
*maybe_grpc_plugin_args,
*pkg_files,
],
# Note: Necessary or else --plugin option needs absolute path.
env={"PATH": protoc_go_plugin_relpath},
input_digest=input_digest,
immutable_input_digests={
protoc_relpath: downloaded_protoc_binary.digest,
protoc_go_plugin_relpath: go_protoc_plugin.digest,
},
description=f"Generating Go sources from {request.import_path}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
if gen_result.exit_code != 0:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=gen_result.exit_code,
stderr=gen_result.stderr.decode(),
)
# Ensure that the generated files are in a single package directory.
gen_sources = await Get(Snapshot, Digest, gen_result.output_digest)
files_by_dir = group_by_dir(gen_sources.files)
if len(files_by_dir) != 1:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=1,
stderr=
("Expected Go files generated from Protobuf sources to be output to a single directory.\n"
f"- import path: {request.import_path}\n"
f"- protobuf files: {', '.join(pkg_files)}"),
)
gen_dir = list(files_by_dir.keys())[0]
# Analyze the generated sources.
input_digest = await Get(
Digest, MergeDigests([gen_sources.digest, analyzer.digest]))
result = await Get(
FallibleProcessResult,
Process(
(analyzer.path, gen_dir),
input_digest=input_digest,
description=
f"Determine metadata for generated Go package for {request.import_path}",
level=LogLevel.DEBUG,
env={"CGO_ENABLED": "0"},
),
)
# Parse the metadata from the analysis.
fallible_analysis = FallibleFirstPartyPkgAnalysis.from_process_result(
result,
dir_path=gen_dir,
import_path=request.import_path,
minimum_go_version="",
description_of_source=
f"Go package generated from protobuf targets `{', '.join(str(addr) for addr in request.addresses)}`",
)
if not fallible_analysis.analysis:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=fallible_analysis.exit_code,
stderr=fallible_analysis.stderr,
)
analysis = fallible_analysis.analysis
# Obtain build requests for third-party dependencies.
# TODO: Consider how to merge this code with existing dependency inference code.
dep_build_request_addrs: list[Address] = []
for dep_import_path in (*analysis.imports, *analysis.test_imports,
*analysis.xtest_imports):
# Infer dependencies on other Go packages.
candidate_addresses = package_mapping.mapping.get(dep_import_path)
if candidate_addresses:
# TODO: Use explicit dependencies to disambiguate? This should never happen with Go backend though.
if len(candidate_addresses) > 1:
return FallibleBuildGoPackageRequest(
request=None,
import_path=request.import_path,
exit_code=result.exit_code,
stderr=
(f"Multiple addresses match import of `{dep_import_path}`.\n"
f"addresses: {', '.join(str(a) for a in candidate_addresses)}"
),
)
dep_build_request_addrs.extend(candidate_addresses)
# Infer dependencies on other generated Go sources.
go_protobuf_candidate_addresses = go_protobuf_mapping.mapping.get(
dep_import_path)
if go_protobuf_candidate_addresses:
dep_build_request_addrs.extend(go_protobuf_candidate_addresses)
dep_build_requests = await MultiGet(
Get(BuildGoPackageRequest, BuildGoPackageTargetRequest(addr))
for addr in dep_build_request_addrs)
return FallibleBuildGoPackageRequest(
request=BuildGoPackageRequest(
import_path=request.import_path,
digest=gen_sources.digest,
dir_path=analysis.dir_path,
go_file_names=analysis.go_files,
s_file_names=analysis.s_files,
direct_dependencies=dep_build_requests,
minimum_go_version=analysis.minimum_go_version,
),
import_path=request.import_path,
)
async def run_go_tests(field_set: GoTestFieldSet,
test_subsystem: TestSubsystem,
go_test_subsystem: GoTestSubsystem) -> TestResult:
maybe_pkg_analysis, maybe_pkg_digest, dependencies = await MultiGet(
Get(FallibleFirstPartyPkgAnalysis,
FirstPartyPkgAnalysisRequest(field_set.address)),
Get(FallibleFirstPartyPkgDigest,
FirstPartyPkgDigestRequest(field_set.address)),
Get(Targets, DependenciesRequest(field_set.dependencies)),
)
def compilation_failure(exit_code: int, stdout: str | None,
stderr: str | None) -> TestResult:
return TestResult(
exit_code=exit_code,
stdout=stdout or "",
stderr=stderr or "",
stdout_digest=EMPTY_FILE_DIGEST,
stderr_digest=EMPTY_FILE_DIGEST,
address=field_set.address,
output_setting=test_subsystem.output,
result_metadata=None,
)
if maybe_pkg_analysis.analysis is None:
assert maybe_pkg_analysis.stderr is not None
return compilation_failure(maybe_pkg_analysis.exit_code, None,
maybe_pkg_analysis.stderr)
if maybe_pkg_digest.pkg_digest is None:
assert maybe_pkg_digest.stderr is not None
return compilation_failure(maybe_pkg_digest.exit_code, None,
maybe_pkg_digest.stderr)
pkg_analysis = maybe_pkg_analysis.analysis
pkg_digest = maybe_pkg_digest.pkg_digest
import_path = pkg_analysis.import_path
testmain = await Get(
GeneratedTestMain,
GenerateTestMainRequest(
pkg_digest.digest,
FrozenOrderedSet(
os.path.join(".", pkg_analysis.dir_path, name)
for name in pkg_analysis.test_go_files),
FrozenOrderedSet(
os.path.join(".", pkg_analysis.dir_path, name)
for name in pkg_analysis.xtest_go_files),
import_path,
field_set.address,
),
)
if testmain.failed_exit_code_and_stderr is not None:
_exit_code, _stderr = testmain.failed_exit_code_and_stderr
return compilation_failure(_exit_code, None, _stderr)
if not testmain.has_tests and not testmain.has_xtests:
return TestResult.skip(field_set.address,
output_setting=test_subsystem.output)
# Construct the build request for the package under test.
maybe_test_pkg_build_request = await Get(
FallibleBuildGoPackageRequest,
BuildGoPackageTargetRequest(field_set.address, for_tests=True),
)
if maybe_test_pkg_build_request.request is None:
assert maybe_test_pkg_build_request.stderr is not None
return compilation_failure(maybe_test_pkg_build_request.exit_code,
None, maybe_test_pkg_build_request.stderr)
test_pkg_build_request = maybe_test_pkg_build_request.request
main_direct_deps = [test_pkg_build_request]
if testmain.has_xtests:
# Build a synthetic package for xtests where the import path is the same as the package under test
# but with "_test" appended.
#
# Subset the direct dependencies to only the dependencies used by the xtest code. (Dependency
# inference will have included all of the regular, test, and xtest dependencies of the package in
# the build graph.) Moreover, ensure that any import of the package under test is on the _test_
# version of the package that was just built.
dep_by_import_path = {
dep.import_path: dep
for dep in test_pkg_build_request.direct_dependencies
}
direct_dependencies: OrderedSet[BuildGoPackageRequest] = OrderedSet()
for xtest_import in pkg_analysis.xtest_imports:
if xtest_import == pkg_analysis.import_path:
direct_dependencies.add(test_pkg_build_request)
elif xtest_import in dep_by_import_path:
direct_dependencies.add(dep_by_import_path[xtest_import])
xtest_pkg_build_request = BuildGoPackageRequest(
import_path=f"{import_path}_test",
digest=pkg_digest.digest,
dir_path=pkg_analysis.dir_path,
go_file_names=pkg_analysis.xtest_go_files,
s_file_names=(), # TODO: Are there .s files for xtest?
direct_dependencies=tuple(direct_dependencies),
minimum_go_version=pkg_analysis.minimum_go_version,
embed_config=pkg_digest.xtest_embed_config,
)
main_direct_deps.append(xtest_pkg_build_request)
# Generate the synthetic main package which imports the test and/or xtest packages.
maybe_built_main_pkg = await Get(
FallibleBuiltGoPackage,
BuildGoPackageRequest(
import_path="main",
digest=testmain.digest,
dir_path="",
go_file_names=(GeneratedTestMain.TEST_MAIN_FILE, ),
s_file_names=(),
direct_dependencies=tuple(main_direct_deps),
minimum_go_version=pkg_analysis.minimum_go_version,
),
)
if maybe_built_main_pkg.output is None:
assert maybe_built_main_pkg.stderr is not None
return compilation_failure(maybe_built_main_pkg.exit_code,
maybe_built_main_pkg.stdout,
maybe_built_main_pkg.stderr)
built_main_pkg = maybe_built_main_pkg.output
main_pkg_a_file_path = built_main_pkg.import_paths_to_pkg_a_files["main"]
import_config = await Get(
ImportConfig,
ImportConfigRequest(built_main_pkg.import_paths_to_pkg_a_files))
linker_input_digest = await Get(
Digest, MergeDigests([built_main_pkg.digest, import_config.digest]))
binary = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=linker_input_digest,
archives=(main_pkg_a_file_path, ),
import_config_path=import_config.CONFIG_PATH,
output_filename=
"./test_runner", # TODO: Name test binary the way that `go` does?
description=f"Link Go test binary for {field_set.address}",
),
)
# To emulate Go's test runner, we set the working directory to the path of the `go_package`.
# This allows tests to open dependencies on `file` targets regardless of where they are
# located. See https://dave.cheney.net/2016/05/10/test-fixtures-in-go.
working_dir = field_set.address.spec_path
binary_with_prefix, files_sources = await MultiGet(
Get(Digest, AddPrefix(binary.digest, working_dir)),
Get(
SourceFiles,
SourceFilesRequest(
(dep.get(SourcesField) for dep in dependencies),
for_sources_types=(FileSourceField, ),
enable_codegen=True,
),
),
)
test_input_digest = await Get(
Digest,
MergeDigests((binary_with_prefix, files_sources.snapshot.digest)))
cache_scope = (ProcessCacheScope.PER_SESSION
if test_subsystem.force else ProcessCacheScope.SUCCESSFUL)
result = await Get(
FallibleProcessResult,
Process(
[
"./test_runner",
*transform_test_args(go_test_subsystem.args,
field_set.timeout.value),
],
input_digest=test_input_digest,
description=f"Run Go tests: {field_set.address}",
cache_scope=cache_scope,
working_directory=working_dir,
level=LogLevel.DEBUG,
),
)
return TestResult.from_fallible_process_result(result, field_set.address,
test_subsystem.output)