async def package_go_binary(field_set: GoBinaryFieldSet) -> BuiltPackage: main_pkg = await Get(GoBinaryMainPackage, GoBinaryMainPackageRequest(field_set.main)) built_package = await Get( BuiltGoPackage, BuildGoPackageTargetRequest(main_pkg.address, is_main=True) ) main_pkg_a_file_path = built_package.import_paths_to_pkg_a_files["main"] import_config = await Get( ImportConfig, ImportConfigRequest(built_package.import_paths_to_pkg_a_files) ) input_digest = await Get(Digest, MergeDigests([built_package.digest, import_config.digest])) output_filename = PurePath(field_set.output_path.value_or_default(file_ending=None)) binary = await Get( LinkedGoBinary, LinkGoBinaryRequest( input_digest=input_digest, archives=(main_pkg_a_file_path,), import_config_path=import_config.CONFIG_PATH, output_filename=f"./{output_filename.name}", description=f"Link Go binary for {field_set.address}", ), ) renamed_output_digest = await Get(Digest, AddPrefix(binary.digest, str(output_filename.parent))) artifact = BuiltPackageArtifact(relpath=str(output_filename)) return BuiltPackage(renamed_output_digest, (artifact,))
async def generate_from_file(request: GoCodegenBuildFilesRequest) -> FallibleBuildGoPackageRequest: content = dedent( """\ package gen import "fmt" import "github.com/google/uuid" func Quote(s string) string { uuid.SetClockSequence(-1) // A trivial line to use uuid. return fmt.Sprintf(">> %s <<", s) } """ ) digest = await Get(Digest, CreateDigest([FileContent("codegen/f.go", content.encode())])) deps = await Get(Addresses, DependenciesRequest(request.target[Dependencies])) assert len(deps) == 1 assert deps[0].generated_name == "github.com/google/uuid" thirdparty_dep = await Get(FallibleBuildGoPackageRequest, BuildGoPackageTargetRequest(deps[0])) assert thirdparty_dep.request is not None return FallibleBuildGoPackageRequest( request=BuildGoPackageRequest( import_path="codegen.com/gen", digest=digest, dir_path="codegen", go_file_names=("f.go",), s_file_names=(), direct_dependencies=(thirdparty_dep.request,), minimum_go_version=None, ), import_path="codegen.com/gen", )
async def check_go(request: GoCheckRequest) -> CheckResults: build_requests = await MultiGet( Get(FallibleBuildGoPackageRequest, BuildGoPackageTargetRequest(field_set.address)) for field_set in request.field_sets) invalid_requests = [] valid_requests = [] for fallible_request in build_requests: if fallible_request.request is None: invalid_requests.append(fallible_request) else: valid_requests.append(fallible_request.request) build_results = await MultiGet( Get(FallibleBuiltGoPackage, BuildGoPackageRequest, request) for request in valid_requests) # NB: We don't pass stdout/stderr as it will have already been rendered as streaming. exit_code = next( ( result.exit_code # type: ignore[attr-defined] for result in (*build_results, *invalid_requests) if result.exit_code != 0 # type: ignore[attr-defined] ), 0, ) return CheckResults([CheckResult(exit_code, "", "")], checker_name=request.name)
def assert_pkg_target_built( rule_runner: RuleRunner, addr: Address, *, expected_import_path: str, expected_dir_path: str, expected_direct_dependency_import_paths: list[str], expected_transitive_dependency_import_paths: list[str], expected_go_file_names: list[str], ) -> None: build_request = rule_runner.request(BuildGoPackageRequest, [BuildGoPackageTargetRequest(addr)]) assert build_request.import_path == expected_import_path assert build_request.dir_path == expected_dir_path assert build_request.go_file_names == tuple(expected_go_file_names) assert not build_request.s_file_names assert [ dep.import_path for dep in build_request.direct_dependencies ] == expected_direct_dependency_import_paths assert_built( rule_runner, build_request, expected_import_paths=[ expected_import_path, *expected_direct_dependency_import_paths, *expected_transitive_dependency_import_paths, ], )
def test_build_invalid_target(rule_runner: RuleRunner) -> None: rule_runner.write_files( { "go.mod": dedent( """\ module example.com/greeter go 1.17 """ ), "BUILD": "go_mod(name='mod')", "direct/f.go": "invalid!!!", "direct/BUILD": "go_package()", "dep/f.go": "invalid!!!", "dep/BUILD": "go_package()", "uses_dep/f.go": dedent( """\ package uses_dep import "example.com/greeter/dep" func Hello() { dep.Foo("Hello world!") } """ ), "uses_dep/BUILD": "go_package()", } ) direct_build_request = rule_runner.request( FallibleBuildGoPackageRequest, [BuildGoPackageTargetRequest(Address("direct"))] ) assert direct_build_request.request is None assert direct_build_request.exit_code == 1 assert "direct/f.go:1:1: expected 'package', found invalid\n" in ( direct_build_request.stderr or "" ) dep_build_request = rule_runner.request( FallibleBuildGoPackageRequest, [BuildGoPackageTargetRequest(Address("uses_dep"))] ) assert dep_build_request.request is None assert dep_build_request.exit_code == 1 assert "dep/f.go:1:1: expected 'package', found invalid\n" in (dep_build_request.stderr or "")
async def run_go_tests( field_set: GoTestFieldSet, test_subsystem: TestSubsystem, go_test_subsystem: GoTestSubsystem ) -> TestResult: maybe_pkg_info, wrapped_target = await MultiGet( Get(FallibleFirstPartyPkgInfo, FirstPartyPkgInfoRequest(field_set.address)), Get(WrappedTarget, Address, field_set.address), ) if maybe_pkg_info.info is None: assert maybe_pkg_info.stderr is not None return TestResult( exit_code=maybe_pkg_info.exit_code, stdout="", stderr=maybe_pkg_info.stderr, stdout_digest=EMPTY_FILE_DIGEST, stderr_digest=EMPTY_FILE_DIGEST, address=field_set.address, output_setting=test_subsystem.output, ) pkg_info = maybe_pkg_info.info target = wrapped_target.target import_path = target[GoImportPathField].value testmain = await Get( GeneratedTestMain, GenerateTestMainRequest( pkg_info.digest, FrozenOrderedSet( os.path.join(".", pkg_info.subpath, name) for name in pkg_info.test_files ), FrozenOrderedSet( os.path.join(".", pkg_info.subpath, name) for name in pkg_info.xtest_files ), import_path=import_path, ), ) if not testmain.has_tests and not testmain.has_xtests: # Nothing to do so return an empty result. # TODO: There should really be a "skipped entirely" mechanism for `TestResult`. return TestResult( exit_code=0, stdout="", stderr="", stdout_digest=EMPTY_FILE_DIGEST, stderr_digest=EMPTY_FILE_DIGEST, address=field_set.address, output_setting=test_subsystem.output, ) # Construct the build request for the package under test. maybe_test_pkg_build_request = await Get( FallibleBuildGoPackageRequest, BuildGoPackageTargetRequest(field_set.address, for_tests=True), ) if maybe_test_pkg_build_request.request is None: assert maybe_test_pkg_build_request.stderr is not None return TestResult( exit_code=maybe_test_pkg_build_request.exit_code, stdout="", stderr=maybe_test_pkg_build_request.stderr, stdout_digest=EMPTY_FILE_DIGEST, stderr_digest=EMPTY_FILE_DIGEST, address=field_set.address, output_setting=test_subsystem.output, ) test_pkg_build_request = maybe_test_pkg_build_request.request main_direct_deps = [test_pkg_build_request] if testmain.has_xtests: # Build a synthetic package for xtests where the import path is the same as the package under test # but with "_test" appended. # # Subset the direct dependencies to only the dependencies used by the xtest code. (Dependency # inference will have included all of the regular, test, and xtest dependencies of the package in # the build graph.) Moreover, ensure that any import of the package under test is on the _test_ # version of the package that was just built. dep_by_import_path = { dep.import_path: dep for dep in test_pkg_build_request.direct_dependencies } direct_dependencies: OrderedSet[BuildGoPackageRequest] = OrderedSet() for xtest_import in pkg_info.xtest_imports: if xtest_import == pkg_info.import_path: direct_dependencies.add(test_pkg_build_request) elif xtest_import in dep_by_import_path: direct_dependencies.add(dep_by_import_path[xtest_import]) xtest_pkg_build_request = BuildGoPackageRequest( import_path=f"{import_path}_test", digest=pkg_info.digest, subpath=pkg_info.subpath, go_file_names=pkg_info.xtest_files, s_file_names=(), # TODO: Are there .s files for xtest? direct_dependencies=tuple(direct_dependencies), minimum_go_version=pkg_info.minimum_go_version, ) main_direct_deps.append(xtest_pkg_build_request) # Generate the synthetic main package which imports the test and/or xtest packages. maybe_built_main_pkg = await Get( FallibleBuiltGoPackage, BuildGoPackageRequest( import_path="main", digest=testmain.digest, subpath="", go_file_names=(GeneratedTestMain.TEST_MAIN_FILE,), s_file_names=(), direct_dependencies=tuple(main_direct_deps), minimum_go_version=pkg_info.minimum_go_version, ), ) if maybe_built_main_pkg.output is None: assert maybe_built_main_pkg.stderr is not None return TestResult( exit_code=maybe_built_main_pkg.exit_code, stdout="", stderr=maybe_built_main_pkg.stderr, stdout_digest=EMPTY_FILE_DIGEST, stderr_digest=EMPTY_FILE_DIGEST, address=field_set.address, output_setting=test_subsystem.output, ) built_main_pkg = maybe_built_main_pkg.output main_pkg_a_file_path = built_main_pkg.import_paths_to_pkg_a_files["main"] import_config = await Get( ImportConfig, ImportConfigRequest(built_main_pkg.import_paths_to_pkg_a_files) ) input_digest = await Get(Digest, MergeDigests([built_main_pkg.digest, import_config.digest])) binary = await Get( LinkedGoBinary, LinkGoBinaryRequest( input_digest=input_digest, archives=(main_pkg_a_file_path,), import_config_path=import_config.CONFIG_PATH, output_filename="./test_runner", # TODO: Name test binary the way that `go` does? description=f"Link Go test binary for {field_set.address}", ), ) cache_scope = ( ProcessCacheScope.PER_SESSION if test_subsystem.force else ProcessCacheScope.SUCCESSFUL ) result = await Get( FallibleProcessResult, Process( ["./test_runner", *transform_test_args(go_test_subsystem.args)], input_digest=binary.digest, description=f"Run Go tests: {field_set.address}", cache_scope=cache_scope, level=LogLevel.INFO, ), ) return TestResult.from_fallible_process_result(result, field_set.address, test_subsystem.output)
async def setup_full_package_build_request( request: _SetupGoProtobufPackageBuildRequest, protoc: Protoc, go_protoc_plugin: _SetupGoProtocPlugin, package_mapping: ImportPathToPackages, go_protobuf_mapping: GoProtobufImportPathMapping, analyzer: PackageAnalyzerSetup, ) -> FallibleBuildGoPackageRequest: output_dir = "_generated_files" protoc_relpath = "__protoc" protoc_go_plugin_relpath = "__protoc_gen_go" transitive_targets, downloaded_protoc_binary, empty_output_dir = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)), Get(DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)), Get(Digest, CreateDigest([Directory(output_dir)])), ) all_sources = await Get( SourceFiles, SourceFilesRequest( sources_fields=(tgt[ProtobufSourceField] for tgt in transitive_targets.closure), for_sources_types=(ProtobufSourceField, ), enable_codegen=True, ), ) source_roots, input_digest = await MultiGet( Get(SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(all_sources.files)), Get(Digest, MergeDigests([all_sources.snapshot.digest, empty_output_dir])), ) source_root_paths = sorted( {sr.path for sr in source_roots.path_to_root.values()}) pkg_sources = await MultiGet( Get(SourcesPaths, SourcesPathsRequest(tgt[ProtobufSourceField])) for tgt in transitive_targets.roots) pkg_files = sorted({f for ps in pkg_sources for f in ps.files}) maybe_grpc_plugin_args = [] if any( tgt.get(ProtobufGrpcToggleField).value for tgt in transitive_targets.roots): maybe_grpc_plugin_args = [ f"--go-grpc_out={output_dir}", "--go-grpc_opt=paths=source_relative", ] gen_result = await Get( FallibleProcessResult, Process( argv=[ os.path.join(protoc_relpath, downloaded_protoc_binary.exe), f"--plugin=go={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go')}", f"--plugin=go-grpc={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go-grpc')}", f"--go_out={output_dir}", "--go_opt=paths=source_relative", *(f"--proto_path={source_root}" for source_root in source_root_paths), *maybe_grpc_plugin_args, *pkg_files, ], # Note: Necessary or else --plugin option needs absolute path. env={"PATH": protoc_go_plugin_relpath}, input_digest=input_digest, immutable_input_digests={ protoc_relpath: downloaded_protoc_binary.digest, protoc_go_plugin_relpath: go_protoc_plugin.digest, }, description=f"Generating Go sources from {request.import_path}.", level=LogLevel.DEBUG, output_directories=(output_dir, ), ), ) if gen_result.exit_code != 0: return FallibleBuildGoPackageRequest( request=None, import_path=request.import_path, exit_code=gen_result.exit_code, stderr=gen_result.stderr.decode(), ) # Ensure that the generated files are in a single package directory. gen_sources = await Get(Snapshot, Digest, gen_result.output_digest) files_by_dir = group_by_dir(gen_sources.files) if len(files_by_dir) != 1: return FallibleBuildGoPackageRequest( request=None, import_path=request.import_path, exit_code=1, stderr= ("Expected Go files generated from Protobuf sources to be output to a single directory.\n" f"- import path: {request.import_path}\n" f"- protobuf files: {', '.join(pkg_files)}"), ) gen_dir = list(files_by_dir.keys())[0] # Analyze the generated sources. input_digest = await Get( Digest, MergeDigests([gen_sources.digest, analyzer.digest])) result = await Get( FallibleProcessResult, Process( (analyzer.path, gen_dir), input_digest=input_digest, description= f"Determine metadata for generated Go package for {request.import_path}", level=LogLevel.DEBUG, env={"CGO_ENABLED": "0"}, ), ) # Parse the metadata from the analysis. fallible_analysis = FallibleFirstPartyPkgAnalysis.from_process_result( result, dir_path=gen_dir, import_path=request.import_path, minimum_go_version="", description_of_source= f"Go package generated from protobuf targets `{', '.join(str(addr) for addr in request.addresses)}`", ) if not fallible_analysis.analysis: return FallibleBuildGoPackageRequest( request=None, import_path=request.import_path, exit_code=fallible_analysis.exit_code, stderr=fallible_analysis.stderr, ) analysis = fallible_analysis.analysis # Obtain build requests for third-party dependencies. # TODO: Consider how to merge this code with existing dependency inference code. dep_build_request_addrs: list[Address] = [] for dep_import_path in (*analysis.imports, *analysis.test_imports, *analysis.xtest_imports): # Infer dependencies on other Go packages. candidate_addresses = package_mapping.mapping.get(dep_import_path) if candidate_addresses: # TODO: Use explicit dependencies to disambiguate? This should never happen with Go backend though. if len(candidate_addresses) > 1: return FallibleBuildGoPackageRequest( request=None, import_path=request.import_path, exit_code=result.exit_code, stderr= (f"Multiple addresses match import of `{dep_import_path}`.\n" f"addresses: {', '.join(str(a) for a in candidate_addresses)}" ), ) dep_build_request_addrs.extend(candidate_addresses) # Infer dependencies on other generated Go sources. go_protobuf_candidate_addresses = go_protobuf_mapping.mapping.get( dep_import_path) if go_protobuf_candidate_addresses: dep_build_request_addrs.extend(go_protobuf_candidate_addresses) dep_build_requests = await MultiGet( Get(BuildGoPackageRequest, BuildGoPackageTargetRequest(addr)) for addr in dep_build_request_addrs) return FallibleBuildGoPackageRequest( request=BuildGoPackageRequest( import_path=request.import_path, digest=gen_sources.digest, dir_path=analysis.dir_path, go_file_names=analysis.go_files, s_file_names=analysis.s_files, direct_dependencies=dep_build_requests, minimum_go_version=analysis.minimum_go_version, ), import_path=request.import_path, )
async def run_go_tests(field_set: GoTestFieldSet, test_subsystem: TestSubsystem, go_test_subsystem: GoTestSubsystem) -> TestResult: maybe_pkg_analysis, maybe_pkg_digest, dependencies = await MultiGet( Get(FallibleFirstPartyPkgAnalysis, FirstPartyPkgAnalysisRequest(field_set.address)), Get(FallibleFirstPartyPkgDigest, FirstPartyPkgDigestRequest(field_set.address)), Get(Targets, DependenciesRequest(field_set.dependencies)), ) def compilation_failure(exit_code: int, stdout: str | None, stderr: str | None) -> TestResult: return TestResult( exit_code=exit_code, stdout=stdout or "", stderr=stderr or "", stdout_digest=EMPTY_FILE_DIGEST, stderr_digest=EMPTY_FILE_DIGEST, address=field_set.address, output_setting=test_subsystem.output, result_metadata=None, ) if maybe_pkg_analysis.analysis is None: assert maybe_pkg_analysis.stderr is not None return compilation_failure(maybe_pkg_analysis.exit_code, None, maybe_pkg_analysis.stderr) if maybe_pkg_digest.pkg_digest is None: assert maybe_pkg_digest.stderr is not None return compilation_failure(maybe_pkg_digest.exit_code, None, maybe_pkg_digest.stderr) pkg_analysis = maybe_pkg_analysis.analysis pkg_digest = maybe_pkg_digest.pkg_digest import_path = pkg_analysis.import_path testmain = await Get( GeneratedTestMain, GenerateTestMainRequest( pkg_digest.digest, FrozenOrderedSet( os.path.join(".", pkg_analysis.dir_path, name) for name in pkg_analysis.test_go_files), FrozenOrderedSet( os.path.join(".", pkg_analysis.dir_path, name) for name in pkg_analysis.xtest_go_files), import_path, field_set.address, ), ) if testmain.failed_exit_code_and_stderr is not None: _exit_code, _stderr = testmain.failed_exit_code_and_stderr return compilation_failure(_exit_code, None, _stderr) if not testmain.has_tests and not testmain.has_xtests: return TestResult.skip(field_set.address, output_setting=test_subsystem.output) # Construct the build request for the package under test. maybe_test_pkg_build_request = await Get( FallibleBuildGoPackageRequest, BuildGoPackageTargetRequest(field_set.address, for_tests=True), ) if maybe_test_pkg_build_request.request is None: assert maybe_test_pkg_build_request.stderr is not None return compilation_failure(maybe_test_pkg_build_request.exit_code, None, maybe_test_pkg_build_request.stderr) test_pkg_build_request = maybe_test_pkg_build_request.request main_direct_deps = [test_pkg_build_request] if testmain.has_xtests: # Build a synthetic package for xtests where the import path is the same as the package under test # but with "_test" appended. # # Subset the direct dependencies to only the dependencies used by the xtest code. (Dependency # inference will have included all of the regular, test, and xtest dependencies of the package in # the build graph.) Moreover, ensure that any import of the package under test is on the _test_ # version of the package that was just built. dep_by_import_path = { dep.import_path: dep for dep in test_pkg_build_request.direct_dependencies } direct_dependencies: OrderedSet[BuildGoPackageRequest] = OrderedSet() for xtest_import in pkg_analysis.xtest_imports: if xtest_import == pkg_analysis.import_path: direct_dependencies.add(test_pkg_build_request) elif xtest_import in dep_by_import_path: direct_dependencies.add(dep_by_import_path[xtest_import]) xtest_pkg_build_request = BuildGoPackageRequest( import_path=f"{import_path}_test", digest=pkg_digest.digest, dir_path=pkg_analysis.dir_path, go_file_names=pkg_analysis.xtest_go_files, s_file_names=(), # TODO: Are there .s files for xtest? direct_dependencies=tuple(direct_dependencies), minimum_go_version=pkg_analysis.minimum_go_version, embed_config=pkg_digest.xtest_embed_config, ) main_direct_deps.append(xtest_pkg_build_request) # Generate the synthetic main package which imports the test and/or xtest packages. maybe_built_main_pkg = await Get( FallibleBuiltGoPackage, BuildGoPackageRequest( import_path="main", digest=testmain.digest, dir_path="", go_file_names=(GeneratedTestMain.TEST_MAIN_FILE, ), s_file_names=(), direct_dependencies=tuple(main_direct_deps), minimum_go_version=pkg_analysis.minimum_go_version, ), ) if maybe_built_main_pkg.output is None: assert maybe_built_main_pkg.stderr is not None return compilation_failure(maybe_built_main_pkg.exit_code, maybe_built_main_pkg.stdout, maybe_built_main_pkg.stderr) built_main_pkg = maybe_built_main_pkg.output main_pkg_a_file_path = built_main_pkg.import_paths_to_pkg_a_files["main"] import_config = await Get( ImportConfig, ImportConfigRequest(built_main_pkg.import_paths_to_pkg_a_files)) linker_input_digest = await Get( Digest, MergeDigests([built_main_pkg.digest, import_config.digest])) binary = await Get( LinkedGoBinary, LinkGoBinaryRequest( input_digest=linker_input_digest, archives=(main_pkg_a_file_path, ), import_config_path=import_config.CONFIG_PATH, output_filename= "./test_runner", # TODO: Name test binary the way that `go` does? description=f"Link Go test binary for {field_set.address}", ), ) # To emulate Go's test runner, we set the working directory to the path of the `go_package`. # This allows tests to open dependencies on `file` targets regardless of where they are # located. See https://dave.cheney.net/2016/05/10/test-fixtures-in-go. working_dir = field_set.address.spec_path binary_with_prefix, files_sources = await MultiGet( Get(Digest, AddPrefix(binary.digest, working_dir)), Get( SourceFiles, SourceFilesRequest( (dep.get(SourcesField) for dep in dependencies), for_sources_types=(FileSourceField, ), enable_codegen=True, ), ), ) test_input_digest = await Get( Digest, MergeDigests((binary_with_prefix, files_sources.snapshot.digest))) cache_scope = (ProcessCacheScope.PER_SESSION if test_subsystem.force else ProcessCacheScope.SUCCESSFUL) result = await Get( FallibleProcessResult, Process( [ "./test_runner", *transform_test_args(go_test_subsystem.args, field_set.timeout.value), ], input_digest=test_input_digest, description=f"Run Go tests: {field_set.address}", cache_scope=cache_scope, working_directory=working_dir, level=LogLevel.DEBUG, ), ) return TestResult.from_fallible_process_result(result, field_set.address, test_subsystem.output)