async def setup_assembly_pre_compilation( request: AssemblyPreCompilationRequest, ) -> AssemblyPreCompilation: # From Go tooling comments: # # Supply an empty go_asm.h as if the compiler had been run. -symabis parsing is lax enough # that we don't need the actual definitions that would appear in go_asm.h. # # See https://go-review.googlesource.com/c/go/+/146999/8/src/cmd/go/internal/work/gc.go go_asm_h_digest = await Get(Digest, CreateDigest([FileContent("go_asm.h", b"")])) symabis_input_digest = await Get( Digest, MergeDigests([request.compilation_input, go_asm_h_digest])) symabis_result = await Get( ProcessResult, GoSdkProcess( input_digest=symabis_input_digest, command=( "tool", "asm", "-I", "go/pkg/include", "-gensymabis", "-o", "symabis", "--", *(f"./{request.source_files_subpath}/{name}" for name in request.s_files), ), description="Generate symabis metadata for assembly files.", output_files=("symabis", ), ), ) merged = await Get( Digest, MergeDigests([request.compilation_input, symabis_result.output_digest]), ) assembly_results = await MultiGet( Get( ProcessResult, GoSdkProcess( input_digest=request.compilation_input, command=( "tool", "asm", "-I", "go/pkg/include", "-o", f"./{request.source_files_subpath}/{PurePath(s_file).with_suffix('.o')}", f"./{request.source_files_subpath}/{s_file}", ), description=f"Assemble {s_file}", output_files= (f"./{request.source_files_subpath}/{PurePath(s_file).with_suffix('.o')}", ), ), ) for s_file in request.s_files) return AssemblyPreCompilation( merged, tuple(result.output_digest for result in assembly_results))
async def link_assembly_post_compilation( request: AssemblyPostCompilationRequest, ) -> AssemblyPostCompilation: merged_digest, asm_tool_id = await MultiGet( Get( Digest, MergeDigests( [request.compilation_result, *request.assembly_digests])), # Use `go tool asm` tool ID since `go tool pack` does not have a version argument. Get(GoSdkToolIDResult, GoSdkToolIDRequest("asm")), ) pack_result = await Get( FallibleProcessResult, GoSdkProcess( input_digest=merged_digest, command=( "tool", "pack", "r", "__pkg__.a", *(f"./{request.dir_path}/{PurePath(name).with_suffix('.o')}" for name in request.s_files), ), env={ "__PANTS_GO_ASM_TOOL_ID": asm_tool_id.tool_id, }, description= f"Link assembly files to Go package archive for {request.dir_path}", output_files=("__pkg__.a", ), ), ) return AssemblyPostCompilation( pack_result, pack_result.output_digest if pack_result.exit_code == 0 else None)
async def resolve_external_go_package( request: ResolveExternalGoPackageRequest, ) -> ResolvedGoPackage: wrapped_target = await Get(WrappedTarget, Address, request.address) target = wrapped_target.target import_path = target[GoExternalPackageImportPathField].value module_path = target[GoExternalModulePathField].value module_version = target[GoExternalModuleVersionField].value module = await Get( DownloadedExternalModule, DownloadExternalModuleRequest(path=module_path, version=module_version), ) assert import_path.startswith(module_path) subpath = import_path[len(module_path):] result = await Get( ProcessResult, GoSdkProcess( input_digest=module.digest, command=("list", "-json", f"./{subpath}"), description="Resolve _go_external_package metadata.", ), ) metadata = json.loads(result.stdout) return ResolvedGoPackage.from_metadata( metadata, import_path=import_path, address=request.address, module_address=None, module_path=module_path, module_version=module_version, )
async def link_go_binary(request: LinkGoBinaryRequest) -> LinkedGoBinary: link_tool_id = await Get(GoSdkToolIDResult, GoSdkToolIDRequest("link")) result = await Get( ProcessResult, GoSdkProcess( input_digest=request.input_digest, command=( "tool", "link", "-importcfg", request.import_config_path, "-o", request.output_filename, "-buildmode=exe", # seen in `go build -x` output *request.archives, ), env={ "__PANTS_GO_LINK_TOOL_ID": link_tool_id.tool_id, }, description=f"Link Go binary: {request.output_filename}", output_files=(request.output_filename, ), ), ) return LinkedGoBinary(result.output_digest)
async def link_assembly_post_compilation( request: AssemblyPostCompilationRequest, ) -> AssemblyPostCompilation: merged_digest = await Get( Digest, MergeDigests([request.compilation_result, *request.assembly_digests]) ) pack_result = await Get( FallibleProcessResult, GoSdkProcess( input_digest=merged_digest, command=( "tool", "pack", "r", "__pkg__.a", *( f"./{request.dir_path}/{PurePath(name).with_suffix('.o')}" for name in request.s_files ), ), description=f"Link assembly files to Go package archive for {request.dir_path}", output_files=("__pkg__.a",), ), ) return AssemblyPostCompilation( pack_result, pack_result.output_digest if pack_result.exit_code == 0 else None )
async def determine_go_mod_info(request: GoModInfoRequest, ) -> GoModInfo: wrapped_target = await Get(WrappedTarget, Address, request.address) sources_field = wrapped_target.target[GoModSourcesField] go_mod_path = sources_field.go_mod_path go_mod_dir = os.path.dirname(go_mod_path) # Get the `go.mod` (and `go.sum`) and strip so the file has no directory prefix. hydrated_sources = await Get(HydratedSources, HydrateSourcesRequest(sources_field)) sources_digest = hydrated_sources.snapshot.digest stripped_source_get = Get(Digest, RemovePrefix(sources_digest, go_mod_dir)) mod_json_get = Get( ProcessResult, GoSdkProcess( command=("mod", "edit", "-json"), input_digest=sources_digest, working_dir=go_mod_dir, description=f"Parse {go_mod_path}", ), ) mod_json, stripped_sources = await MultiGet(mod_json_get, stripped_source_get) module_metadata = json.loads(mod_json.stdout) return GoModInfo( import_path=module_metadata["Module"]["Path"], digest=sources_digest, stripped_digest=stripped_sources, minimum_go_version=module_metadata.get("Go"), )
def test_fuzz_target_supported(rule_runner: RuleRunner) -> None: go_version_result = rule_runner.request( ProcessResult, [GoSdkProcess(["version"], description="Get `go` version.")]) if "go1.18" not in go_version_result.stdout.decode(): pytest.skip("Skipping because Go SDK is not 1.18 or higher.") rule_runner.write_files({ "foo/BUILD": "go_mod(name='mod')\ngo_package()", "foo/go.mod": "module foo", "foo/fuzz_test.go": textwrap.dedent(""" package foo import ( "testing" ) func FuzzFoo(f *testing.F) { f.Add("foo") f.Fuzz(func(t *testing.T, v string) { if v != "foo" { t.Fail() } }) } """), }) tgt = rule_runner.get_target(Address("foo")) result = rule_runner.request(TestResult, [GoTestFieldSet.create(tgt)]) assert result.exit_code == 0 assert "PASS: FuzzFoo" in result.stdout
async def determine_go_mod_info(request: GoModInfoRequest, ) -> GoModInfo: if isinstance(request.source, Address): wrapped_target = await Get( WrappedTarget, WrappedTargetRequest(request.source, description_of_origin="<go mod info rule>"), ) sources_field = wrapped_target.target[GoModSourcesField] else: sources_field = request.source go_mod_path = sources_field.go_mod_path go_mod_dir = os.path.dirname(go_mod_path) # Get the `go.mod` (and `go.sum`) and strip so the file has no directory prefix. hydrated_sources = await Get(HydratedSources, HydrateSourcesRequest(sources_field)) sources_digest = hydrated_sources.snapshot.digest mod_json = await Get( ProcessResult, GoSdkProcess( command=("mod", "edit", "-json"), input_digest=sources_digest, working_dir=go_mod_dir, description=f"Parse {go_mod_path}", ), ) module_metadata = json.loads(mod_json.stdout) return GoModInfo( import_path=module_metadata["Module"]["Path"], digest=sources_digest, mod_path=go_mod_path, minimum_go_version=module_metadata.get("Go"), )
async def resolve_go_module( request: ResolveGoModuleRequest, ) -> ResolvedGoModule: wrapped_target = await Get(WrappedTarget, Address, request.address) target = wrapped_target.target sources = await Get(SourceFiles, SourceFilesRequest([target.get(GoModuleSources)])) flattened_sources_snapshot = await Get( Snapshot, RemovePrefix(sources.snapshot.digest, request.address.spec_path)) # Parse the go.mod for the module path and minimum Go version. parse_result = await Get( ProcessResult, GoSdkProcess( input_digest=flattened_sources_snapshot.digest, command=("mod", "edit", "-json"), description=f"Parse go.mod for {request.address}.", ), ) module_metadata = json.loads(parse_result.stdout) module_path = module_metadata["Module"]["Path"] minimum_go_version = module_metadata.get( "Go", "1.16" ) # TODO: Figure out better default if missing. Use the SDKs version versus this hard-code. # Resolve the dependencies in the go.mod. list_modules_result = await Get( ProcessResult, GoSdkProcess( input_digest=flattened_sources_snapshot.digest, command=("list", "-m", "-json", "all"), description=f"List modules in build of {request.address}.", ), ) modules = parse_module_descriptors(list_modules_result.stdout) return ResolvedGoModule( target=target, import_path=module_path, minimum_go_version=minimum_go_version, modules=FrozenOrderedSet(modules), digest=flattened_sources_snapshot. digest, # TODO: Is this a resolved version? Need to update for go-resolve goal? )
async def determine_go_std_lib_imports() -> GoStdLibImports: list_result = await Get( ProcessResult, GoSdkProcess( command=("list", "-json", "std"), description="Ask Go for its available import paths", absolutify_goroot=False, ), ) result = {} for package_descriptor in ijson.items(list_result.stdout, "", multiple_values=True): import_path = package_descriptor.get("ImportPath") target = package_descriptor.get("Target") if not import_path or not target: continue result[import_path] = target return GoStdLibImports(result)
async def run_go_vet(request: GoVetRequest, go_vet_subsystem: GoVetSubsystem) -> LintResults: if go_vet_subsystem.skip: return LintResults([], linter_name=request.name) source_files = await Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in request.field_sets), ) owning_go_mods = await MultiGet( Get(OwningGoMod, OwningGoModRequest(field_set.address)) for field_set in request.field_sets) owning_go_mod_addresses = {x.address for x in owning_go_mods} go_mod_infos = await MultiGet( Get(GoModInfo, GoModInfoRequest(address)) for address in owning_go_mod_addresses) input_digest = await Get( Digest, MergeDigests([ source_files.snapshot.digest, *(info.digest for info in set(go_mod_infos)) ]), ) package_dirs = sorted( {os.path.dirname(f) for f in source_files.snapshot.files}) process_result = await Get( FallibleProcessResult, GoSdkProcess( ("vet", *(f"./{p}" for p in package_dirs)), input_digest=input_digest, description= f"Run `go vet` on {pluralize(len(source_files.snapshot.files), 'file')}.", ), ) result = LintResult.from_fallible_process_result(process_result) return LintResults([result], linter_name=request.name)
async def link_go_binary(request: LinkGoBinaryRequest) -> LinkedGoBinary: result = await Get( ProcessResult, GoSdkProcess( input_digest=request.input_digest, command=( "tool", "link", "-importcfg", request.import_config_path, "-o", request.output_filename, "-buildmode=exe", # seen in `go build -x` output *request.archives, ), description=f"Link Go binary: {request.output_filename}", output_files=(request.output_filename, ), ), ) return LinkedGoBinary(result.output_digest)
async def analyze_go_third_party_module( request: AnalyzeThirdPartyModuleRequest, analyzer: PackageAnalyzerSetup, golang_subsystem: GolangSubsystem, ) -> AnalyzedThirdPartyModule: # Download the module. download_result = await Get( ProcessResult, GoSdkProcess( ("mod", "download", "-json", f"{request.name}@{request.version}"), input_digest=request.go_mod_digest, # for go.sum working_dir=os.path.dirname(request.go_mod_path) if request.go_mod_path else None, # Allow downloads of the module sources. allow_downloads=True, output_directories=("gopath", ), description=f"Download Go module {request.name}@{request.version}.", ), ) if len(download_result.stdout) == 0: raise AssertionError( f"Expected output from `go mod download` for {request.name}@{request.version}." ) module_metadata = json.loads(download_result.stdout) module_sources_relpath = strip_sandbox_prefix(module_metadata["Dir"], "gopath/") go_mod_relpath = strip_sandbox_prefix(module_metadata["GoMod"], "gopath/") # Subset the output directory to just the module sources and go.mod (which may be generated). module_sources_snapshot = await Get( Snapshot, DigestSubset( download_result.output_digest, PathGlobs( [f"{module_sources_relpath}/**", go_mod_relpath], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin= f"the download of Go module {request.name}@{request.version}", ), ), ) # Determine directories with potential Go packages in them. candidate_package_dirs = [] files_by_dir = group_by_dir(p for p in module_sources_snapshot.files if p.startswith(module_sources_relpath)) for maybe_pkg_dir, files in files_by_dir.items(): # Skip directories where "testdata" would end up in the import path. # See https://github.com/golang/go/blob/f005df8b582658d54e63d59953201299d6fee880/src/go/build/build.go#L580-L585 if "testdata" in maybe_pkg_dir.split("/"): continue # Consider directories with at least one `.go` file as package candidates. if any(f for f in files if f.endswith(".go")): candidate_package_dirs.append(maybe_pkg_dir) candidate_package_dirs.sort() # Analyze all of the packages in this module. analyzer_relpath = "__analyzer" analysis_result = await Get( ProcessResult, Process( [ os.path.join(analyzer_relpath, analyzer.path), *candidate_package_dirs ], input_digest=module_sources_snapshot.digest, immutable_input_digests={ analyzer_relpath: analyzer.digest, }, description= f"Analyze metadata for Go third-party module: {request.name}@{request.version}", level=LogLevel.DEBUG, env={"CGO_ENABLED": "0"}, ), ) if len(analysis_result.stdout) == 0: return AnalyzedThirdPartyModule(FrozenOrderedSet()) package_analysis_gets = [] for pkg_path, pkg_json in zip( candidate_package_dirs, ijson.items(analysis_result.stdout, "", multiple_values=True)): package_analysis_gets.append( Get( FallibleThirdPartyPkgAnalysis, AnalyzeThirdPartyPackageRequest( pkg_json=_freeze_json_dict(pkg_json), module_sources_digest=module_sources_snapshot.digest, module_sources_path=module_sources_relpath, module_import_path=request.name, package_path=pkg_path, minimum_go_version=request.minimum_go_version, ), )) analyzed_packages_fallible = await MultiGet(package_analysis_gets) analyzed_packages = [ pkg.analysis for pkg in analyzed_packages_fallible if pkg.analysis and pkg.exit_code == 0 ] return AnalyzedThirdPartyModule(FrozenOrderedSet(analyzed_packages))
async def build_target(request: BuildGoPackageRequest, ) -> BuiltGoPackage: wrapped_target = await Get(WrappedTarget, Address, request.address) target = wrapped_target.target if is_first_party_package_target(target): source_files, resolved_package = await MultiGet( Get( SourceFiles, SourceFilesRequest((target[GoPackageSources], )), ), Get(ResolvedGoPackage, ResolveGoPackageRequest(address=target.address)), ) source_files_digest = source_files.snapshot.digest source_files_subpath = target.address.spec_path elif is_third_party_package_target(target): module_path = target[GoExternalModulePathField].value module, resolved_package = await MultiGet( Get( DownloadedExternalModule, DownloadExternalModuleRequest( path=module_path, version=target[GoExternalModuleVersionField].value, ), ), Get(ResolvedGoPackage, ResolveExternalGoPackageRequest(address=request.address)), ) source_files_digest = module.digest source_files_subpath = resolved_package.import_path[len(module_path):] else: raise ValueError( f"Unknown how to build Go target at address {request.address}.") dependencies = await Get(Addresses, DependenciesRequest(field=target[Dependencies])) dependencies_targets = await Get(UnexpandedTargets, Addresses(dependencies)) buildable_dependencies_targets = [ dep_tgt for dep_tgt in dependencies_targets if is_first_party_package_target(dep_tgt) or is_third_party_package_target(dep_tgt) ] built_go_deps = await MultiGet( Get(BuiltGoPackage, BuildGoPackageRequest(tgt.address)) for tgt in buildable_dependencies_targets) gathered_imports = await Get( GatheredImports, GatherImportsRequest(packages=FrozenOrderedSet(built_go_deps), include_stdlib=True), ) import_path = resolved_package.import_path if request.is_main: import_path = "main" compile_command = [ "tool", "compile", "-p", import_path, "-importcfg", "./importcfg", "-pack", "-o", "__pkg__.a", ] input_digest = await Get( Digest, MergeDigests([gathered_imports.digest, source_files_digest])) assembly_digests = None if resolved_package.s_files: assembly_setup = await Get( AssemblyPreCompilation, AssemblyPreCompilationRequest(input_digest, resolved_package.s_files, source_files_subpath), ) input_digest = assembly_setup.merged_compilation_input_digest assembly_digests = assembly_setup.assembly_digests compile_command.extend(assembly_setup.EXTRA_COMPILATION_ARGS) compile_command.append("--") compile_command.extend(f"./{source_files_subpath}/{name}" for name in resolved_package.go_files) result = await Get( ProcessResult, GoSdkProcess( input_digest=input_digest, command=tuple(compile_command), description= f"Compile Go package with {pluralize(len(resolved_package.go_files), 'file')}. [{request.address}]", output_files=("__pkg__.a", ), ), ) output_digest = result.output_digest if assembly_digests: assembly_result = await Get( AssemblyPostCompilation, AssemblyPostCompilationRequest(output_digest, assembly_digests, resolved_package.s_files, source_files_subpath), ) output_digest = assembly_result.merged_output_digest return BuiltGoPackage(import_path=import_path, object_digest=output_digest)
async def package_go_binary(field_set: GoBinaryFieldSet, ) -> BuiltPackage: main_address = field_set.main_address.value or "" main_go_package_address = await Get( Address, AddressInput, AddressInput.parse(main_address, relative_to=field_set.address.spec_path), ) wrapped_main_go_package_target = await Get(WrappedTarget, Address, main_go_package_address) main_go_package_target = wrapped_main_go_package_target.target built_main_go_package = await Get( BuiltGoPackage, BuildGoPackageRequest(address=main_go_package_target.address, is_main=True)) transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(roots=[main_go_package_target.address])) buildable_deps = [ tgt for tgt in transitive_targets.dependencies if is_first_party_package_target(tgt) or is_third_party_package_target(tgt) ] built_transitive_go_deps_requests = [ Get(BuiltGoPackage, BuildGoPackageRequest(address=tgt.address)) for tgt in buildable_deps ] built_transitive_go_deps = await MultiGet(built_transitive_go_deps_requests ) gathered_imports = await Get( GatheredImports, GatherImportsRequest( packages=FrozenOrderedSet(built_transitive_go_deps), include_stdlib=True, ), ) input_digest = await Get( Digest, MergeDigests( [gathered_imports.digest, built_main_go_package.object_digest])) output_filename = PurePath( field_set.output_path.value_or_default(file_ending=None)) result = await Get( ProcessResult, GoSdkProcess( input_digest=input_digest, command=( "tool", "link", "-importcfg", "./importcfg", "-o", f"./{output_filename.name}", "-buildmode=exe", # seen in `go build -x` output "./__pkg__.a", ), description="Link Go binary.", output_files=(f"./{output_filename.name}", ), ), ) renamed_output_digest = await Get( Digest, AddPrefix(result.output_digest, str(output_filename.parent))) artifact = BuiltPackageArtifact(relpath=str(output_filename)) return BuiltPackage(digest=renamed_output_digest, artifacts=(artifact, ))
async def setup_go_protoc_plugin(platform: Platform) -> _SetupGoProtocPlugin: go_mod_digest = await Get( Digest, CreateDigest([ FileContent("go.mod", GO_PROTOBUF_GO_MOD.encode()), FileContent("go.sum", GO_PROTOBUF_GO_SUM.encode()), ]), ) download_sources_result = await Get( ProcessResult, GoSdkProcess( ["mod", "download", "all"], input_digest=go_mod_digest, output_directories=("gopath", ), description="Download Go `protoc` plugin sources.", allow_downloads=True, ), ) go_plugin_build_result, go_grpc_plugin_build_result = await MultiGet( Get( ProcessResult, GoSdkProcess( [ "install", "google.golang.org/protobuf/cmd/[email protected]" ], input_digest=download_sources_result.output_digest, output_files=["gopath/bin/protoc-gen-go"], description="Build Go protobuf plugin for `protoc`.", platform=platform, ), ), Get( ProcessResult, GoSdkProcess( [ "install", "google.golang.org/grpc/cmd/[email protected]", ], input_digest=download_sources_result.output_digest, output_files=["gopath/bin/protoc-gen-go-grpc"], description="Build Go gRPC protobuf plugin for `protoc`.", platform=platform, ), ), ) if go_plugin_build_result.output_digest == EMPTY_DIGEST: raise AssertionError( f"Failed to build protoc-gen-go:\n" f"stdout:\n{go_plugin_build_result.stdout.decode()}\n\n" f"stderr:\n{go_plugin_build_result.stderr.decode()}") if go_grpc_plugin_build_result.output_digest == EMPTY_DIGEST: raise AssertionError( f"Failed to build protoc-gen-go-grpc:\n" f"stdout:\n{go_grpc_plugin_build_result.stdout.decode()}\n\n" f"stderr:\n{go_grpc_plugin_build_result.stderr.decode()}") merged_output_digests = await Get( Digest, MergeDigests([ go_plugin_build_result.output_digest, go_grpc_plugin_build_result.output_digest ]), ) plugin_digest = await Get( Digest, RemovePrefix(merged_output_digests, "gopath/bin")) return _SetupGoProtocPlugin(plugin_digest)
async def download_external_module( request: DownloadExternalModuleRequest, ) -> DownloadedExternalModule: result = await Get( ProcessResult, GoSdkProcess( input_digest=EMPTY_DIGEST, command=("mod", "download", "-json", f"{request.path}@{request.version}"), description= f"Download external Go module at {request.path}@{request.version}.", output_directories=("gopath", ), ), ) # Decode the module metadata. metadata = json.loads(result.stdout) # Find the path within the digest where the source was downloaded. The path will have a sandbox-specific # prefix that we need to strip down to the `gopath` path component. absolute_source_path = metadata["Dir"] gopath_index = absolute_source_path.index("gopath/") source_path = absolute_source_path[gopath_index:] source_digest = await Get( Digest, DigestSubset( result.output_digest, PathGlobs( [f"{source_path}/**"], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin= f"the DownloadExternalModuleRequest for {request.path}@{request.version}", ), ), ) source_snapshot_stripped = await Get( Snapshot, RemovePrefix(source_digest, source_path)) if "go.mod" not in source_snapshot_stripped.files: # There was no go.mod in the downloaded source. Use the generated go.mod from the go tooling which # was returned in the module metadata. go_mod_absolute_path = metadata.get("GoMod") if not go_mod_absolute_path: raise ValueError( f"No go.mod was provided in download of Go external module {request.path}@{request.version}, " "and the module metadata did not identify a generated go.mod file to use instead." ) gopath_index = go_mod_absolute_path.index("gopath/") go_mod_path = go_mod_absolute_path[gopath_index:] go_mod_digest = await Get( Digest, DigestSubset( result.output_digest, PathGlobs( [f"{go_mod_path}"], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin= f"the DownloadExternalModuleRequest for {request.path}@{request.version}", ), ), ) go_mod_digest_stripped = await Get( Digest, RemovePrefix(go_mod_digest, os.path.dirname(go_mod_path))) # There should now be one file in the digest. Create a digest where that file is named go.mod # and then merge it into the sources. contents = await Get(DigestContents, Digest, go_mod_digest_stripped) assert len(contents) == 1 go_mod_only_digest = await Get( Digest, CreateDigest( [FileContent( path="go.mod", content=contents[0].content, )]), ) source_digest_final = await Get( Digest, MergeDigests([go_mod_only_digest, source_snapshot_stripped.digest])) else: # If the module download has a go.mod, then just use the sources as is. source_digest_final = source_snapshot_stripped.digest return DownloadedExternalModule( path=request.path, version=request.version, digest=source_digest_final, sum=metadata["Sum"], go_mod_sum=metadata["GoModSum"], )
async def resolve_external_module_to_go_packages( request: ResolveExternalGoModuleToPackagesRequest, ) -> ResolveExternalGoModuleToPackagesResult: module_path = request.path assert module_path module_version = request.version assert module_version downloaded_module = await Get( DownloadedExternalModule, DownloadExternalModuleRequest(path=module_path, version=module_version), ) sources_digest = await Get( Digest, AddPrefix(downloaded_module.digest, "__sources__")) # TODO: Super hacky merge of go.sum from both digests. We should really just pass in the fully-resolved # go.sum and use that, but this allows the go.sum from the downloaded module to have some effect. Not sure # if that is right call, but hackity hack! left_digest_contents = await Get(DigestContents, Digest, sources_digest) left_go_sum_contents = b"" for fc in left_digest_contents: if fc.path == "__sources__/go.sum": left_go_sum_contents = fc.content break go_sum_only_digest = await Get( Digest, DigestSubset(request.go_sum_digest, PathGlobs(["go.sum"]))) go_sum_prefixed_digest = await Get( Digest, AddPrefix(go_sum_only_digest, "__sources__")) right_digest_contents = await Get(DigestContents, Digest, go_sum_prefixed_digest) right_go_sum_contents = b"" for fc in right_digest_contents: if fc.path == "__sources__/go.sum": right_go_sum_contents = fc.content break go_sum_contents = left_go_sum_contents + b"\n" + right_go_sum_contents go_sum_digest = await Get( Digest, CreateDigest([ FileContent( path="__sources__/go.sum", content=go_sum_contents, ) ]), ) sources_digest_no_go_sum = await Get( Digest, DigestSubset( sources_digest, PathGlobs( ["!__sources__/go.sum", "__sources__/**"], conjunction=GlobExpansionConjunction.all_match, glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="FUNKY", ), ), ) input_digest = await Get( Digest, MergeDigests([sources_digest_no_go_sum, go_sum_digest])) result = await Get( ProcessResult, GoSdkProcess( input_digest=input_digest, command=("list", "-json", "./..."), working_dir="__sources__", description= f"Resolve packages in Go external module {module_path}@{module_version}", ), ) packages: OrderedSet[ResolvedGoPackage] = OrderedSet() for metadata in ijson.items(result.stdout, "", multiple_values=True): package = ResolvedGoPackage.from_metadata( metadata, module_path=module_path, module_version=module_version) packages.add(package) return ResolveExternalGoModuleToPackagesResult( packages=FrozenOrderedSet(packages))
async def resolve_go_package( request: ResolveGoPackageRequest, ) -> ResolvedGoPackage: wrapped_target, owning_go_module_result = await MultiGet( Get(WrappedTarget, Address, request.address), Get(ResolvedOwningGoModule, FindNearestGoModuleRequest(request.address.spec_path)), ) target = wrapped_target.target if not owning_go_module_result.module_address: raise ValueError(f"The go_package at address {request.address} has no owning go_module.") resolved_go_module = await Get( ResolvedGoModule, ResolveGoModuleRequest(owning_go_module_result.module_address) ) go_module_spec_path = resolved_go_module.target.address.spec_path assert request.address.spec_path.startswith(go_module_spec_path) spec_subpath = request.address.spec_path[len(go_module_spec_path) :] # Compute the import_path for this go_package. import_path_field = target.get(GoImportPath) if import_path_field and import_path_field.value: # Use any explicit import path set on the `go_package` target. import_path = import_path_field.value else: # Otherwise infer the import path from the owning `go_module` target. The inferred import path will be the # module's import path plus any subdirectories in the spec_path between the go_module and go_package target. if not resolved_go_module.import_path: raise ValueError( f"Unable to infer import path for the `go_package` at address {request.address} " f"because the owning go_module at address {resolved_go_module.target.address} " "does not have an import path defined nor could one be inferred." ) import_path = f"{resolved_go_module.import_path}/" if spec_subpath.startswith("/"): import_path += spec_subpath[1:] else: import_path += spec_subpath sources = await Get( SourceFiles, SourceFilesRequest( [ target.get(GoPackageSources), resolved_go_module.target.get(GoModuleSources), ] ), ) result = await Get( ProcessResult, GoSdkProcess( input_digest=sources.snapshot.digest, command=("list", "-json", f"./{spec_subpath}"), description="Resolve go_package metadata.", working_dir=resolved_go_module.target.address.spec_path, ), ) metadata = json.loads(result.stdout) return ResolvedGoPackage.from_metadata( metadata, import_path=import_path, address=request.address, module_address=owning_go_module_result.module_address, )
async def download_and_analyze_third_party_packages( request: AllThirdPartyPackagesRequest, ) -> AllThirdPartyPackages: # NB: We download all modules to GOPATH=$(pwd)/gopath. Running `go list ...` from $(pwd) would # naively try analyzing the contents of the GOPATH like they were first-party packages. This # results in errors like this: # # package <import_path>/gopath/pkg/mod/golang.org/x/[email protected]/unicode: can only use # path@version syntax with 'go get' and 'go install' in module-aware mode # # Instead, we run `go list` from a subdirectory of the chroot. It can still access the # contents of `GOPATH`, but won't incorrectly treat its contents as first-party packages. go_mod_prefix = "go_mod_prefix" go_mod_prefixed_digest = await Get( Digest, AddPrefix(request.go_mod_stripped_digest, go_mod_prefix)) list_argv = ( "list", # This rule can't modify `go.mod` and `go.sum` as it would require mutating the workspace. # Instead, we expect them to be well-formed already. # # It would be convenient to set `-mod=mod` to allow edits, and then compare the resulting # files to the input so that we could print a diff for the user to know how to update. But # `-mod=mod` results in more packages being downloaded and added to `go.mod` than is # actually necessary. # TODO: nice error when `go.mod` and `go.sum` would need to change. Right now, it's a # message from Go and won't be intuitive for Pants users what to do. "-mod=readonly", # There may be some packages in the transitive closure that cannot be built, but we should # not blow up Pants. # # For example, a package that sets the special value `package documentation` and has no # source files would naively error due to `build constraints exclude all Go files`, even # though we should not error on that package. "-e", "-json", # This matches all packages. `all` only matches first-party packages and complains that # there are no `.go` files. "...", ) list_result = await Get( ProcessResult, GoSdkProcess( command=list_argv, # TODO: make this more descriptive: point to the actual `go_mod` target or path. description= "Run `go list` to download and analyze all third-party Go packages", input_digest=go_mod_prefixed_digest, output_directories=("gopath/pkg/mod", ), working_dir=go_mod_prefix, allow_downloads=True, ), ) stripped_result_digest = await Get( Digest, RemovePrefix(list_result.output_digest, "gopath/pkg/mod")) all_digest_subset_gets = [] all_pkg_info_kwargs = [] all_failed_pkg_info = [] for pkg_json in ijson.items(list_result.stdout, "", multiple_values=True): if "Standard" in pkg_json: continue import_path = pkg_json["ImportPath"] maybe_error, maybe_failed_pkg_info = maybe_raise_or_create_error_or_create_failed_pkg_info( pkg_json, import_path) if maybe_failed_pkg_info: all_failed_pkg_info.append(maybe_failed_pkg_info) continue dir_path = strip_prefix(strip_v2_chroot_path(pkg_json["Dir"]), "gopath/pkg/mod/") all_pkg_info_kwargs.append( dict( import_path=import_path, subpath=dir_path, imports=tuple(pkg_json.get("Imports", ())), go_files=tuple(pkg_json.get("GoFiles", ())), s_files=tuple(pkg_json.get("SFiles", ())), minimum_go_version=pkg_json.get("Module", {}).get("GoVersion"), error=maybe_error, )) all_digest_subset_gets.append( Get( Digest, DigestSubset( stripped_result_digest, PathGlobs( [os.path.join(dir_path, "*")], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin=f"downloading {import_path}", ), ), )) all_digest_subsets = await MultiGet(all_digest_subset_gets) import_path_to_info = { pkg_info_kwargs["import_path"]: ThirdPartyPkgInfo(digest=digest_subset, **pkg_info_kwargs) for pkg_info_kwargs, digest_subset in zip(all_pkg_info_kwargs, all_digest_subsets) } import_path_to_info.update( (pkg_info.import_path, pkg_info) for pkg_info in all_failed_pkg_info) return AllThirdPartyPackages(list_result.output_digest, FrozenDict(import_path_to_info))
async def setup_assembly_pre_compilation( request: AssemblyPreCompilationRequest, goroot: GoRoot, ) -> FallibleAssemblyPreCompilation: # From Go tooling comments: # # Supply an empty go_asm.h as if the compiler had been run. -symabis parsing is lax enough # that we don't need the actual definitions that would appear in go_asm.h. # # See https://go-review.googlesource.com/c/go/+/146999/8/src/cmd/go/internal/work/gc.go go_asm_h_digest, asm_tool_id = await MultiGet( Get(Digest, CreateDigest([FileContent("go_asm.h", b"")])), Get(GoSdkToolIDResult, GoSdkToolIDRequest("asm")), ) symabis_input_digest = await Get( Digest, MergeDigests([request.compilation_input, go_asm_h_digest])) symabis_result = await Get( FallibleProcessResult, GoSdkProcess( input_digest=symabis_input_digest, command=( "tool", "asm", "-I", os.path.join(goroot.path, "pkg", "include"), "-gensymabis", "-o", "symabis", "--", *(f"./{request.dir_path}/{name}" for name in request.s_files), ), env={ "__PANTS_GO_ASM_TOOL_ID": asm_tool_id.tool_id, }, description= f"Generate symabis metadata for assembly files for {request.dir_path}", output_files=("symabis", ), ), ) if symabis_result.exit_code != 0: return FallibleAssemblyPreCompilation( None, symabis_result.exit_code, symabis_result.stderr.decode("utf-8")) merged = await Get( Digest, MergeDigests([request.compilation_input, symabis_result.output_digest]), ) assembly_results = await MultiGet( Get( FallibleProcessResult, GoSdkProcess( input_digest=request.compilation_input, command=( "tool", "asm", "-I", os.path.join(goroot.path, "pkg", "include"), "-o", f"./{request.dir_path}/{PurePath(s_file).with_suffix('.o')}", f"./{request.dir_path}/{s_file}", ), env={ "__PANTS_GO_ASM_TOOL_ID": asm_tool_id.tool_id, }, description=f"Assemble {s_file} with Go", output_files= (f"./{request.dir_path}/{PurePath(s_file).with_suffix('.o')}", ), ), ) for s_file in request.s_files) exit_code = max(result.exit_code for result in assembly_results) if exit_code != 0: stdout = "\n\n".join( result.stdout.decode("utf-8") for result in assembly_results if result.stdout) stderr = "\n\n".join( result.stderr.decode("utf-8") for result in assembly_results if result.stderr) return FallibleAssemblyPreCompilation(None, exit_code, stdout, stderr) return FallibleAssemblyPreCompilation( AssemblyPreCompilation( merged, tuple(result.output_digest for result in assembly_results)))
async def build_go_package(request: BuildGoPackageRequest) -> FallibleBuiltGoPackage: maybe_built_deps = await MultiGet( Get(FallibleBuiltGoPackage, BuildGoPackageRequest, build_request) for build_request in request.direct_dependencies ) import_paths_to_pkg_a_files: dict[str, str] = {} dep_digests = [] for maybe_dep in maybe_built_deps: if maybe_dep.output is None: return dataclasses.replace( maybe_dep, import_path=request.import_path, dependency_failed=True ) dep = maybe_dep.output import_paths_to_pkg_a_files.update(dep.import_paths_to_pkg_a_files) dep_digests.append(dep.digest) merged_deps_digest, import_config, embedcfg = await MultiGet( Get(Digest, MergeDigests(dep_digests)), Get(ImportConfig, ImportConfigRequest(FrozenDict(import_paths_to_pkg_a_files))), Get(RenderedEmbedConfig, RenderEmbedConfigRequest(request.embed_config)), ) input_digest = await Get( Digest, MergeDigests([merged_deps_digest, import_config.digest, embedcfg.digest, request.digest]), ) assembly_digests = None symabis_path = None if request.s_file_names: assembly_setup = await Get( FallibleAssemblyPreCompilation, AssemblyPreCompilationRequest(input_digest, request.s_file_names, request.subpath), ) if assembly_setup.result is None: return FallibleBuiltGoPackage( None, request.import_path, assembly_setup.exit_code, stdout=assembly_setup.stdout, stderr=assembly_setup.stderr, ) input_digest = assembly_setup.result.merged_compilation_input_digest assembly_digests = assembly_setup.result.assembly_digests symabis_path = "./symabis" compile_args = [ "tool", "compile", "-o", "__pkg__.a", "-pack", "-p", request.import_path, "-importcfg", import_config.CONFIG_PATH, # See https://github.com/golang/go/blob/f229e7031a6efb2f23241b5da000c3b3203081d6/src/cmd/go/internal/work/gc.go#L79-L100 # for why Go sets the default to 1.16. f"-lang=go{request.minimum_go_version or '1.16'}", ] if symabis_path: compile_args.extend(["-symabis", symabis_path]) if embedcfg.digest != EMPTY_DIGEST: compile_args.extend(["-embedcfg", RenderedEmbedConfig.PATH]) if not request.s_file_names: # If there are no non-Go sources, then pass -complete flag which tells the compiler that the provided # Go files are the entire package. compile_args.append("-complete") relativized_sources = ( f"./{request.subpath}/{name}" if request.subpath else f"./{name}" for name in request.go_file_names ) compile_args.extend(["--", *relativized_sources]) compile_result = await Get( FallibleProcessResult, GoSdkProcess( input_digest=input_digest, command=tuple(compile_args), description=f"Compile Go package: {request.import_path}", output_files=("__pkg__.a",), ), ) if compile_result.exit_code != 0: return FallibleBuiltGoPackage( None, request.import_path, compile_result.exit_code, stdout=compile_result.stdout.decode("utf-8"), stderr=compile_result.stderr.decode("utf-8"), ) compilation_digest = compile_result.output_digest if assembly_digests: assembly_result = await Get( AssemblyPostCompilation, AssemblyPostCompilationRequest( compilation_digest, assembly_digests, request.s_file_names, request.subpath, ), ) if assembly_result.result.exit_code != 0: return FallibleBuiltGoPackage( None, request.import_path, assembly_result.result.exit_code, stdout=assembly_result.result.stdout.decode("utf-8"), stderr=assembly_result.result.stderr.decode("utf-8"), ) assert assembly_result.merged_output_digest compilation_digest = assembly_result.merged_output_digest path_prefix = os.path.join("__pkgs__", path_safe(request.import_path)) import_paths_to_pkg_a_files[request.import_path] = os.path.join(path_prefix, "__pkg__.a") output_digest = await Get(Digest, AddPrefix(compilation_digest, path_prefix)) merged_result_digest = await Get(Digest, MergeDigests([*dep_digests, output_digest])) output = BuiltGoPackage(merged_result_digest, FrozenDict(import_paths_to_pkg_a_files)) return FallibleBuiltGoPackage(output, request.import_path)
async def analyze_module_dependencies( request: ModuleDescriptorsRequest) -> ModuleDescriptors: # List the modules used directly and indirectly by this module. # # This rule can't modify `go.mod` and `go.sum` as it would require mutating the workspace. # Instead, we expect them to be well-formed already. # # Options used: # - `-mod=readonly': It would be convenient to set `-mod=mod` to allow edits, and then compare the # resulting files to the input so that we could print a diff for the user to know how to update. But # `-mod=mod` results in more packages being downloaded and added to `go.mod` than is # actually necessary. # TODO: nice error when `go.mod` and `go.sum` would need to change. Right now, it's a # message from Go and won't be intuitive for Pants users what to do. # - `-e` is used to not fail if one of the modules is problematic. There may be some packages in the transitive # closure that cannot be built, but we should not blow up Pants. For example, a package that sets the # special value `package documentation` and has no source files would naively error due to # `build constraints exclude all Go files`, even though we should not error on that package. mod_list_result = await Get( ProcessResult, GoSdkProcess( command=["list", "-mod=readonly", "-e", "-m", "-json", "all"], input_digest=request.digest, output_directories=("gopath", ), working_dir=request.path if request.path else None, # Allow downloads of the module metadata (i.e., go.mod files). allow_downloads=True, description="Analyze Go module dependencies.", ), ) if len(mod_list_result.stdout) == 0: return ModuleDescriptors(FrozenOrderedSet(), EMPTY_DIGEST) descriptors: dict[tuple[str, str], ModuleDescriptor] = {} for mod_json in ijson.items(mod_list_result.stdout, "", multiple_values=True): # Skip the first-party module being analyzed. if "Main" in mod_json and mod_json["Main"]: continue if "Replace" in mod_json: # TODO: Reject local file path replacements? Gazelle does. name = mod_json["Replace"]["Path"] version = mod_json["Replace"]["Version"] else: name = mod_json["Path"] version = mod_json["Version"] descriptors[(name, version)] = ModuleDescriptor( import_path=mod_json["Path"], name=name, version=version, indirect=mod_json.get("Indirect", False), minimum_go_version=mod_json.get("GoVersion"), ) # TODO: Augment the modules with go.sum entries? # Gazelle does this, mainly to store the sum on the go_repository rule. We could store it (or its # absence) to be able to download sums automatically. return ModuleDescriptors(FrozenOrderedSet(descriptors.values()), mod_list_result.output_digest)