async def setup_assembly_pre_compilation(
request: AssemblyPreCompilationRequest, ) -> AssemblyPreCompilation:
# From Go tooling comments:
#
# Supply an empty go_asm.h as if the compiler had been run. -symabis parsing is lax enough
# that we don't need the actual definitions that would appear in go_asm.h.
#
# See https://go-review.googlesource.com/c/go/+/146999/8/src/cmd/go/internal/work/gc.go
go_asm_h_digest = await Get(Digest,
CreateDigest([FileContent("go_asm.h", b"")]))
symabis_input_digest = await Get(
Digest, MergeDigests([request.compilation_input, go_asm_h_digest]))
symabis_result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=symabis_input_digest,
command=(
"tool",
"asm",
"-I",
"go/pkg/include",
"-gensymabis",
"-o",
"symabis",
"--",
*(f"./{request.source_files_subpath}/{name}"
for name in request.s_files),
),
description="Generate symabis metadata for assembly files.",
output_files=("symabis", ),
),
)
merged = await Get(
Digest,
MergeDigests([request.compilation_input,
symabis_result.output_digest]),
)
assembly_results = await MultiGet(
Get(
ProcessResult,
GoSdkProcess(
input_digest=request.compilation_input,
command=(
"tool",
"asm",
"-I",
"go/pkg/include",
"-o",
f"./{request.source_files_subpath}/{PurePath(s_file).with_suffix('.o')}",
f"./{request.source_files_subpath}/{s_file}",
),
description=f"Assemble {s_file}",
output_files=
(f"./{request.source_files_subpath}/{PurePath(s_file).with_suffix('.o')}",
),
),
) for s_file in request.s_files)
return AssemblyPreCompilation(
merged, tuple(result.output_digest for result in assembly_results))
async def link_assembly_post_compilation(
request: AssemblyPostCompilationRequest, ) -> AssemblyPostCompilation:
merged_digest, asm_tool_id = await MultiGet(
Get(
Digest,
MergeDigests(
[request.compilation_result, *request.assembly_digests])),
# Use `go tool asm` tool ID since `go tool pack` does not have a version argument.
Get(GoSdkToolIDResult, GoSdkToolIDRequest("asm")),
)
pack_result = await Get(
FallibleProcessResult,
GoSdkProcess(
input_digest=merged_digest,
command=(
"tool",
"pack",
"r",
"__pkg__.a",
*(f"./{request.dir_path}/{PurePath(name).with_suffix('.o')}"
for name in request.s_files),
),
env={
"__PANTS_GO_ASM_TOOL_ID": asm_tool_id.tool_id,
},
description=
f"Link assembly files to Go package archive for {request.dir_path}",
output_files=("__pkg__.a", ),
),
)
return AssemblyPostCompilation(
pack_result,
pack_result.output_digest if pack_result.exit_code == 0 else None)
async def resolve_external_go_package(
request: ResolveExternalGoPackageRequest, ) -> ResolvedGoPackage:
wrapped_target = await Get(WrappedTarget, Address, request.address)
target = wrapped_target.target
import_path = target[GoExternalPackageImportPathField].value
module_path = target[GoExternalModulePathField].value
module_version = target[GoExternalModuleVersionField].value
module = await Get(
DownloadedExternalModule,
DownloadExternalModuleRequest(path=module_path,
version=module_version),
)
assert import_path.startswith(module_path)
subpath = import_path[len(module_path):]
result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=module.digest,
command=("list", "-json", f"./{subpath}"),
description="Resolve _go_external_package metadata.",
),
)
metadata = json.loads(result.stdout)
return ResolvedGoPackage.from_metadata(
metadata,
import_path=import_path,
address=request.address,
module_address=None,
module_path=module_path,
module_version=module_version,
)
async def link_go_binary(request: LinkGoBinaryRequest) -> LinkedGoBinary:
link_tool_id = await Get(GoSdkToolIDResult, GoSdkToolIDRequest("link"))
result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=request.input_digest,
command=(
"tool",
"link",
"-importcfg",
request.import_config_path,
"-o",
request.output_filename,
"-buildmode=exe", # seen in `go build -x` output
*request.archives,
),
env={
"__PANTS_GO_LINK_TOOL_ID": link_tool_id.tool_id,
},
description=f"Link Go binary: {request.output_filename}",
output_files=(request.output_filename, ),
),
)
return LinkedGoBinary(result.output_digest)
async def link_assembly_post_compilation(
request: AssemblyPostCompilationRequest,
) -> AssemblyPostCompilation:
merged_digest = await Get(
Digest, MergeDigests([request.compilation_result, *request.assembly_digests])
)
pack_result = await Get(
FallibleProcessResult,
GoSdkProcess(
input_digest=merged_digest,
command=(
"tool",
"pack",
"r",
"__pkg__.a",
*(
f"./{request.dir_path}/{PurePath(name).with_suffix('.o')}"
for name in request.s_files
),
),
description=f"Link assembly files to Go package archive for {request.dir_path}",
output_files=("__pkg__.a",),
),
)
return AssemblyPostCompilation(
pack_result, pack_result.output_digest if pack_result.exit_code == 0 else None
)
async def determine_go_mod_info(request: GoModInfoRequest, ) -> GoModInfo:
wrapped_target = await Get(WrappedTarget, Address, request.address)
sources_field = wrapped_target.target[GoModSourcesField]
go_mod_path = sources_field.go_mod_path
go_mod_dir = os.path.dirname(go_mod_path)
# Get the `go.mod` (and `go.sum`) and strip so the file has no directory prefix.
hydrated_sources = await Get(HydratedSources,
HydrateSourcesRequest(sources_field))
sources_digest = hydrated_sources.snapshot.digest
stripped_source_get = Get(Digest, RemovePrefix(sources_digest, go_mod_dir))
mod_json_get = Get(
ProcessResult,
GoSdkProcess(
command=("mod", "edit", "-json"),
input_digest=sources_digest,
working_dir=go_mod_dir,
description=f"Parse {go_mod_path}",
),
)
mod_json, stripped_sources = await MultiGet(mod_json_get,
stripped_source_get)
module_metadata = json.loads(mod_json.stdout)
return GoModInfo(
import_path=module_metadata["Module"]["Path"],
digest=sources_digest,
stripped_digest=stripped_sources,
minimum_go_version=module_metadata.get("Go"),
)
def test_fuzz_target_supported(rule_runner: RuleRunner) -> None:
go_version_result = rule_runner.request(
ProcessResult,
[GoSdkProcess(["version"], description="Get `go` version.")])
if "go1.18" not in go_version_result.stdout.decode():
pytest.skip("Skipping because Go SDK is not 1.18 or higher.")
rule_runner.write_files({
"foo/BUILD":
"go_mod(name='mod')\ngo_package()",
"foo/go.mod":
"module foo",
"foo/fuzz_test.go":
textwrap.dedent("""
package foo
import (
"testing"
)
func FuzzFoo(f *testing.F) {
f.Add("foo")
f.Fuzz(func(t *testing.T, v string) {
if v != "foo" {
t.Fail()
}
})
}
"""),
})
tgt = rule_runner.get_target(Address("foo"))
result = rule_runner.request(TestResult, [GoTestFieldSet.create(tgt)])
assert result.exit_code == 0
assert "PASS: FuzzFoo" in result.stdout
async def determine_go_mod_info(request: GoModInfoRequest, ) -> GoModInfo:
if isinstance(request.source, Address):
wrapped_target = await Get(
WrappedTarget,
WrappedTargetRequest(request.source,
description_of_origin="<go mod info rule>"),
)
sources_field = wrapped_target.target[GoModSourcesField]
else:
sources_field = request.source
go_mod_path = sources_field.go_mod_path
go_mod_dir = os.path.dirname(go_mod_path)
# Get the `go.mod` (and `go.sum`) and strip so the file has no directory prefix.
hydrated_sources = await Get(HydratedSources,
HydrateSourcesRequest(sources_field))
sources_digest = hydrated_sources.snapshot.digest
mod_json = await Get(
ProcessResult,
GoSdkProcess(
command=("mod", "edit", "-json"),
input_digest=sources_digest,
working_dir=go_mod_dir,
description=f"Parse {go_mod_path}",
),
)
module_metadata = json.loads(mod_json.stdout)
return GoModInfo(
import_path=module_metadata["Module"]["Path"],
digest=sources_digest,
mod_path=go_mod_path,
minimum_go_version=module_metadata.get("Go"),
)
async def resolve_go_module(
request: ResolveGoModuleRequest, ) -> ResolvedGoModule:
wrapped_target = await Get(WrappedTarget, Address, request.address)
target = wrapped_target.target
sources = await Get(SourceFiles,
SourceFilesRequest([target.get(GoModuleSources)]))
flattened_sources_snapshot = await Get(
Snapshot,
RemovePrefix(sources.snapshot.digest, request.address.spec_path))
# Parse the go.mod for the module path and minimum Go version.
parse_result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=flattened_sources_snapshot.digest,
command=("mod", "edit", "-json"),
description=f"Parse go.mod for {request.address}.",
),
)
module_metadata = json.loads(parse_result.stdout)
module_path = module_metadata["Module"]["Path"]
minimum_go_version = module_metadata.get(
"Go", "1.16"
) # TODO: Figure out better default if missing. Use the SDKs version versus this hard-code.
# Resolve the dependencies in the go.mod.
list_modules_result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=flattened_sources_snapshot.digest,
command=("list", "-m", "-json", "all"),
description=f"List modules in build of {request.address}.",
),
)
modules = parse_module_descriptors(list_modules_result.stdout)
return ResolvedGoModule(
target=target,
import_path=module_path,
minimum_go_version=minimum_go_version,
modules=FrozenOrderedSet(modules),
digest=flattened_sources_snapshot.
digest, # TODO: Is this a resolved version? Need to update for go-resolve goal?
)
async def determine_go_std_lib_imports() -> GoStdLibImports:
list_result = await Get(
ProcessResult,
GoSdkProcess(
command=("list", "-json", "std"),
description="Ask Go for its available import paths",
absolutify_goroot=False,
),
)
result = {}
for package_descriptor in ijson.items(list_result.stdout, "", multiple_values=True):
import_path = package_descriptor.get("ImportPath")
target = package_descriptor.get("Target")
if not import_path or not target:
continue
result[import_path] = target
return GoStdLibImports(result)
async def run_go_vet(request: GoVetRequest,
go_vet_subsystem: GoVetSubsystem) -> LintResults:
if go_vet_subsystem.skip:
return LintResults([], linter_name=request.name)
source_files = await Get(
SourceFiles,
SourceFilesRequest(field_set.sources
for field_set in request.field_sets),
)
owning_go_mods = await MultiGet(
Get(OwningGoMod, OwningGoModRequest(field_set.address))
for field_set in request.field_sets)
owning_go_mod_addresses = {x.address for x in owning_go_mods}
go_mod_infos = await MultiGet(
Get(GoModInfo, GoModInfoRequest(address))
for address in owning_go_mod_addresses)
input_digest = await Get(
Digest,
MergeDigests([
source_files.snapshot.digest,
*(info.digest for info in set(go_mod_infos))
]),
)
package_dirs = sorted(
{os.path.dirname(f)
for f in source_files.snapshot.files})
process_result = await Get(
FallibleProcessResult,
GoSdkProcess(
("vet", *(f"./{p}" for p in package_dirs)),
input_digest=input_digest,
description=
f"Run `go vet` on {pluralize(len(source_files.snapshot.files), 'file')}.",
),
)
result = LintResult.from_fallible_process_result(process_result)
return LintResults([result], linter_name=request.name)
async def link_go_binary(request: LinkGoBinaryRequest) -> LinkedGoBinary:
result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=request.input_digest,
command=(
"tool",
"link",
"-importcfg",
request.import_config_path,
"-o",
request.output_filename,
"-buildmode=exe", # seen in `go build -x` output
*request.archives,
),
description=f"Link Go binary: {request.output_filename}",
output_files=(request.output_filename, ),
),
)
return LinkedGoBinary(result.output_digest)
async def analyze_go_third_party_module(
request: AnalyzeThirdPartyModuleRequest,
analyzer: PackageAnalyzerSetup,
golang_subsystem: GolangSubsystem,
) -> AnalyzedThirdPartyModule:
# Download the module.
download_result = await Get(
ProcessResult,
GoSdkProcess(
("mod", "download", "-json", f"{request.name}@{request.version}"),
input_digest=request.go_mod_digest, # for go.sum
working_dir=os.path.dirname(request.go_mod_path)
if request.go_mod_path else None,
# Allow downloads of the module sources.
allow_downloads=True,
output_directories=("gopath", ),
description=f"Download Go module {request.name}@{request.version}.",
),
)
if len(download_result.stdout) == 0:
raise AssertionError(
f"Expected output from `go mod download` for {request.name}@{request.version}."
)
module_metadata = json.loads(download_result.stdout)
module_sources_relpath = strip_sandbox_prefix(module_metadata["Dir"],
"gopath/")
go_mod_relpath = strip_sandbox_prefix(module_metadata["GoMod"], "gopath/")
# Subset the output directory to just the module sources and go.mod (which may be generated).
module_sources_snapshot = await Get(
Snapshot,
DigestSubset(
download_result.output_digest,
PathGlobs(
[f"{module_sources_relpath}/**", go_mod_relpath],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin=
f"the download of Go module {request.name}@{request.version}",
),
),
)
# Determine directories with potential Go packages in them.
candidate_package_dirs = []
files_by_dir = group_by_dir(p for p in module_sources_snapshot.files
if p.startswith(module_sources_relpath))
for maybe_pkg_dir, files in files_by_dir.items():
# Skip directories where "testdata" would end up in the import path.
# See https://github.com/golang/go/blob/f005df8b582658d54e63d59953201299d6fee880/src/go/build/build.go#L580-L585
if "testdata" in maybe_pkg_dir.split("/"):
continue
# Consider directories with at least one `.go` file as package candidates.
if any(f for f in files if f.endswith(".go")):
candidate_package_dirs.append(maybe_pkg_dir)
candidate_package_dirs.sort()
# Analyze all of the packages in this module.
analyzer_relpath = "__analyzer"
analysis_result = await Get(
ProcessResult,
Process(
[
os.path.join(analyzer_relpath, analyzer.path),
*candidate_package_dirs
],
input_digest=module_sources_snapshot.digest,
immutable_input_digests={
analyzer_relpath: analyzer.digest,
},
description=
f"Analyze metadata for Go third-party module: {request.name}@{request.version}",
level=LogLevel.DEBUG,
env={"CGO_ENABLED": "0"},
),
)
if len(analysis_result.stdout) == 0:
return AnalyzedThirdPartyModule(FrozenOrderedSet())
package_analysis_gets = []
for pkg_path, pkg_json in zip(
candidate_package_dirs,
ijson.items(analysis_result.stdout, "", multiple_values=True)):
package_analysis_gets.append(
Get(
FallibleThirdPartyPkgAnalysis,
AnalyzeThirdPartyPackageRequest(
pkg_json=_freeze_json_dict(pkg_json),
module_sources_digest=module_sources_snapshot.digest,
module_sources_path=module_sources_relpath,
module_import_path=request.name,
package_path=pkg_path,
minimum_go_version=request.minimum_go_version,
),
))
analyzed_packages_fallible = await MultiGet(package_analysis_gets)
analyzed_packages = [
pkg.analysis for pkg in analyzed_packages_fallible
if pkg.analysis and pkg.exit_code == 0
]
return AnalyzedThirdPartyModule(FrozenOrderedSet(analyzed_packages))
async def build_target(request: BuildGoPackageRequest, ) -> BuiltGoPackage:
wrapped_target = await Get(WrappedTarget, Address, request.address)
target = wrapped_target.target
if is_first_party_package_target(target):
source_files, resolved_package = await MultiGet(
Get(
SourceFiles,
SourceFilesRequest((target[GoPackageSources], )),
),
Get(ResolvedGoPackage,
ResolveGoPackageRequest(address=target.address)),
)
source_files_digest = source_files.snapshot.digest
source_files_subpath = target.address.spec_path
elif is_third_party_package_target(target):
module_path = target[GoExternalModulePathField].value
module, resolved_package = await MultiGet(
Get(
DownloadedExternalModule,
DownloadExternalModuleRequest(
path=module_path,
version=target[GoExternalModuleVersionField].value,
),
),
Get(ResolvedGoPackage,
ResolveExternalGoPackageRequest(address=request.address)),
)
source_files_digest = module.digest
source_files_subpath = resolved_package.import_path[len(module_path):]
else:
raise ValueError(
f"Unknown how to build Go target at address {request.address}.")
dependencies = await Get(Addresses,
DependenciesRequest(field=target[Dependencies]))
dependencies_targets = await Get(UnexpandedTargets,
Addresses(dependencies))
buildable_dependencies_targets = [
dep_tgt for dep_tgt in dependencies_targets
if is_first_party_package_target(dep_tgt)
or is_third_party_package_target(dep_tgt)
]
built_go_deps = await MultiGet(
Get(BuiltGoPackage, BuildGoPackageRequest(tgt.address))
for tgt in buildable_dependencies_targets)
gathered_imports = await Get(
GatheredImports,
GatherImportsRequest(packages=FrozenOrderedSet(built_go_deps),
include_stdlib=True),
)
import_path = resolved_package.import_path
if request.is_main:
import_path = "main"
compile_command = [
"tool",
"compile",
"-p",
import_path,
"-importcfg",
"./importcfg",
"-pack",
"-o",
"__pkg__.a",
]
input_digest = await Get(
Digest, MergeDigests([gathered_imports.digest, source_files_digest]))
assembly_digests = None
if resolved_package.s_files:
assembly_setup = await Get(
AssemblyPreCompilation,
AssemblyPreCompilationRequest(input_digest,
resolved_package.s_files,
source_files_subpath),
)
input_digest = assembly_setup.merged_compilation_input_digest
assembly_digests = assembly_setup.assembly_digests
compile_command.extend(assembly_setup.EXTRA_COMPILATION_ARGS)
compile_command.append("--")
compile_command.extend(f"./{source_files_subpath}/{name}"
for name in resolved_package.go_files)
result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=input_digest,
command=tuple(compile_command),
description=
f"Compile Go package with {pluralize(len(resolved_package.go_files), 'file')}. [{request.address}]",
output_files=("__pkg__.a", ),
),
)
output_digest = result.output_digest
if assembly_digests:
assembly_result = await Get(
AssemblyPostCompilation,
AssemblyPostCompilationRequest(output_digest, assembly_digests,
resolved_package.s_files,
source_files_subpath),
)
output_digest = assembly_result.merged_output_digest
return BuiltGoPackage(import_path=import_path, object_digest=output_digest)
async def package_go_binary(field_set: GoBinaryFieldSet, ) -> BuiltPackage:
main_address = field_set.main_address.value or ""
main_go_package_address = await Get(
Address,
AddressInput,
AddressInput.parse(main_address,
relative_to=field_set.address.spec_path),
)
wrapped_main_go_package_target = await Get(WrappedTarget, Address,
main_go_package_address)
main_go_package_target = wrapped_main_go_package_target.target
built_main_go_package = await Get(
BuiltGoPackage,
BuildGoPackageRequest(address=main_go_package_target.address,
is_main=True))
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest(roots=[main_go_package_target.address]))
buildable_deps = [
tgt for tgt in transitive_targets.dependencies
if is_first_party_package_target(tgt)
or is_third_party_package_target(tgt)
]
built_transitive_go_deps_requests = [
Get(BuiltGoPackage, BuildGoPackageRequest(address=tgt.address))
for tgt in buildable_deps
]
built_transitive_go_deps = await MultiGet(built_transitive_go_deps_requests
)
gathered_imports = await Get(
GatheredImports,
GatherImportsRequest(
packages=FrozenOrderedSet(built_transitive_go_deps),
include_stdlib=True,
),
)
input_digest = await Get(
Digest,
MergeDigests(
[gathered_imports.digest, built_main_go_package.object_digest]))
output_filename = PurePath(
field_set.output_path.value_or_default(file_ending=None))
result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=input_digest,
command=(
"tool",
"link",
"-importcfg",
"./importcfg",
"-o",
f"./{output_filename.name}",
"-buildmode=exe", # seen in `go build -x` output
"./__pkg__.a",
),
description="Link Go binary.",
output_files=(f"./{output_filename.name}", ),
),
)
renamed_output_digest = await Get(
Digest, AddPrefix(result.output_digest, str(output_filename.parent)))
artifact = BuiltPackageArtifact(relpath=str(output_filename))
return BuiltPackage(digest=renamed_output_digest, artifacts=(artifact, ))
async def setup_go_protoc_plugin(platform: Platform) -> _SetupGoProtocPlugin:
go_mod_digest = await Get(
Digest,
CreateDigest([
FileContent("go.mod", GO_PROTOBUF_GO_MOD.encode()),
FileContent("go.sum", GO_PROTOBUF_GO_SUM.encode()),
]),
)
download_sources_result = await Get(
ProcessResult,
GoSdkProcess(
["mod", "download", "all"],
input_digest=go_mod_digest,
output_directories=("gopath", ),
description="Download Go `protoc` plugin sources.",
allow_downloads=True,
),
)
go_plugin_build_result, go_grpc_plugin_build_result = await MultiGet(
Get(
ProcessResult,
GoSdkProcess(
[
"install",
"google.golang.org/protobuf/cmd/[email protected]"
],
input_digest=download_sources_result.output_digest,
output_files=["gopath/bin/protoc-gen-go"],
description="Build Go protobuf plugin for `protoc`.",
platform=platform,
),
),
Get(
ProcessResult,
GoSdkProcess(
[
"install",
"google.golang.org/grpc/cmd/[email protected]",
],
input_digest=download_sources_result.output_digest,
output_files=["gopath/bin/protoc-gen-go-grpc"],
description="Build Go gRPC protobuf plugin for `protoc`.",
platform=platform,
),
),
)
if go_plugin_build_result.output_digest == EMPTY_DIGEST:
raise AssertionError(
f"Failed to build protoc-gen-go:\n"
f"stdout:\n{go_plugin_build_result.stdout.decode()}\n\n"
f"stderr:\n{go_plugin_build_result.stderr.decode()}")
if go_grpc_plugin_build_result.output_digest == EMPTY_DIGEST:
raise AssertionError(
f"Failed to build protoc-gen-go-grpc:\n"
f"stdout:\n{go_grpc_plugin_build_result.stdout.decode()}\n\n"
f"stderr:\n{go_grpc_plugin_build_result.stderr.decode()}")
merged_output_digests = await Get(
Digest,
MergeDigests([
go_plugin_build_result.output_digest,
go_grpc_plugin_build_result.output_digest
]),
)
plugin_digest = await Get(
Digest, RemovePrefix(merged_output_digests, "gopath/bin"))
return _SetupGoProtocPlugin(plugin_digest)
async def download_external_module(
request: DownloadExternalModuleRequest, ) -> DownloadedExternalModule:
result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=EMPTY_DIGEST,
command=("mod", "download", "-json",
f"{request.path}@{request.version}"),
description=
f"Download external Go module at {request.path}@{request.version}.",
output_directories=("gopath", ),
),
)
# Decode the module metadata.
metadata = json.loads(result.stdout)
# Find the path within the digest where the source was downloaded. The path will have a sandbox-specific
# prefix that we need to strip down to the `gopath` path component.
absolute_source_path = metadata["Dir"]
gopath_index = absolute_source_path.index("gopath/")
source_path = absolute_source_path[gopath_index:]
source_digest = await Get(
Digest,
DigestSubset(
result.output_digest,
PathGlobs(
[f"{source_path}/**"],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=
f"the DownloadExternalModuleRequest for {request.path}@{request.version}",
),
),
)
source_snapshot_stripped = await Get(
Snapshot, RemovePrefix(source_digest, source_path))
if "go.mod" not in source_snapshot_stripped.files:
# There was no go.mod in the downloaded source. Use the generated go.mod from the go tooling which
# was returned in the module metadata.
go_mod_absolute_path = metadata.get("GoMod")
if not go_mod_absolute_path:
raise ValueError(
f"No go.mod was provided in download of Go external module {request.path}@{request.version}, "
"and the module metadata did not identify a generated go.mod file to use instead."
)
gopath_index = go_mod_absolute_path.index("gopath/")
go_mod_path = go_mod_absolute_path[gopath_index:]
go_mod_digest = await Get(
Digest,
DigestSubset(
result.output_digest,
PathGlobs(
[f"{go_mod_path}"],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=
f"the DownloadExternalModuleRequest for {request.path}@{request.version}",
),
),
)
go_mod_digest_stripped = await Get(
Digest, RemovePrefix(go_mod_digest, os.path.dirname(go_mod_path)))
# There should now be one file in the digest. Create a digest where that file is named go.mod
# and then merge it into the sources.
contents = await Get(DigestContents, Digest, go_mod_digest_stripped)
assert len(contents) == 1
go_mod_only_digest = await Get(
Digest,
CreateDigest(
[FileContent(
path="go.mod",
content=contents[0].content,
)]),
)
source_digest_final = await Get(
Digest,
MergeDigests([go_mod_only_digest,
source_snapshot_stripped.digest]))
else:
# If the module download has a go.mod, then just use the sources as is.
source_digest_final = source_snapshot_stripped.digest
return DownloadedExternalModule(
path=request.path,
version=request.version,
digest=source_digest_final,
sum=metadata["Sum"],
go_mod_sum=metadata["GoModSum"],
)
async def resolve_external_module_to_go_packages(
request: ResolveExternalGoModuleToPackagesRequest,
) -> ResolveExternalGoModuleToPackagesResult:
module_path = request.path
assert module_path
module_version = request.version
assert module_version
downloaded_module = await Get(
DownloadedExternalModule,
DownloadExternalModuleRequest(path=module_path,
version=module_version),
)
sources_digest = await Get(
Digest, AddPrefix(downloaded_module.digest, "__sources__"))
# TODO: Super hacky merge of go.sum from both digests. We should really just pass in the fully-resolved
# go.sum and use that, but this allows the go.sum from the downloaded module to have some effect. Not sure
# if that is right call, but hackity hack!
left_digest_contents = await Get(DigestContents, Digest, sources_digest)
left_go_sum_contents = b""
for fc in left_digest_contents:
if fc.path == "__sources__/go.sum":
left_go_sum_contents = fc.content
break
go_sum_only_digest = await Get(
Digest, DigestSubset(request.go_sum_digest, PathGlobs(["go.sum"])))
go_sum_prefixed_digest = await Get(
Digest, AddPrefix(go_sum_only_digest, "__sources__"))
right_digest_contents = await Get(DigestContents, Digest,
go_sum_prefixed_digest)
right_go_sum_contents = b""
for fc in right_digest_contents:
if fc.path == "__sources__/go.sum":
right_go_sum_contents = fc.content
break
go_sum_contents = left_go_sum_contents + b"\n" + right_go_sum_contents
go_sum_digest = await Get(
Digest,
CreateDigest([
FileContent(
path="__sources__/go.sum",
content=go_sum_contents,
)
]),
)
sources_digest_no_go_sum = await Get(
Digest,
DigestSubset(
sources_digest,
PathGlobs(
["!__sources__/go.sum", "__sources__/**"],
conjunction=GlobExpansionConjunction.all_match,
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="FUNKY",
),
),
)
input_digest = await Get(
Digest, MergeDigests([sources_digest_no_go_sum, go_sum_digest]))
result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=input_digest,
command=("list", "-json", "./..."),
working_dir="__sources__",
description=
f"Resolve packages in Go external module {module_path}@{module_version}",
),
)
packages: OrderedSet[ResolvedGoPackage] = OrderedSet()
for metadata in ijson.items(result.stdout, "", multiple_values=True):
package = ResolvedGoPackage.from_metadata(
metadata, module_path=module_path, module_version=module_version)
packages.add(package)
return ResolveExternalGoModuleToPackagesResult(
packages=FrozenOrderedSet(packages))
async def resolve_go_package(
request: ResolveGoPackageRequest,
) -> ResolvedGoPackage:
wrapped_target, owning_go_module_result = await MultiGet(
Get(WrappedTarget, Address, request.address),
Get(ResolvedOwningGoModule, FindNearestGoModuleRequest(request.address.spec_path)),
)
target = wrapped_target.target
if not owning_go_module_result.module_address:
raise ValueError(f"The go_package at address {request.address} has no owning go_module.")
resolved_go_module = await Get(
ResolvedGoModule, ResolveGoModuleRequest(owning_go_module_result.module_address)
)
go_module_spec_path = resolved_go_module.target.address.spec_path
assert request.address.spec_path.startswith(go_module_spec_path)
spec_subpath = request.address.spec_path[len(go_module_spec_path) :]
# Compute the import_path for this go_package.
import_path_field = target.get(GoImportPath)
if import_path_field and import_path_field.value:
# Use any explicit import path set on the `go_package` target.
import_path = import_path_field.value
else:
# Otherwise infer the import path from the owning `go_module` target. The inferred import path will be the
# module's import path plus any subdirectories in the spec_path between the go_module and go_package target.
if not resolved_go_module.import_path:
raise ValueError(
f"Unable to infer import path for the `go_package` at address {request.address} "
f"because the owning go_module at address {resolved_go_module.target.address} "
"does not have an import path defined nor could one be inferred."
)
import_path = f"{resolved_go_module.import_path}/"
if spec_subpath.startswith("/"):
import_path += spec_subpath[1:]
else:
import_path += spec_subpath
sources = await Get(
SourceFiles,
SourceFilesRequest(
[
target.get(GoPackageSources),
resolved_go_module.target.get(GoModuleSources),
]
),
)
result = await Get(
ProcessResult,
GoSdkProcess(
input_digest=sources.snapshot.digest,
command=("list", "-json", f"./{spec_subpath}"),
description="Resolve go_package metadata.",
working_dir=resolved_go_module.target.address.spec_path,
),
)
metadata = json.loads(result.stdout)
return ResolvedGoPackage.from_metadata(
metadata,
import_path=import_path,
address=request.address,
module_address=owning_go_module_result.module_address,
)
async def download_and_analyze_third_party_packages(
request: AllThirdPartyPackagesRequest, ) -> AllThirdPartyPackages:
# NB: We download all modules to GOPATH=$(pwd)/gopath. Running `go list ...` from $(pwd) would
# naively try analyzing the contents of the GOPATH like they were first-party packages. This
# results in errors like this:
#
# package <import_path>/gopath/pkg/mod/golang.org/x/[email protected]/unicode: can only use
# path@version syntax with 'go get' and 'go install' in module-aware mode
#
# Instead, we run `go list` from a subdirectory of the chroot. It can still access the
# contents of `GOPATH`, but won't incorrectly treat its contents as first-party packages.
go_mod_prefix = "go_mod_prefix"
go_mod_prefixed_digest = await Get(
Digest, AddPrefix(request.go_mod_stripped_digest, go_mod_prefix))
list_argv = (
"list",
# This rule can't modify `go.mod` and `go.sum` as it would require mutating the workspace.
# Instead, we expect them to be well-formed already.
#
# It would be convenient to set `-mod=mod` to allow edits, and then compare the resulting
# files to the input so that we could print a diff for the user to know how to update. But
# `-mod=mod` results in more packages being downloaded and added to `go.mod` than is
# actually necessary.
# TODO: nice error when `go.mod` and `go.sum` would need to change. Right now, it's a
# message from Go and won't be intuitive for Pants users what to do.
"-mod=readonly",
# There may be some packages in the transitive closure that cannot be built, but we should
# not blow up Pants.
#
# For example, a package that sets the special value `package documentation` and has no
# source files would naively error due to `build constraints exclude all Go files`, even
# though we should not error on that package.
"-e",
"-json",
# This matches all packages. `all` only matches first-party packages and complains that
# there are no `.go` files.
"...",
)
list_result = await Get(
ProcessResult,
GoSdkProcess(
command=list_argv,
# TODO: make this more descriptive: point to the actual `go_mod` target or path.
description=
"Run `go list` to download and analyze all third-party Go packages",
input_digest=go_mod_prefixed_digest,
output_directories=("gopath/pkg/mod", ),
working_dir=go_mod_prefix,
allow_downloads=True,
),
)
stripped_result_digest = await Get(
Digest, RemovePrefix(list_result.output_digest, "gopath/pkg/mod"))
all_digest_subset_gets = []
all_pkg_info_kwargs = []
all_failed_pkg_info = []
for pkg_json in ijson.items(list_result.stdout, "", multiple_values=True):
if "Standard" in pkg_json:
continue
import_path = pkg_json["ImportPath"]
maybe_error, maybe_failed_pkg_info = maybe_raise_or_create_error_or_create_failed_pkg_info(
pkg_json, import_path)
if maybe_failed_pkg_info:
all_failed_pkg_info.append(maybe_failed_pkg_info)
continue
dir_path = strip_prefix(strip_v2_chroot_path(pkg_json["Dir"]),
"gopath/pkg/mod/")
all_pkg_info_kwargs.append(
dict(
import_path=import_path,
subpath=dir_path,
imports=tuple(pkg_json.get("Imports", ())),
go_files=tuple(pkg_json.get("GoFiles", ())),
s_files=tuple(pkg_json.get("SFiles", ())),
minimum_go_version=pkg_json.get("Module", {}).get("GoVersion"),
error=maybe_error,
))
all_digest_subset_gets.append(
Get(
Digest,
DigestSubset(
stripped_result_digest,
PathGlobs(
[os.path.join(dir_path, "*")],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=f"downloading {import_path}",
),
),
))
all_digest_subsets = await MultiGet(all_digest_subset_gets)
import_path_to_info = {
pkg_info_kwargs["import_path"]: ThirdPartyPkgInfo(digest=digest_subset,
**pkg_info_kwargs)
for pkg_info_kwargs, digest_subset in zip(all_pkg_info_kwargs,
all_digest_subsets)
}
import_path_to_info.update(
(pkg_info.import_path, pkg_info) for pkg_info in all_failed_pkg_info)
return AllThirdPartyPackages(list_result.output_digest,
FrozenDict(import_path_to_info))
async def setup_assembly_pre_compilation(
request: AssemblyPreCompilationRequest,
goroot: GoRoot,
) -> FallibleAssemblyPreCompilation:
# From Go tooling comments:
#
# Supply an empty go_asm.h as if the compiler had been run. -symabis parsing is lax enough
# that we don't need the actual definitions that would appear in go_asm.h.
#
# See https://go-review.googlesource.com/c/go/+/146999/8/src/cmd/go/internal/work/gc.go
go_asm_h_digest, asm_tool_id = await MultiGet(
Get(Digest, CreateDigest([FileContent("go_asm.h", b"")])),
Get(GoSdkToolIDResult, GoSdkToolIDRequest("asm")),
)
symabis_input_digest = await Get(
Digest, MergeDigests([request.compilation_input, go_asm_h_digest]))
symabis_result = await Get(
FallibleProcessResult,
GoSdkProcess(
input_digest=symabis_input_digest,
command=(
"tool",
"asm",
"-I",
os.path.join(goroot.path, "pkg", "include"),
"-gensymabis",
"-o",
"symabis",
"--",
*(f"./{request.dir_path}/{name}" for name in request.s_files),
),
env={
"__PANTS_GO_ASM_TOOL_ID": asm_tool_id.tool_id,
},
description=
f"Generate symabis metadata for assembly files for {request.dir_path}",
output_files=("symabis", ),
),
)
if symabis_result.exit_code != 0:
return FallibleAssemblyPreCompilation(
None, symabis_result.exit_code,
symabis_result.stderr.decode("utf-8"))
merged = await Get(
Digest,
MergeDigests([request.compilation_input,
symabis_result.output_digest]),
)
assembly_results = await MultiGet(
Get(
FallibleProcessResult,
GoSdkProcess(
input_digest=request.compilation_input,
command=(
"tool",
"asm",
"-I",
os.path.join(goroot.path, "pkg", "include"),
"-o",
f"./{request.dir_path}/{PurePath(s_file).with_suffix('.o')}",
f"./{request.dir_path}/{s_file}",
),
env={
"__PANTS_GO_ASM_TOOL_ID": asm_tool_id.tool_id,
},
description=f"Assemble {s_file} with Go",
output_files=
(f"./{request.dir_path}/{PurePath(s_file).with_suffix('.o')}",
),
),
) for s_file in request.s_files)
exit_code = max(result.exit_code for result in assembly_results)
if exit_code != 0:
stdout = "\n\n".join(
result.stdout.decode("utf-8") for result in assembly_results
if result.stdout)
stderr = "\n\n".join(
result.stderr.decode("utf-8") for result in assembly_results
if result.stderr)
return FallibleAssemblyPreCompilation(None, exit_code, stdout, stderr)
return FallibleAssemblyPreCompilation(
AssemblyPreCompilation(
merged,
tuple(result.output_digest for result in assembly_results)))
async def build_go_package(request: BuildGoPackageRequest) -> FallibleBuiltGoPackage:
maybe_built_deps = await MultiGet(
Get(FallibleBuiltGoPackage, BuildGoPackageRequest, build_request)
for build_request in request.direct_dependencies
)
import_paths_to_pkg_a_files: dict[str, str] = {}
dep_digests = []
for maybe_dep in maybe_built_deps:
if maybe_dep.output is None:
return dataclasses.replace(
maybe_dep, import_path=request.import_path, dependency_failed=True
)
dep = maybe_dep.output
import_paths_to_pkg_a_files.update(dep.import_paths_to_pkg_a_files)
dep_digests.append(dep.digest)
merged_deps_digest, import_config, embedcfg = await MultiGet(
Get(Digest, MergeDigests(dep_digests)),
Get(ImportConfig, ImportConfigRequest(FrozenDict(import_paths_to_pkg_a_files))),
Get(RenderedEmbedConfig, RenderEmbedConfigRequest(request.embed_config)),
)
input_digest = await Get(
Digest,
MergeDigests([merged_deps_digest, import_config.digest, embedcfg.digest, request.digest]),
)
assembly_digests = None
symabis_path = None
if request.s_file_names:
assembly_setup = await Get(
FallibleAssemblyPreCompilation,
AssemblyPreCompilationRequest(input_digest, request.s_file_names, request.subpath),
)
if assembly_setup.result is None:
return FallibleBuiltGoPackage(
None,
request.import_path,
assembly_setup.exit_code,
stdout=assembly_setup.stdout,
stderr=assembly_setup.stderr,
)
input_digest = assembly_setup.result.merged_compilation_input_digest
assembly_digests = assembly_setup.result.assembly_digests
symabis_path = "./symabis"
compile_args = [
"tool",
"compile",
"-o",
"__pkg__.a",
"-pack",
"-p",
request.import_path,
"-importcfg",
import_config.CONFIG_PATH,
# See https://github.com/golang/go/blob/f229e7031a6efb2f23241b5da000c3b3203081d6/src/cmd/go/internal/work/gc.go#L79-L100
# for why Go sets the default to 1.16.
f"-lang=go{request.minimum_go_version or '1.16'}",
]
if symabis_path:
compile_args.extend(["-symabis", symabis_path])
if embedcfg.digest != EMPTY_DIGEST:
compile_args.extend(["-embedcfg", RenderedEmbedConfig.PATH])
if not request.s_file_names:
# If there are no non-Go sources, then pass -complete flag which tells the compiler that the provided
# Go files are the entire package.
compile_args.append("-complete")
relativized_sources = (
f"./{request.subpath}/{name}" if request.subpath else f"./{name}"
for name in request.go_file_names
)
compile_args.extend(["--", *relativized_sources])
compile_result = await Get(
FallibleProcessResult,
GoSdkProcess(
input_digest=input_digest,
command=tuple(compile_args),
description=f"Compile Go package: {request.import_path}",
output_files=("__pkg__.a",),
),
)
if compile_result.exit_code != 0:
return FallibleBuiltGoPackage(
None,
request.import_path,
compile_result.exit_code,
stdout=compile_result.stdout.decode("utf-8"),
stderr=compile_result.stderr.decode("utf-8"),
)
compilation_digest = compile_result.output_digest
if assembly_digests:
assembly_result = await Get(
AssemblyPostCompilation,
AssemblyPostCompilationRequest(
compilation_digest,
assembly_digests,
request.s_file_names,
request.subpath,
),
)
if assembly_result.result.exit_code != 0:
return FallibleBuiltGoPackage(
None,
request.import_path,
assembly_result.result.exit_code,
stdout=assembly_result.result.stdout.decode("utf-8"),
stderr=assembly_result.result.stderr.decode("utf-8"),
)
assert assembly_result.merged_output_digest
compilation_digest = assembly_result.merged_output_digest
path_prefix = os.path.join("__pkgs__", path_safe(request.import_path))
import_paths_to_pkg_a_files[request.import_path] = os.path.join(path_prefix, "__pkg__.a")
output_digest = await Get(Digest, AddPrefix(compilation_digest, path_prefix))
merged_result_digest = await Get(Digest, MergeDigests([*dep_digests, output_digest]))
output = BuiltGoPackage(merged_result_digest, FrozenDict(import_paths_to_pkg_a_files))
return FallibleBuiltGoPackage(output, request.import_path)
async def analyze_module_dependencies(
request: ModuleDescriptorsRequest) -> ModuleDescriptors:
# List the modules used directly and indirectly by this module.
#
# This rule can't modify `go.mod` and `go.sum` as it would require mutating the workspace.
# Instead, we expect them to be well-formed already.
#
# Options used:
# - `-mod=readonly': It would be convenient to set `-mod=mod` to allow edits, and then compare the
# resulting files to the input so that we could print a diff for the user to know how to update. But
# `-mod=mod` results in more packages being downloaded and added to `go.mod` than is
# actually necessary.
# TODO: nice error when `go.mod` and `go.sum` would need to change. Right now, it's a
# message from Go and won't be intuitive for Pants users what to do.
# - `-e` is used to not fail if one of the modules is problematic. There may be some packages in the transitive
# closure that cannot be built, but we should not blow up Pants. For example, a package that sets the
# special value `package documentation` and has no source files would naively error due to
# `build constraints exclude all Go files`, even though we should not error on that package.
mod_list_result = await Get(
ProcessResult,
GoSdkProcess(
command=["list", "-mod=readonly", "-e", "-m", "-json", "all"],
input_digest=request.digest,
output_directories=("gopath", ),
working_dir=request.path if request.path else None,
# Allow downloads of the module metadata (i.e., go.mod files).
allow_downloads=True,
description="Analyze Go module dependencies.",
),
)
if len(mod_list_result.stdout) == 0:
return ModuleDescriptors(FrozenOrderedSet(), EMPTY_DIGEST)
descriptors: dict[tuple[str, str], ModuleDescriptor] = {}
for mod_json in ijson.items(mod_list_result.stdout,
"",
multiple_values=True):
# Skip the first-party module being analyzed.
if "Main" in mod_json and mod_json["Main"]:
continue
if "Replace" in mod_json:
# TODO: Reject local file path replacements? Gazelle does.
name = mod_json["Replace"]["Path"]
version = mod_json["Replace"]["Version"]
else:
name = mod_json["Path"]
version = mod_json["Version"]
descriptors[(name, version)] = ModuleDescriptor(
import_path=mod_json["Path"],
name=name,
version=version,
indirect=mod_json.get("Indirect", False),
minimum_go_version=mod_json.get("GoVersion"),
)
# TODO: Augment the modules with go.sum entries?
# Gazelle does this, mainly to store the sum on the go_repository rule. We could store it (or its
# absence) to be able to download sums automatically.
return ModuleDescriptors(FrozenOrderedSet(descriptors.values()),
mod_list_result.output_digest)
