async def isort_fmt(request: IsortRequest, isort: Isort) -> FmtResult:
if isort.skip:
return FmtResult.skip(formatter_name=request.name)
setup = await Get(Setup, SetupRequest(request, check_only=False))
result = await Get(ProcessResult, Process, setup.process)
output_snapshot = await Get(Snapshot, Digest, result.output_digest)
return FmtResult(
setup.original_snapshot,
output_snapshot,
stdout=strip_v2_chroot_path(result.stdout),
stderr=strip_v2_chroot_path(result.stderr),
formatter_name=request.name,
)
async def google_java_format_fmt(request: GoogleJavaFormatRequest,
tool: GoogleJavaFormatSubsystem) -> FmtResult:
if tool.skip:
return FmtResult.skip(formatter_name=request.name)
setup = await Get(Setup, SetupRequest(request, check_only=False))
result = await Get(ProcessResult, JvmProcess, setup.process)
output_snapshot = await Get(Snapshot, Digest, result.output_digest)
return FmtResult(
setup.original_snapshot,
output_snapshot,
stdout=strip_v2_chroot_path(result.stdout),
stderr=strip_v2_chroot_path(result.stderr),
formatter_name=request.name,
)
def test_strip_chroot_path() -> None:
assert (strip_v2_chroot_path(
dedent("""\
Would reformat /private/var/folders/sx/pdpbqz4x5cscn9hhfpbsbqvm0000gn/T/process-execution3zt5Ph/src/python/example.py
Would reformat /var/folders/sx/pdpbqz4x5cscn9hhfpbsbqvm0000gn/T/process-executionOCnquv/test.py
Would reformat /custom-tmpdir/process-execution7zt4pH/custom_tmpdir.py
Some other output.
""")) == dedent("""\
Would reformat src/python/example.py
Would reformat test.py
Would reformat custom_tmpdir.py
Some other output.
"""))
# A subdir must be prefixed with `process-execution`, then some characters after it.
assert (strip_v2_chroot_path("/var/process_executionOCnquv/test.py") ==
"/var/process_executionOCnquv/test.py")
assert (strip_v2_chroot_path("/var/process-execution/test.py") ==
"/var/process-execution/test.py")
# Our heuristic requires absolute paths.
assert (strip_v2_chroot_path("var/process-executionOCnquv/test.py") ==
"var/process-executionOCnquv/test.py")
# Confirm we can handle values with no chroot path.
assert strip_v2_chroot_path("") == ""
assert strip_v2_chroot_path("hello world") == "hello world"
def prep_output(s: bytes) -> str:
return strip_v2_chroot_path(s) if strip_chroot_path else s.decode()
async def download_and_analyze_third_party_packages(
request: AllThirdPartyPackagesRequest, ) -> AllThirdPartyPackages:
# NB: We download all modules to GOPATH=$(pwd)/gopath. Running `go list ...` from $(pwd) would
# naively try analyzing the contents of the GOPATH like they were first-party packages. This
# results in errors like this:
#
# package <import_path>/gopath/pkg/mod/golang.org/x/[email protected]/unicode: can only use
# path@version syntax with 'go get' and 'go install' in module-aware mode
#
# Instead, we run `go list` from a subdirectory of the chroot. It can still access the
# contents of `GOPATH`, but won't incorrectly treat its contents as first-party packages.
go_mod_prefix = "go_mod_prefix"
go_mod_prefixed_digest = await Get(
Digest, AddPrefix(request.go_mod_stripped_digest, go_mod_prefix))
list_argv = (
"list",
# This rule can't modify `go.mod` and `go.sum` as it would require mutating the workspace.
# Instead, we expect them to be well-formed already.
#
# It would be convenient to set `-mod=mod` to allow edits, and then compare the resulting
# files to the input so that we could print a diff for the user to know how to update. But
# `-mod=mod` results in more packages being downloaded and added to `go.mod` than is
# actually necessary.
# TODO: nice error when `go.mod` and `go.sum` would need to change. Right now, it's a
# message from Go and won't be intuitive for Pants users what to do.
"-mod=readonly",
# There may be some packages in the transitive closure that cannot be built, but we should
# not blow up Pants.
#
# For example, a package that sets the special value `package documentation` and has no
# source files would naively error due to `build constraints exclude all Go files`, even
# though we should not error on that package.
"-e",
"-json",
# This matches all packages. `all` only matches first-party packages and complains that
# there are no `.go` files.
"...",
)
list_result = await Get(
ProcessResult,
GoSdkProcess(
command=list_argv,
# TODO: make this more descriptive: point to the actual `go_mod` target or path.
description=
"Run `go list` to download and analyze all third-party Go packages",
input_digest=go_mod_prefixed_digest,
output_directories=("gopath/pkg/mod", ),
working_dir=go_mod_prefix,
allow_downloads=True,
),
)
stripped_result_digest = await Get(
Digest, RemovePrefix(list_result.output_digest, "gopath/pkg/mod"))
all_digest_subset_gets = []
all_pkg_info_kwargs = []
all_failed_pkg_info = []
for pkg_json in ijson.items(list_result.stdout, "", multiple_values=True):
if "Standard" in pkg_json:
continue
import_path = pkg_json["ImportPath"]
maybe_error, maybe_failed_pkg_info = maybe_raise_or_create_error_or_create_failed_pkg_info(
pkg_json, import_path)
if maybe_failed_pkg_info:
all_failed_pkg_info.append(maybe_failed_pkg_info)
continue
dir_path = strip_prefix(strip_v2_chroot_path(pkg_json["Dir"]),
"gopath/pkg/mod/")
all_pkg_info_kwargs.append(
dict(
import_path=import_path,
subpath=dir_path,
imports=tuple(pkg_json.get("Imports", ())),
go_files=tuple(pkg_json.get("GoFiles", ())),
s_files=tuple(pkg_json.get("SFiles", ())),
minimum_go_version=pkg_json.get("Module", {}).get("GoVersion"),
error=maybe_error,
))
all_digest_subset_gets.append(
Get(
Digest,
DigestSubset(
stripped_result_digest,
PathGlobs(
[os.path.join(dir_path, "*")],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=f"downloading {import_path}",
),
),
))
all_digest_subsets = await MultiGet(all_digest_subset_gets)
import_path_to_info = {
pkg_info_kwargs["import_path"]: ThirdPartyPkgInfo(digest=digest_subset,
**pkg_info_kwargs)
for pkg_info_kwargs, digest_subset in zip(all_pkg_info_kwargs,
all_digest_subsets)
}
import_path_to_info.update(
(pkg_info.import_path, pkg_info) for pkg_info in all_failed_pkg_info)
return AllThirdPartyPackages(list_result.output_digest,
FrozenDict(import_path_to_info))
