async def isort_fmt(request: IsortRequest, isort: Isort) -> FmtResult: if isort.skip: return FmtResult.skip(formatter_name=request.name) setup = await Get(Setup, SetupRequest(request, check_only=False)) result = await Get(ProcessResult, Process, setup.process) output_snapshot = await Get(Snapshot, Digest, result.output_digest) return FmtResult( setup.original_snapshot, output_snapshot, stdout=strip_v2_chroot_path(result.stdout), stderr=strip_v2_chroot_path(result.stderr), formatter_name=request.name, )
async def google_java_format_fmt(request: GoogleJavaFormatRequest, tool: GoogleJavaFormatSubsystem) -> FmtResult: if tool.skip: return FmtResult.skip(formatter_name=request.name) setup = await Get(Setup, SetupRequest(request, check_only=False)) result = await Get(ProcessResult, JvmProcess, setup.process) output_snapshot = await Get(Snapshot, Digest, result.output_digest) return FmtResult( setup.original_snapshot, output_snapshot, stdout=strip_v2_chroot_path(result.stdout), stderr=strip_v2_chroot_path(result.stderr), formatter_name=request.name, )
def test_strip_chroot_path() -> None: assert (strip_v2_chroot_path( dedent("""\ Would reformat /private/var/folders/sx/pdpbqz4x5cscn9hhfpbsbqvm0000gn/T/process-execution3zt5Ph/src/python/example.py Would reformat /var/folders/sx/pdpbqz4x5cscn9hhfpbsbqvm0000gn/T/process-executionOCnquv/test.py Would reformat /custom-tmpdir/process-execution7zt4pH/custom_tmpdir.py Some other output. """)) == dedent("""\ Would reformat src/python/example.py Would reformat test.py Would reformat custom_tmpdir.py Some other output. """)) # A subdir must be prefixed with `process-execution`, then some characters after it. assert (strip_v2_chroot_path("/var/process_executionOCnquv/test.py") == "/var/process_executionOCnquv/test.py") assert (strip_v2_chroot_path("/var/process-execution/test.py") == "/var/process-execution/test.py") # Our heuristic requires absolute paths. assert (strip_v2_chroot_path("var/process-executionOCnquv/test.py") == "var/process-executionOCnquv/test.py") # Confirm we can handle values with no chroot path. assert strip_v2_chroot_path("") == "" assert strip_v2_chroot_path("hello world") == "hello world"
def prep_output(s: bytes) -> str: return strip_v2_chroot_path(s) if strip_chroot_path else s.decode()
async def download_and_analyze_third_party_packages( request: AllThirdPartyPackagesRequest, ) -> AllThirdPartyPackages: # NB: We download all modules to GOPATH=$(pwd)/gopath. Running `go list ...` from $(pwd) would # naively try analyzing the contents of the GOPATH like they were first-party packages. This # results in errors like this: # # package <import_path>/gopath/pkg/mod/golang.org/x/[email protected]/unicode: can only use # path@version syntax with 'go get' and 'go install' in module-aware mode # # Instead, we run `go list` from a subdirectory of the chroot. It can still access the # contents of `GOPATH`, but won't incorrectly treat its contents as first-party packages. go_mod_prefix = "go_mod_prefix" go_mod_prefixed_digest = await Get( Digest, AddPrefix(request.go_mod_stripped_digest, go_mod_prefix)) list_argv = ( "list", # This rule can't modify `go.mod` and `go.sum` as it would require mutating the workspace. # Instead, we expect them to be well-formed already. # # It would be convenient to set `-mod=mod` to allow edits, and then compare the resulting # files to the input so that we could print a diff for the user to know how to update. But # `-mod=mod` results in more packages being downloaded and added to `go.mod` than is # actually necessary. # TODO: nice error when `go.mod` and `go.sum` would need to change. Right now, it's a # message from Go and won't be intuitive for Pants users what to do. "-mod=readonly", # There may be some packages in the transitive closure that cannot be built, but we should # not blow up Pants. # # For example, a package that sets the special value `package documentation` and has no # source files would naively error due to `build constraints exclude all Go files`, even # though we should not error on that package. "-e", "-json", # This matches all packages. `all` only matches first-party packages and complains that # there are no `.go` files. "...", ) list_result = await Get( ProcessResult, GoSdkProcess( command=list_argv, # TODO: make this more descriptive: point to the actual `go_mod` target or path. description= "Run `go list` to download and analyze all third-party Go packages", input_digest=go_mod_prefixed_digest, output_directories=("gopath/pkg/mod", ), working_dir=go_mod_prefix, allow_downloads=True, ), ) stripped_result_digest = await Get( Digest, RemovePrefix(list_result.output_digest, "gopath/pkg/mod")) all_digest_subset_gets = [] all_pkg_info_kwargs = [] all_failed_pkg_info = [] for pkg_json in ijson.items(list_result.stdout, "", multiple_values=True): if "Standard" in pkg_json: continue import_path = pkg_json["ImportPath"] maybe_error, maybe_failed_pkg_info = maybe_raise_or_create_error_or_create_failed_pkg_info( pkg_json, import_path) if maybe_failed_pkg_info: all_failed_pkg_info.append(maybe_failed_pkg_info) continue dir_path = strip_prefix(strip_v2_chroot_path(pkg_json["Dir"]), "gopath/pkg/mod/") all_pkg_info_kwargs.append( dict( import_path=import_path, subpath=dir_path, imports=tuple(pkg_json.get("Imports", ())), go_files=tuple(pkg_json.get("GoFiles", ())), s_files=tuple(pkg_json.get("SFiles", ())), minimum_go_version=pkg_json.get("Module", {}).get("GoVersion"), error=maybe_error, )) all_digest_subset_gets.append( Get( Digest, DigestSubset( stripped_result_digest, PathGlobs( [os.path.join(dir_path, "*")], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin=f"downloading {import_path}", ), ), )) all_digest_subsets = await MultiGet(all_digest_subset_gets) import_path_to_info = { pkg_info_kwargs["import_path"]: ThirdPartyPkgInfo(digest=digest_subset, **pkg_info_kwargs) for pkg_info_kwargs, digest_subset in zip(all_pkg_info_kwargs, all_digest_subsets) } import_path_to_info.update( (pkg_info.import_path, pkg_info) for pkg_info in all_failed_pkg_info) return AllThirdPartyPackages(list_result.output_digest, FrozenDict(import_path_to_info))