def test_find_binary_file_path(rule_runner: RuleRunner, tmp_path: Path) -> None: binary_path_abs = MyBin.create(tmp_path) binary_paths = rule_runner.request( BinaryPaths, [ BinaryPathRequest( binary_name=MyBin.binary_name, search_path=[str(binary_path_abs)], ) ], ) assert binary_paths.first_path is None, "By default, PATH file entries should not be checked." binary_paths = rule_runner.request( BinaryPaths, [ BinaryPathRequest( binary_name=MyBin.binary_name, search_path=[str(binary_path_abs)], check_file_entries=True, ) ], ) assert binary_paths.first_path is not None assert binary_paths.first_path.path == str(binary_path_abs)
async def determine_shunit2_shell(request: Shunit2RunnerRequest, shell_setup: ShellSetup) -> Shunit2Runner: if request.shell_field.value is not None: tgt_shell = Shunit2Shell(request.shell_field.value) else: parse_result = Shunit2Shell.parse_shebang( request.test_file_content.content) if parse_result is None: raise ShellNotConfigured( f"Could not determine which shell to use to run shunit2 on {request.address}.\n\n" f"Please either specify the `{Shunit2ShellField.alias}` field or add a " f"shebang to {request.test_file_content.path} with one of the supported shells in " f"the format `!#/path/to/shell` or `!#/path/to/env shell`" f"(run `{bin_name()} help {Shunit2TestsGeneratorTarget.alias}` for valid shells)." ) tgt_shell = parse_result env = await Get(Environment, EnvironmentRequest(["PATH"])) path_request = BinaryPathRequest( binary_name=tgt_shell.name, search_path=shell_setup.executable_search_path(env), test=tgt_shell.binary_path_test, ) paths = await Get(BinaryPaths, BinaryPathRequest, path_request) first_path = paths.first_path if not first_path: raise BinaryNotFoundError.from_request( path_request, rationale=f"run shunit2 on {request.address}") return Shunit2Runner(tgt_shell, first_path)
def test_find_binary_non_existent(rule_runner: RuleRunner, tmp_path: Path) -> None: binary_paths = rule_runner.request(BinaryPaths, [ BinaryPathRequest(binary_name="nonexistent-bin", search_path=[str(tmp_path)]) ]) assert binary_paths.first_path is None
def test_find_binary_respects_search_path_order(rule_runner: RuleRunner, tmp_path: Path) -> None: binary_path_abs1 = MyBin.create(tmp_path / "bin1") binary_path_abs2 = MyBin.create(tmp_path / "bin2") binary_path_abs3 = MyBin.create(tmp_path / "bin3") binary_paths = rule_runner.request( BinaryPaths, [ BinaryPathRequest( binary_name=MyBin.binary_name, search_path=[ str(binary_path_abs1.parent), str(binary_path_abs2), str(binary_path_abs3.parent), ], check_file_entries=True, ) ], ) assert binary_paths.first_path is not None assert binary_paths.first_path.path == str(binary_path_abs1) assert [ str(p) for p in (binary_path_abs1, binary_path_abs2, binary_path_abs3) ] == [binary_path.path for binary_path in binary_paths.paths]
async def package_debian_package(field_set: DebianPackageFieldSet, tar_binary_path: TarBinary) -> BuiltPackage: dpkg_deb_path = await Get( BinaryPaths, BinaryPathRequest( binary_name="dpkg-deb", search_path=["/usr/bin"], ), ) if not dpkg_deb_path.first_path: raise OSError( f"Could not find the `{dpkg_deb_path.binary_name}` program in `/usr/bin`." ) hydrated_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources_dir)) # Since all the sources are coming only from a single directory, it is # safe to pick an arbitrary file and get its root directory name. # Validation of the resolved files has been called on the target, so it is known that # snapshot.files isn't empty. sources_directory_name = PurePath( hydrated_sources.snapshot.files[0]).parts[0] result = await Get( ProcessResult, Process( argv=( dpkg_deb_path.first_path.path, "--build", sources_directory_name, ), description="Create a Debian package from the produced packages.", input_digest=hydrated_sources.snapshot.digest, # dpkg-deb produces a file with the same name as the input directory output_files=(f"{sources_directory_name}.deb", ), env={"PATH": str(PurePath(tar_binary_path.path).parent)}, ), ) # The output Debian package file needs to be renamed to match the output_path field. output_filename = field_set.output_path.value_or_default( file_ending="deb", ) digest_entries = await Get(DigestEntries, Digest, result.output_digest) assert len(digest_entries) == 1 result_file_entry = digest_entries[0] assert isinstance(result_file_entry, FileEntry) new_file = FileEntry(output_filename, result_file_entry.file_digest) final_result = await Get(Digest, CreateDigest([new_file])) return BuiltPackage(final_result, artifacts=(BuiltPackageArtifact(output_filename), ))
def test_find_binary_on_path_without_bash(rule_runner: RuleRunner, tmp_path: Path) -> None: # Test that locating a binary on a PATH which does not include bash works (by recursing to # locate bash first). binary_dir_abs = tmp_path / "bin" binary_path_abs = MyBin.create(binary_dir_abs) binary_paths = rule_runner.request( BinaryPaths, [ BinaryPathRequest(binary_name=MyBin.binary_name, search_path=[str(binary_dir_abs)]) ], ) assert binary_paths.first_path is not None assert binary_paths.first_path.path == str(binary_path_abs)
async def find_docker(docker_request: DockerBinaryRequest, docker_options: DockerOptions) -> DockerBinary: env = await Get(Environment, EnvironmentRequest(["PATH"])) search_path = docker_options.executable_search_path(env) request = BinaryPathRequest( binary_name="docker", search_path=search_path, test=BinaryPathTest(args=["-v"]), ) paths = await Get(BinaryPaths, BinaryPathRequest, request) first_path = paths.first_path_or_raise( request, rationale="interact with the docker daemon") if not docker_options.tools: return DockerBinary(first_path.path, first_path.fingerprint) tools = await Get( BinaryShims, BinaryShimsRequest, BinaryShimsRequest.for_binaries( *docker_options.tools, rationale="use docker", output_directory="bin", search_path=search_path, ), ) tools_path = ".shims" extra_env = { "PATH": os.path.join("{chroot}", tools_path, tools.bin_directory) } extra_input_digests = {tools_path: tools.digest} return DockerBinary( first_path.path, first_path.fingerprint, extra_env=extra_env, extra_input_digests=extra_input_digests, )
async def prepare_shell_command_process(request: ShellCommandProcessRequest, shell_setup: ShellSetup, bash: BashBinary) -> Process: shell_command = request.target interactive = shell_command.has_field(ShellCommandRunWorkdirField) if interactive: working_directory = shell_command[ ShellCommandRunWorkdirField].value or "" else: working_directory = shell_command.address.spec_path command = shell_command[ShellCommandCommandField].value timeout = shell_command.get(ShellCommandTimeoutField).value tools = shell_command.get(ShellCommandToolsField, default_raw_value=()).value outputs = shell_command.get(ShellCommandOutputsField).value or () extra_env_vars = shell_command.get( ShellCommandExtraEnvVarsField).value or () if not command: raise ValueError( f"Missing `command` line in `{shell_command.alias}` target {shell_command.address}." ) if interactive: command_env = { "CHROOT": "{chroot}", } else: if not tools: raise ValueError( f"Must provide any `tools` used by the `{shell_command.alias}` {shell_command.address}." ) env = await Get(Environment, EnvironmentRequest(["PATH"])) search_path = shell_setup.executable_search_path(env) tool_requests = [ BinaryPathRequest( binary_name=tool, search_path=search_path, ) for tool in {*tools, *["mkdir", "ln"]} if tool not in BASH_BUILTIN_COMMANDS ] tool_paths = await MultiGet( Get(BinaryPaths, BinaryPathRequest, request) for request in tool_requests) command_env = { "TOOLS": " ".join( _shell_tool_safe_env_name(tool.binary_name) for tool in tool_requests), } for binary, tool_request in zip(tool_paths, tool_requests): if binary.first_path: command_env[_shell_tool_safe_env_name( tool_request.binary_name)] = binary.first_path.path else: raise BinaryNotFoundError.from_request( tool_request, rationale= f"execute `{shell_command.alias}` {shell_command.address}", ) extra_env = await Get(Environment, EnvironmentRequest(extra_env_vars)) command_env.update(extra_env) transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([shell_command.address]), ) sources, pkgs_per_target = await MultiGet( Get( SourceFiles, SourceFilesRequest( sources_fields=[ tgt.get(SourcesField) for tgt in transitive_targets.dependencies ], for_sources_types=(SourcesField, FileSourceField), enable_codegen=True, ), ), Get( FieldSetsPerTarget, FieldSetsPerTargetRequest(PackageFieldSet, transitive_targets.dependencies), ), ) packages = await MultiGet( Get(BuiltPackage, PackageFieldSet, field_set) for field_set in pkgs_per_target.field_sets) if interactive or not working_directory or working_directory in sources.snapshot.dirs: work_dir = EMPTY_DIGEST else: work_dir = await Get(Digest, CreateDigest([Directory(working_directory)])) input_digest = await Get( Digest, MergeDigests([ sources.snapshot.digest, work_dir, *(pkg.digest for pkg in packages) ])) output_files = [f for f in outputs if not f.endswith("/")] output_directories = [d for d in outputs if d.endswith("/")] if interactive: relpath = os.path.relpath( working_directory or ".", start="/" if os.path.isabs(working_directory) else ".") boot_script = f"cd {shlex.quote(relpath)}; " if relpath != "." else "" else: # Setup bin_relpath dir with symlinks to all requested tools, so that we can use PATH, force # symlinks to avoid issues with repeat runs using the __run.sh script in the sandbox. bin_relpath = ".bin" boot_script = ";".join( dedent(f"""\ $mkdir -p {bin_relpath} for tool in $TOOLS; do $ln -sf ${{!tool}} {bin_relpath}; done export PATH="$PWD/{bin_relpath}" """).split("\n")) return Process( argv=(bash.path, "-c", boot_script + command), description=f"Running {shell_command.alias} {shell_command.address}", env=command_env, input_digest=input_digest, output_directories=output_directories, output_files=output_files, timeout_seconds=timeout, working_directory=working_directory, )
async def setup_goroot(golang_subsystem: GolangSubsystem) -> GoRoot: env = await Get(Environment, EnvironmentRequest(["PATH"])) search_paths = golang_subsystem.go_search_paths(env) all_go_binary_paths = await Get( BinaryPaths, BinaryPathRequest( search_path=search_paths, binary_name="go", test=BinaryPathTest(["version"]), ), ) if not all_go_binary_paths.paths: raise BinaryNotFoundError( "Cannot find any `go` binaries using the option " f"`[golang].go_search_paths`: {list(search_paths)}\n\n" "To fix, please install Go (https://golang.org/doc/install) with the version " f"{golang_subsystem.expected_version} (set by `[golang].expected_version`) and ensure " "that it is discoverable via `[golang].go_search_paths`.") # `go env GOVERSION` does not work in earlier Go versions (like 1.15), so we must run # `go version` and `go env GOROOT` to calculate both the version and GOROOT. version_results = await MultiGet( Get( ProcessResult, Process( (binary_path.path, "version"), description=f"Determine Go version for {binary_path.path}", level=LogLevel.DEBUG, cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL, ), ) for binary_path in all_go_binary_paths.paths) invalid_versions = [] for binary_path, version_result in zip(all_go_binary_paths.paths, version_results): try: _raw_version = version_result.stdout.decode("utf-8").split()[ 2] # e.g. go1.17 or go1.17.1 _version_components = _raw_version[2:].split( ".") # e.g. [1, 17] or [1, 17, 1] version = f"{_version_components[0]}.{_version_components[1]}" except IndexError: raise AssertionError( f"Failed to parse `go version` output for {binary_path}. Please open a bug at " f"https://github.com/pantsbuild/pants/issues/new/choose with the below data." f"\n\n" f"{version_result}") if version == golang_subsystem.expected_version: env_result = await Get( ProcessResult, Process( (binary_path.path, "env", "-json"), description= f"Determine Go SDK metadata for {binary_path.path}", level=LogLevel.DEBUG, cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL, env={"GOPATH": "/does/not/matter"}, ), ) sdk_metadata = json.loads(env_result.stdout.decode()) return GoRoot(path=sdk_metadata["GOROOT"], version=version, _raw_metadata=FrozenDict(sdk_metadata)) logger.debug( f"Go binary at {binary_path.path} has version {version}, but this " f"project is using {golang_subsystem.expected_version} " "(set by `[golang].expected_version`). Ignoring.") invalid_versions.append((binary_path.path, version)) invalid_versions_str = bullet_list( f"{path}: {version}" for path, version in sorted(invalid_versions)) raise BinaryNotFoundError( "Cannot find a `go` binary with the expected version of " f"{golang_subsystem.expected_version} (set by `[golang].expected_version`).\n\n" f"Found these `go` binaries, but they had different versions:\n\n" f"{invalid_versions_str}\n\n" "To fix, please install the expected version (https://golang.org/doc/install) and ensure " "that it is discoverable via the option `[golang].go_search_paths`, or change " "`[golang].expected_version`.")
async def setup_thrift_tool( apache_thrift: ApacheThriftSubsystem) -> ApacheThriftSetup: env = await Get(Environment, EnvironmentRequest(["PATH"])) search_paths = apache_thrift.thrift_search_paths(env) all_thrift_binary_paths = await Get( BinaryPaths, BinaryPathRequest( search_path=search_paths, binary_name="thrift", test=BinaryPathTest(["-version"]), ), ) if not all_thrift_binary_paths.paths: raise BinaryNotFoundError( softwrap(f""" Cannot find any `thrift` binaries using the option `[apache-thrift].thrift_search_paths`: {list(search_paths)} To fix, please install Apache Thrift (https://thrift.apache.org/) with the version {apache_thrift.expected_version} (set by `[apache-thrift].expected_version`) and ensure that it is discoverable via `[apache-thrift].thrift_search_paths`. """)) version_results = await MultiGet( Get( ProcessResult, Process( (binary_path.path, "-version"), description= f"Determine Apache Thrift version for {binary_path.path}", level=LogLevel.DEBUG, cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL, ), ) for binary_path in all_thrift_binary_paths.paths) invalid_versions = [] for binary_path, version_result in zip(all_thrift_binary_paths.paths, version_results): try: _raw_version = version_result.stdout.decode("utf-8").split()[2] _version_components = _raw_version.split( ".") # e.g. [1, 17] or [1, 17, 1] version = f"{_version_components[0]}.{_version_components[1]}" except IndexError: raise AssertionError( softwrap(f""" Failed to parse `thrift -version` output for {binary_path}. Please open a bug at https://github.com/pantsbuild/pants/issues/new/choose with the below data: {version_result} """)) if version == apache_thrift.expected_version: return ApacheThriftSetup(binary_path.path) logger.debug( softwrap(f""" The Thrift binary at {binary_path.path} has version {version}, but this project is using {apache_thrift.expected_version} (set by `[apache-thrift].expected_version`). Ignoring. """)) invalid_versions.append((binary_path.path, version)) invalid_versions_str = bullet_list( f"{path}: {version}" for path, version in sorted(invalid_versions)) raise BinaryNotFoundError( softwrap(f""" Cannot find a `thrift` binary with the expected version of {apache_thrift.expected_version} (set by `[apache-thrift].expected_version`). Found these `thrift` binaries, but they had different versions: {invalid_versions_str} To fix, please install the expected version (https://thrift.apache.org/) and ensure that it is discoverable via the option `[apache-thrift].thrift_search_paths`, or change `[apache-thrift].expected_version`. """))