def test_find_binary_file_path(rule_runner: RuleRunner,
tmp_path: Path) -> None:
binary_path_abs = MyBin.create(tmp_path)
binary_paths = rule_runner.request(
BinaryPaths,
[
BinaryPathRequest(
binary_name=MyBin.binary_name,
search_path=[str(binary_path_abs)],
)
],
)
assert binary_paths.first_path is None, "By default, PATH file entries should not be checked."
binary_paths = rule_runner.request(
BinaryPaths,
[
BinaryPathRequest(
binary_name=MyBin.binary_name,
search_path=[str(binary_path_abs)],
check_file_entries=True,
)
],
)
assert binary_paths.first_path is not None
assert binary_paths.first_path.path == str(binary_path_abs)
async def determine_shunit2_shell(request: Shunit2RunnerRequest,
shell_setup: ShellSetup) -> Shunit2Runner:
if request.shell_field.value is not None:
tgt_shell = Shunit2Shell(request.shell_field.value)
else:
parse_result = Shunit2Shell.parse_shebang(
request.test_file_content.content)
if parse_result is None:
raise ShellNotConfigured(
f"Could not determine which shell to use to run shunit2 on {request.address}.\n\n"
f"Please either specify the `{Shunit2ShellField.alias}` field or add a "
f"shebang to {request.test_file_content.path} with one of the supported shells in "
f"the format `!#/path/to/shell` or `!#/path/to/env shell`"
f"(run `{bin_name()} help {Shunit2TestsGeneratorTarget.alias}` for valid shells)."
)
tgt_shell = parse_result
env = await Get(Environment, EnvironmentRequest(["PATH"]))
path_request = BinaryPathRequest(
binary_name=tgt_shell.name,
search_path=shell_setup.executable_search_path(env),
test=tgt_shell.binary_path_test,
)
paths = await Get(BinaryPaths, BinaryPathRequest, path_request)
first_path = paths.first_path
if not first_path:
raise BinaryNotFoundError.from_request(
path_request, rationale=f"run shunit2 on {request.address}")
return Shunit2Runner(tgt_shell, first_path)
def test_find_binary_non_existent(rule_runner: RuleRunner,
tmp_path: Path) -> None:
binary_paths = rule_runner.request(BinaryPaths, [
BinaryPathRequest(binary_name="nonexistent-bin",
search_path=[str(tmp_path)])
])
assert binary_paths.first_path is None
def test_find_binary_respects_search_path_order(rule_runner: RuleRunner,
tmp_path: Path) -> None:
binary_path_abs1 = MyBin.create(tmp_path / "bin1")
binary_path_abs2 = MyBin.create(tmp_path / "bin2")
binary_path_abs3 = MyBin.create(tmp_path / "bin3")
binary_paths = rule_runner.request(
BinaryPaths,
[
BinaryPathRequest(
binary_name=MyBin.binary_name,
search_path=[
str(binary_path_abs1.parent),
str(binary_path_abs2),
str(binary_path_abs3.parent),
],
check_file_entries=True,
)
],
)
assert binary_paths.first_path is not None
assert binary_paths.first_path.path == str(binary_path_abs1)
assert [
str(p) for p in (binary_path_abs1, binary_path_abs2, binary_path_abs3)
] == [binary_path.path for binary_path in binary_paths.paths]
async def package_debian_package(field_set: DebianPackageFieldSet,
tar_binary_path: TarBinary) -> BuiltPackage:
dpkg_deb_path = await Get(
BinaryPaths,
BinaryPathRequest(
binary_name="dpkg-deb",
search_path=["/usr/bin"],
),
)
if not dpkg_deb_path.first_path:
raise OSError(
f"Could not find the `{dpkg_deb_path.binary_name}` program in `/usr/bin`."
)
hydrated_sources = await Get(HydratedSources,
HydrateSourcesRequest(field_set.sources_dir))
# Since all the sources are coming only from a single directory, it is
# safe to pick an arbitrary file and get its root directory name.
# Validation of the resolved files has been called on the target, so it is known that
# snapshot.files isn't empty.
sources_directory_name = PurePath(
hydrated_sources.snapshot.files[0]).parts[0]
result = await Get(
ProcessResult,
Process(
argv=(
dpkg_deb_path.first_path.path,
"--build",
sources_directory_name,
),
description="Create a Debian package from the produced packages.",
input_digest=hydrated_sources.snapshot.digest,
# dpkg-deb produces a file with the same name as the input directory
output_files=(f"{sources_directory_name}.deb", ),
env={"PATH": str(PurePath(tar_binary_path.path).parent)},
),
)
# The output Debian package file needs to be renamed to match the output_path field.
output_filename = field_set.output_path.value_or_default(
file_ending="deb", )
digest_entries = await Get(DigestEntries, Digest, result.output_digest)
assert len(digest_entries) == 1
result_file_entry = digest_entries[0]
assert isinstance(result_file_entry, FileEntry)
new_file = FileEntry(output_filename, result_file_entry.file_digest)
final_result = await Get(Digest, CreateDigest([new_file]))
return BuiltPackage(final_result,
artifacts=(BuiltPackageArtifact(output_filename), ))
def test_find_binary_on_path_without_bash(rule_runner: RuleRunner,
tmp_path: Path) -> None:
# Test that locating a binary on a PATH which does not include bash works (by recursing to
# locate bash first).
binary_dir_abs = tmp_path / "bin"
binary_path_abs = MyBin.create(binary_dir_abs)
binary_paths = rule_runner.request(
BinaryPaths,
[
BinaryPathRequest(binary_name=MyBin.binary_name,
search_path=[str(binary_dir_abs)])
],
)
assert binary_paths.first_path is not None
assert binary_paths.first_path.path == str(binary_path_abs)
async def find_docker(docker_request: DockerBinaryRequest,
docker_options: DockerOptions) -> DockerBinary:
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_path = docker_options.executable_search_path(env)
request = BinaryPathRequest(
binary_name="docker",
search_path=search_path,
test=BinaryPathTest(args=["-v"]),
)
paths = await Get(BinaryPaths, BinaryPathRequest, request)
first_path = paths.first_path_or_raise(
request, rationale="interact with the docker daemon")
if not docker_options.tools:
return DockerBinary(first_path.path, first_path.fingerprint)
tools = await Get(
BinaryShims,
BinaryShimsRequest,
BinaryShimsRequest.for_binaries(
*docker_options.tools,
rationale="use docker",
output_directory="bin",
search_path=search_path,
),
)
tools_path = ".shims"
extra_env = {
"PATH": os.path.join("{chroot}", tools_path, tools.bin_directory)
}
extra_input_digests = {tools_path: tools.digest}
return DockerBinary(
first_path.path,
first_path.fingerprint,
extra_env=extra_env,
extra_input_digests=extra_input_digests,
)
async def prepare_shell_command_process(request: ShellCommandProcessRequest,
shell_setup: ShellSetup,
bash: BashBinary) -> Process:
shell_command = request.target
interactive = shell_command.has_field(ShellCommandRunWorkdirField)
if interactive:
working_directory = shell_command[
ShellCommandRunWorkdirField].value or ""
else:
working_directory = shell_command.address.spec_path
command = shell_command[ShellCommandCommandField].value
timeout = shell_command.get(ShellCommandTimeoutField).value
tools = shell_command.get(ShellCommandToolsField,
default_raw_value=()).value
outputs = shell_command.get(ShellCommandOutputsField).value or ()
extra_env_vars = shell_command.get(
ShellCommandExtraEnvVarsField).value or ()
if not command:
raise ValueError(
f"Missing `command` line in `{shell_command.alias}` target {shell_command.address}."
)
if interactive:
command_env = {
"CHROOT": "{chroot}",
}
else:
if not tools:
raise ValueError(
f"Must provide any `tools` used by the `{shell_command.alias}` {shell_command.address}."
)
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_path = shell_setup.executable_search_path(env)
tool_requests = [
BinaryPathRequest(
binary_name=tool,
search_path=search_path,
) for tool in {*tools, *["mkdir", "ln"]}
if tool not in BASH_BUILTIN_COMMANDS
]
tool_paths = await MultiGet(
Get(BinaryPaths, BinaryPathRequest, request)
for request in tool_requests)
command_env = {
"TOOLS":
" ".join(
_shell_tool_safe_env_name(tool.binary_name)
for tool in tool_requests),
}
for binary, tool_request in zip(tool_paths, tool_requests):
if binary.first_path:
command_env[_shell_tool_safe_env_name(
tool_request.binary_name)] = binary.first_path.path
else:
raise BinaryNotFoundError.from_request(
tool_request,
rationale=
f"execute `{shell_command.alias}` {shell_command.address}",
)
extra_env = await Get(Environment, EnvironmentRequest(extra_env_vars))
command_env.update(extra_env)
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([shell_command.address]),
)
sources, pkgs_per_target = await MultiGet(
Get(
SourceFiles,
SourceFilesRequest(
sources_fields=[
tgt.get(SourcesField)
for tgt in transitive_targets.dependencies
],
for_sources_types=(SourcesField, FileSourceField),
enable_codegen=True,
),
),
Get(
FieldSetsPerTarget,
FieldSetsPerTargetRequest(PackageFieldSet,
transitive_targets.dependencies),
),
)
packages = await MultiGet(
Get(BuiltPackage, PackageFieldSet, field_set)
for field_set in pkgs_per_target.field_sets)
if interactive or not working_directory or working_directory in sources.snapshot.dirs:
work_dir = EMPTY_DIGEST
else:
work_dir = await Get(Digest,
CreateDigest([Directory(working_directory)]))
input_digest = await Get(
Digest,
MergeDigests([
sources.snapshot.digest, work_dir,
*(pkg.digest for pkg in packages)
]))
output_files = [f for f in outputs if not f.endswith("/")]
output_directories = [d for d in outputs if d.endswith("/")]
if interactive:
relpath = os.path.relpath(
working_directory or ".",
start="/" if os.path.isabs(working_directory) else ".")
boot_script = f"cd {shlex.quote(relpath)}; " if relpath != "." else ""
else:
# Setup bin_relpath dir with symlinks to all requested tools, so that we can use PATH, force
# symlinks to avoid issues with repeat runs using the __run.sh script in the sandbox.
bin_relpath = ".bin"
boot_script = ";".join(
dedent(f"""\
$mkdir -p {bin_relpath}
for tool in $TOOLS; do $ln -sf ${{!tool}} {bin_relpath}; done
export PATH="$PWD/{bin_relpath}"
""").split("\n"))
return Process(
argv=(bash.path, "-c", boot_script + command),
description=f"Running {shell_command.alias} {shell_command.address}",
env=command_env,
input_digest=input_digest,
output_directories=output_directories,
output_files=output_files,
timeout_seconds=timeout,
working_directory=working_directory,
)
async def setup_goroot(golang_subsystem: GolangSubsystem) -> GoRoot:
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_paths = golang_subsystem.go_search_paths(env)
all_go_binary_paths = await Get(
BinaryPaths,
BinaryPathRequest(
search_path=search_paths,
binary_name="go",
test=BinaryPathTest(["version"]),
),
)
if not all_go_binary_paths.paths:
raise BinaryNotFoundError(
"Cannot find any `go` binaries using the option "
f"`[golang].go_search_paths`: {list(search_paths)}\n\n"
"To fix, please install Go (https://golang.org/doc/install) with the version "
f"{golang_subsystem.expected_version} (set by `[golang].expected_version`) and ensure "
"that it is discoverable via `[golang].go_search_paths`.")
# `go env GOVERSION` does not work in earlier Go versions (like 1.15), so we must run
# `go version` and `go env GOROOT` to calculate both the version and GOROOT.
version_results = await MultiGet(
Get(
ProcessResult,
Process(
(binary_path.path, "version"),
description=f"Determine Go version for {binary_path.path}",
level=LogLevel.DEBUG,
cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL,
),
) for binary_path in all_go_binary_paths.paths)
invalid_versions = []
for binary_path, version_result in zip(all_go_binary_paths.paths,
version_results):
try:
_raw_version = version_result.stdout.decode("utf-8").split()[
2] # e.g. go1.17 or go1.17.1
_version_components = _raw_version[2:].split(
".") # e.g. [1, 17] or [1, 17, 1]
version = f"{_version_components[0]}.{_version_components[1]}"
except IndexError:
raise AssertionError(
f"Failed to parse `go version` output for {binary_path}. Please open a bug at "
f"https://github.com/pantsbuild/pants/issues/new/choose with the below data."
f"\n\n"
f"{version_result}")
if version == golang_subsystem.expected_version:
env_result = await Get(
ProcessResult,
Process(
(binary_path.path, "env", "-json"),
description=
f"Determine Go SDK metadata for {binary_path.path}",
level=LogLevel.DEBUG,
cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL,
env={"GOPATH": "/does/not/matter"},
),
)
sdk_metadata = json.loads(env_result.stdout.decode())
return GoRoot(path=sdk_metadata["GOROOT"],
version=version,
_raw_metadata=FrozenDict(sdk_metadata))
logger.debug(
f"Go binary at {binary_path.path} has version {version}, but this "
f"project is using {golang_subsystem.expected_version} "
"(set by `[golang].expected_version`). Ignoring.")
invalid_versions.append((binary_path.path, version))
invalid_versions_str = bullet_list(
f"{path}: {version}" for path, version in sorted(invalid_versions))
raise BinaryNotFoundError(
"Cannot find a `go` binary with the expected version of "
f"{golang_subsystem.expected_version} (set by `[golang].expected_version`).\n\n"
f"Found these `go` binaries, but they had different versions:\n\n"
f"{invalid_versions_str}\n\n"
"To fix, please install the expected version (https://golang.org/doc/install) and ensure "
"that it is discoverable via the option `[golang].go_search_paths`, or change "
"`[golang].expected_version`.")
async def setup_thrift_tool(
apache_thrift: ApacheThriftSubsystem) -> ApacheThriftSetup:
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_paths = apache_thrift.thrift_search_paths(env)
all_thrift_binary_paths = await Get(
BinaryPaths,
BinaryPathRequest(
search_path=search_paths,
binary_name="thrift",
test=BinaryPathTest(["-version"]),
),
)
if not all_thrift_binary_paths.paths:
raise BinaryNotFoundError(
softwrap(f"""
Cannot find any `thrift` binaries using the option
`[apache-thrift].thrift_search_paths`: {list(search_paths)}
To fix, please install Apache Thrift (https://thrift.apache.org/) with the version
{apache_thrift.expected_version} (set by `[apache-thrift].expected_version`) and ensure
that it is discoverable via `[apache-thrift].thrift_search_paths`.
"""))
version_results = await MultiGet(
Get(
ProcessResult,
Process(
(binary_path.path, "-version"),
description=
f"Determine Apache Thrift version for {binary_path.path}",
level=LogLevel.DEBUG,
cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL,
),
) for binary_path in all_thrift_binary_paths.paths)
invalid_versions = []
for binary_path, version_result in zip(all_thrift_binary_paths.paths,
version_results):
try:
_raw_version = version_result.stdout.decode("utf-8").split()[2]
_version_components = _raw_version.split(
".") # e.g. [1, 17] or [1, 17, 1]
version = f"{_version_components[0]}.{_version_components[1]}"
except IndexError:
raise AssertionError(
softwrap(f"""
Failed to parse `thrift -version` output for {binary_path}. Please open a bug at
https://github.com/pantsbuild/pants/issues/new/choose with the below data:
{version_result}
"""))
if version == apache_thrift.expected_version:
return ApacheThriftSetup(binary_path.path)
logger.debug(
softwrap(f"""
The Thrift binary at {binary_path.path} has version {version}, but this
project is using {apache_thrift.expected_version}
(set by `[apache-thrift].expected_version`). Ignoring.
"""))
invalid_versions.append((binary_path.path, version))
invalid_versions_str = bullet_list(
f"{path}: {version}" for path, version in sorted(invalid_versions))
raise BinaryNotFoundError(
softwrap(f"""
Cannot find a `thrift` binary with the expected version of
{apache_thrift.expected_version} (set by `[apache-thrift].expected_version`).
Found these `thrift` binaries, but they had different versions:
{invalid_versions_str}
To fix, please install the expected version (https://thrift.apache.org/) and ensure
that it is discoverable via the option `[apache-thrift].thrift_search_paths`, or change
`[apache-thrift].expected_version`.
"""))