def test_find_binary_file_path(rule_runner: RuleRunner,
tmp_path: Path) -> None:
binary_path_abs = MyBin.create(tmp_path)
binary_paths = rule_runner.request(
BinaryPaths,
[
BinaryPathRequest(
binary_name=MyBin.binary_name,
search_path=[str(binary_path_abs)],
)
],
)
assert binary_paths.first_path is None, "By default, PATH file entries should not be checked."
binary_paths = rule_runner.request(
BinaryPaths,
[
BinaryPathRequest(
binary_name=MyBin.binary_name,
search_path=[str(binary_path_abs)],
check_file_entries=True,
)
],
)
assert binary_paths.first_path is not None
assert binary_paths.first_path.path == str(binary_path_abs)
async def find_open_program(request: OpenFilesRequest,
plat: Platform) -> OpenFiles:
open_program_name = "open" if plat == Platform.darwin else "xdg-open"
open_program_paths = await Get(
BinaryPaths,
BinaryPathRequest(binary_name=open_program_name,
search_path=("/bin", "/usr/bin")),
)
if not open_program_paths.first_path:
error = (
f"Could not find the program '{open_program_name}' on `/bin` or `/usr/bin`, so cannot "
f"open the files {sorted(request.files)}.")
if request.error_if_open_not_found:
raise OSError(error)
logger.error(error)
return OpenFiles(())
if plat == Platform.darwin:
processes = [
InteractiveProcess(
argv=(open_program_paths.first_path,
*(str(f) for f in request.files)),
run_in_workspace=True,
)
]
else:
processes = [
InteractiveProcess(argv=(open_program_paths.first_path, str(f)),
run_in_workspace=True) for f in request.files
]
return OpenFiles(tuple(processes))
async def determine_shunit2_shell(request: Shunit2RunnerRequest,
shell_setup: ShellSetup) -> Shunit2Runner:
if request.shell_field.value is not None:
tgt_shell = Shunit2Shell(request.shell_field.value)
else:
parse_result = Shunit2Shell.parse_shebang(
request.test_file_content.content)
if parse_result is None:
raise ShellNotConfigured(
f"Could not determine which shell to use to run shunit2 on {request.address}.\n\n"
f"Please either specify the `{Shunit2ShellField.alias}` field or add a "
f"shebang to {request.test_file_content.path} with one of the supported shells in "
f"the format `!#/path/to/shell` or `!#/path/to/env shell`"
f"(run `./pants help {Shunit2Tests.alias}` for valid shells).")
tgt_shell = parse_result
env = await Get(Environment, EnvironmentRequest(["PATH"]))
path_request = BinaryPathRequest(
binary_name=tgt_shell.name,
search_path=shell_setup.executable_search_path(env),
test=tgt_shell.binary_path_test,
)
paths = await Get(BinaryPaths, BinaryPathRequest, path_request)
first_path = paths.first_path
if not first_path:
raise BinaryNotFoundError(
path_request, rationale=f"run shunit2 on {request.address}")
return Shunit2Runner(tgt_shell, first_path)
def test_find_binary_non_existent(rule_runner: RuleRunner) -> None:
with temporary_dir() as tmpdir:
search_path = [tmpdir]
binary_paths = rule_runner.request(
BinaryPaths,
[BinaryPathRequest(binary_name="anybin", search_path=search_path)])
assert binary_paths.first_path is None
async def package_debian_package(field_set: DebianPackageFieldSet) -> BuiltPackage:
dpkg_deb_path = await Get(
BinaryPaths,
BinaryPathRequest(
binary_name="touch",
search_path=["/usr/bin"],
),
)
if not dpkg_deb_path.first_path:
raise EnvironmentError("Could not find the `touch` program on search paths ")
output_filename = field_set.output_path.value_or_default(file_ending="deb")
# TODO(alexey): add Debian packaging logic
result = await Get(
ProcessResult,
Process(
argv=(
"touch",
output_filename,
),
description="Create a Debian package from the produced packages.",
output_files=(output_filename,),
),
)
return BuiltPackage(result.output_digest, artifacts=(BuiltPackageArtifact(output_filename),))
def test_find_binary_respects_search_path_order(rule_runner: RuleRunner,
tmp_path: Path) -> None:
binary_path_abs1 = MyBin.create(tmp_path / "bin1")
binary_path_abs2 = MyBin.create(tmp_path / "bin2")
binary_path_abs3 = MyBin.create(tmp_path / "bin3")
binary_paths = rule_runner.request(
BinaryPaths,
[
BinaryPathRequest(
binary_name=MyBin.binary_name,
search_path=[
str(binary_path_abs1.parent),
str(binary_path_abs2),
str(binary_path_abs3.parent),
],
check_file_entries=True,
)
],
)
assert binary_paths.first_path is not None
assert binary_paths.first_path.path == str(binary_path_abs1)
assert [
str(p) for p in (binary_path_abs1, binary_path_abs2, binary_path_abs3)
] == [binary_path.path for binary_path in binary_paths.paths]
async def find_pex_python(
python_setup: PythonSetup,
pex_runtime_environment: PexRuntimeEnvironment,
subprocess_environment: SubprocessEnvironment,
) -> PexEnvironment:
# PEX files are compatible with bootstrapping via python2.7 or python 3.5+. The bootstrap
# code will then re-exec itself if the underlying PEX user code needs a more specific python
# interpreter. As such, we look for many Pythons usable by the PEX bootstrap code here for
# maximum flexibility.
all_python_binary_paths = await MultiGet([
Get(
BinaryPaths,
BinaryPathRequest(
search_path=python_setup.interpreter_search_paths,
binary_name=binary_name),
)
for binary_name in pex_runtime_environment.bootstrap_interpreter_names
])
def first_python_binary() -> Optional[str]:
for binary_paths in all_python_binary_paths:
if binary_paths.first_path:
return binary_paths.first_path
return None
return PexEnvironment(
path=pex_runtime_environment.path,
interpreter_search_paths=tuple(python_setup.interpreter_search_paths),
subprocess_environment_dict=FrozenDict(
subprocess_environment.environment_dict),
bootstrap_python=first_python_binary(),
)
def test_find_binary_non_existent(rule_runner: RuleRunner,
tmp_path: Path) -> None:
binary_paths = rule_runner.request(BinaryPaths, [
BinaryPathRequest(binary_name="nonexistent-bin",
search_path=[str(tmp_path)])
])
assert binary_paths.first_path is None
async def find_tar() -> TarBinary:
request = BinaryPathRequest(binary_name="tar",
search_path=SEARCH_PATHS,
test=BinaryPathTest(args=["--version"]))
paths = await Get(BinaryPaths, BinaryPathRequest, request)
first_path = paths.first_path_or_raise(
request, rationale="download the tools Pants needs to run")
return TarBinary(first_path.path, first_path.fingerprint)
async def find_zip() -> ZipBinary:
request = BinaryPathRequest(binary_name="zip",
search_path=SEARCH_PATHS,
test=BinaryPathTest(args=["-v"]))
paths = await Get(BinaryPaths, BinaryPathRequest, request)
first_path = paths.first_path_or_raise(request,
rationale="create `.zip` archives")
return ZipBinary(first_path.path, first_path.fingerprint)
async def find_unzip() -> UnzipBinary:
request = BinaryPathRequest(binary_name="unzip",
search_path=SEARCH_PATHS,
test=BinaryPathTest(args=["-v"]))
paths = await Get(BinaryPaths, BinaryPathRequest, request)
first_path = paths.first_path
if not first_path:
raise BinaryNotFoundError(
request, rationale="download the tools Pants needs to run")
return UnzipBinary(first_path.path, first_path.fingerprint)
def test_find_binary_non_existent(rule_runner: RuleRunner,
which_path: str | None) -> None:
with temporary_dir() as tmpdir:
if which_path:
os.symlink(which_path, os.path.join(tmpdir, "which"))
binary_paths = rule_runner.request(BinaryPaths, [
BinaryPathRequest(binary_name="nonexistent-bin",
search_path=[tmpdir])
])
assert binary_paths.first_path is None
async def find_docker(docker_request: DockerBinaryRequest) -> DockerBinary:
request = BinaryPathRequest(
binary_name="docker",
search_path=docker_request.search_path,
test=BinaryPathTest(args=["-v"]),
)
paths = await Get(BinaryPaths, BinaryPathRequest, request)
first_path = paths.first_path
if not first_path:
raise BinaryNotFoundError.from_request(request, rationale="interact with the docker daemon")
return DockerBinary(first_path.path, first_path.fingerprint)
async def package_bash_binary(field_set: BashBinaryFieldSet,
bash_setup: BashSetup) -> BuiltPackage:
# We first locate the `zip` program using `BinaryPaths`. We use the option
# `--bash-executable-search-paths` to determine which paths to search, such as `/bin` and
# `/usr/bin`. See https://www.pantsbuild.org/v2.0/docs/rules-api-installing-tools.
zip_program_paths = await Get(
BinaryPaths,
BinaryPathRequest(
binary_name="zip",
search_path=bash_setup.executable_search_path,
# This will run `zip --version` to ensure it's a valid binary and to allow
# invalidating the cache if the version changes.
test=BinaryPathTest(args=["-v"]),
),
)
if not zip_program_paths.first_path:
raise EnvironmentError(
f"Could not find the `zip` program on search paths "
f"{list(bash_setup.executable_search_path)}, so cannot create a binary for "
f"{field_set.address}. Please check that `zip` is installed and possibly modify the "
"option `executable_search_paths` in the `[bash-setup]` options scope."
)
# We need to include all relevant transitive dependencies in the zip. See
# https://www.pantsbuild.org/v2.0/docs/rules-api-and-target-api.
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest([field_set.address]))
sources = await Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
for_sources_types=(BashSources, FilesSources, ResourcesSources),
),
)
output_filename = field_set.output_path.value_or_default(
field_set.address, file_ending="zip", use_legacy_format=False)
result = await Get(
ProcessResult,
Process(
argv=(
zip_program_paths.first_path.path,
output_filename,
*sources.snapshot.files,
),
input_digest=sources.snapshot.digest,
description=f"Zip {field_set.address} and its dependencies.",
output_files=(output_filename, ),
),
)
return BuiltPackage(result.output_digest,
artifacts=(BuiltPackageArtifact(output_filename), ))
def test_find_binary_on_path_without_bash(rule_runner: RuleRunner,
tmp_path: Path) -> None:
# Test that locating a binary on a PATH which does not include bash works (by recursing to
# locate bash first).
binary_dir_abs = tmp_path / "bin"
binary_path_abs = MyBin.create(binary_dir_abs)
binary_paths = rule_runner.request(
BinaryPaths,
[
BinaryPathRequest(binary_name=MyBin.binary_name,
search_path=[str(binary_dir_abs)])
],
)
assert binary_paths.first_path is not None
assert binary_paths.first_path.path == str(binary_path_abs)
def test_find_binary_on_path_without_bash(rule_runner: RuleRunner) -> None:
# Test that locating a binary on a PATH which does not include bash works (by recursing to
# locate bash first).
binary_name = "mybin"
binary_dir = "bin"
with temporary_dir() as tmpdir:
binary_dir_abs = os.path.join(tmpdir, binary_dir)
binary_path_abs = os.path.join(binary_dir_abs, binary_name)
safe_mkdir(binary_dir_abs)
touch(binary_path_abs)
search_path = [binary_dir_abs]
binary_paths = rule_runner.request(BinaryPaths, [
BinaryPathRequest(binary_name=binary_name, search_path=search_path)
])
assert os.path.exists(os.path.join(binary_dir_abs, binary_name))
assert binary_paths.first_path is not None
assert binary_paths.first_path.path == binary_path_abs
async def find_open_program(
request: OpenFilesRequest,
plat: Platform,
complete_env: CompleteEnvironment,
) -> OpenFiles:
open_program_name = "open" if plat.is_macos else "xdg-open"
open_program_paths = await Get(
BinaryPaths,
BinaryPathRequest(binary_name=open_program_name,
search_path=("/bin", "/usr/bin")),
)
if not open_program_paths.first_path:
error = (
f"Could not find the program '{open_program_name}' on `/bin` or `/usr/bin`, so cannot "
f"open the files {sorted(request.files)}.")
if request.error_if_open_not_found:
raise OSError(error)
logger.error(error)
return OpenFiles(())
if plat.is_macos:
processes = [
InteractiveProcess(
argv=(open_program_paths.first_path.path,
*(str(f) for f in request.files)),
run_in_workspace=True,
restartable=True,
)
]
else:
processes = [
InteractiveProcess(
argv=(open_program_paths.first_path.path, str(f)),
run_in_workspace=True,
# The xdg-open binary needs many environment variables to work properly. In addition
# to the various XDG_* environment variables, DISPLAY and other X11 variables are
# required. Instead of attempting to track all of these we just export the full user
# environment since this is not a cached process.
env=complete_env,
restartable=True,
) for f in request.files
]
return OpenFiles(tuple(processes))
async def run_bash_binary(bash_setup: BashSetup) -> BashProgram:
# We expect Bash to already be installed. See
# https://www.pantsbuild.org/v2.0/docs/rules-api-installing-tools.
bash_program_paths = await Get(
BinaryPaths,
BinaryPathRequest(
binary_name="bash",
search_path=bash_setup.executable_search_path,
# This will run `bash --version` to ensure it's a valid binary and to allow
# invalidating the cache if the version changes.
test=BinaryPathTest(args=["--version"]),
),
)
if not bash_program_paths.first_path:
raise EnvironmentError(
"Could not find the `bash` program on search paths "
f"{list(bash_setup.executable_search_path)}. Please check that `bash` is installed and "
"possibly modify the option `executable_search_paths` in the `[bash-setup]` options "
"scope.")
return BashProgram(bash_program_paths.first_path.path)
async def package_into_image(
field_set: DockerPackageFieldSet,
union_membership: UnionMembership,
) -> BuiltPackage:
"""Build a docker image from a 'docker' build target.
Creates a build context & dockerfile from the build target & its
dependencies. Then builds & tags that image. (see the module
docstring for more information)
"""
target_name = field_set.address.target_name
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest([field_set.address])
)
component_list = []
logger.debug("Building Target %s", target_name)
for field_set_type in union_membership[DockerComponentFieldSet]:
for target in transitive_targets.dependencies:
if field_set_type.is_applicable(target):
logger.debug(
"Dependent Target %s applies to as component %s",
target.address,
field_set_type.__name__,
)
component_list.append(field_set_type.create(target))
components = await MultiGet(
Get(DockerComponent, DockerComponentFieldSet, fs) for fs in component_list
)
source_digests = []
run_commands = []
components = sorted(components, key=lambda c: c.order)
for component in components:
if component.sources:
source_digests.append(component.sources)
run_commands.extend(component.commands)
source_digest = await Get(Digest, MergeDigests(source_digests))
application_snapshot = await Get(Snapshot, AddPrefix(source_digest, "application"))
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Files to be copied into the docker container")
for file in application_snapshot.files:
logger.debug("* %s", file)
dockerfile_contents = _create_dockerfile(
field_set.base_image.value,
field_set.workdir.value,
field_set.image_setup.value,
run_commands,
field_set.command.value,
)
logger.debug(dockerfile_contents)
dockerfile = await Get(
Digest,
CreateDigest([FileContent("Dockerfile", dockerfile_contents.encode("utf-8"))]),
)
# create docker build context of all merged files & fetch docker
# connection enviornment variables
# and the location of the docker process
search_path = ["/bin", "/usr/bin", "/usr/local/bin", "$HOME/"]
docker_context, docker_env, docker_paths = await MultiGet(
Get(Digest, MergeDigests([dockerfile, application_snapshot.digest])),
Get(Environment, EnvironmentRequest(utils.DOCKER_ENV_VARS)),
Get(
BinaryPaths,
BinaryPathRequest(
binary_name="docker",
search_path=search_path,
),
),
)
if not docker_paths.first_path:
raise ValueError("Unable to locate Docker binary on paths: %s", search_path)
process_path = docker_paths.first_path.path
# build an list of arguments of the form ["-t",
# "registry/name:tag"] to pass to the docker executable
tag_arguments = _build_tag_argument_list(
target_name, field_set.tags.value or [], field_set.registry.value
)
# create the image
process_args = [process_path, "build"]
if not logger.isEnabledFor(logging.DEBUG):
process_args.append("-q") # only output the hash of the image
process_args.extend(tag_arguments)
process_args.append(".") # use current (sealed) directory as build context
process_result = await Get(
ProcessResult,
Process(
env=docker_env,
argv=process_args,
input_digest=docker_context,
description=f"Creating Docker Image from {target_name}",
),
)
logger.info(process_result.stdout.decode())
o = await Get(Snapshot, Digest, process_result.output_digest)
return BuiltPackage(
digest=process_result.output_digest,
artifacts=([BuiltPackageArtifact(f, ()) for f in o.files]),
)
async def find_pex_python(
python_setup: PythonSetup,
pex_runtime_env: PexRuntimeEnvironment,
subprocess_env_vars: SubprocessEnvironmentVars,
) -> PexEnvironment:
# PEX files are compatible with bootstrapping via Python 2.7 or Python 3.5+. The bootstrap
# code will then re-exec itself if the underlying PEX user code needs a more specific python
# interpreter. As such, we look for many Pythons usable by the PEX bootstrap code here for
# maximum flexibility.
all_python_binary_paths = await MultiGet(
Get(
BinaryPaths,
BinaryPathRequest(
search_path=python_setup.interpreter_search_paths,
binary_name=binary_name,
test=BinaryPathTest(
args=[
"-c",
# N.B.: The following code snippet must be compatible with Python 2.7 and
# Python 3.5+.
#
# We hash the underlying Python interpreter executable to ensure we detect
# changes in the real interpreter that might otherwise be masked by Pyenv
# shim scripts found on the search path. Naively, just printing out the full
# version_info would be enough, but that does not account for supported abi
# changes (e.g.: a pyenv switch from a py27mu interpreter to a py27m
# interpreter.)
#
# When hashing, we pick 8192 for efficiency of reads and fingerprint updates
# (writes) since it's a common OS buffer size and an even multiple of the
# hash block size.
dedent("""\
import sys
major, minor = sys.version_info[:2]
if (major, minor) != (2, 7) and not (major == 3 and minor >= 5):
sys.exit(1)
import hashlib
hasher = hashlib.sha256()
with open(sys.executable, "rb") as fp:
for chunk in iter(lambda: fp.read(8192), b""):
hasher.update(chunk)
sys.stdout.write(hasher.hexdigest())
"""),
],
fingerprint_stdout=
False, # We already emit a usable fingerprint to stdout.
),
),
) for binary_name in pex_runtime_env.bootstrap_interpreter_names)
def first_python_binary() -> Optional[PythonExecutable]:
for binary_paths in all_python_binary_paths:
if binary_paths.first_path:
return PythonExecutable(
path=binary_paths.first_path.path,
fingerprint=binary_paths.first_path.fingerprint,
)
return None
return PexEnvironment(
path=pex_runtime_env.path,
interpreter_search_paths=tuple(python_setup.interpreter_search_paths),
subprocess_environment_dict=subprocess_env_vars.vars,
bootstrap_python=first_python_binary(),
)
async def prepare_shell_command_process(request: ShellCommandProcessRequest,
shell_setup: ShellSetup,
bash: BashBinary) -> Process:
shell_command = request.target
interactive = shell_command.has_field(ShellCommandRunWorkdirField)
if interactive:
working_directory = shell_command[
ShellCommandRunWorkdirField].value or ""
else:
working_directory = shell_command.address.spec_path
command = shell_command[ShellCommandCommandField].value
timeout = shell_command.get(ShellCommandTimeoutField).value
tools = shell_command.get(ShellCommandToolsField,
default_raw_value=()).value
outputs = shell_command.get(ShellCommandOutputsField).value or ()
if not command:
raise ValueError(
f"Missing `command` line in `{shell_command.alias}` target {shell_command.address}."
)
if interactive:
command_env = {
"CHROOT": "{chroot}",
}
else:
if not tools:
raise ValueError(
f"Must provide any `tools` used by the `{shell_command.alias}` {shell_command.address}."
)
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_path = shell_setup.executable_search_path(env)
tool_requests = [
BinaryPathRequest(
binary_name=tool,
search_path=search_path,
) for tool in {*tools, *["mkdir", "ln"]}
if tool not in BASH_BUILTIN_COMMANDS
]
tool_paths = await MultiGet(
Get(BinaryPaths, BinaryPathRequest, request)
for request in tool_requests)
command_env = {
"TOOLS":
" ".join(
_shell_tool_safe_env_name(tool.binary_name)
for tool in tool_requests),
}
for binary, tool_request in zip(tool_paths, tool_requests):
if binary.first_path:
command_env[_shell_tool_safe_env_name(
tool_request.binary_name)] = binary.first_path.path
else:
raise BinaryNotFoundError.from_request(
tool_request,
rationale=
f"execute `{shell_command.alias}` {shell_command.address}",
)
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([shell_command.address]),
)
sources, pkgs_per_target = await MultiGet(
Get(
SourceFiles,
SourceFilesRequest(
sources_fields=[
tgt.get(SourcesField)
for tgt in transitive_targets.dependencies
],
for_sources_types=(SourcesField, FileSourceField),
enable_codegen=True,
),
),
Get(
FieldSetsPerTarget,
FieldSetsPerTargetRequest(PackageFieldSet,
transitive_targets.dependencies),
),
)
packages = await MultiGet(
Get(BuiltPackage, PackageFieldSet, field_set)
for field_set in pkgs_per_target.field_sets)
if interactive or not working_directory or working_directory in sources.snapshot.dirs:
work_dir = EMPTY_DIGEST
else:
work_dir = await Get(Digest,
CreateDigest([Directory(working_directory)]))
input_digest = await Get(
Digest,
MergeDigests([
sources.snapshot.digest, work_dir,
*(pkg.digest for pkg in packages)
]))
output_files = [f for f in outputs if not f.endswith("/")]
output_directories = [d for d in outputs if d.endswith("/")]
if interactive:
relpath = os.path.relpath(
working_directory or ".",
start="/" if os.path.isabs(working_directory) else ".")
boot_script = f"cd {shlex.quote(relpath)}; " if relpath != "." else ""
else:
# Setup bin_relpath dir with symlinks to all requested tools, so that we can use PATH, force
# symlinks to avoid issues with repeat runs using the __run.sh script in the sandbox.
bin_relpath = ".bin"
boot_script = ";".join(
dedent(f"""\
$mkdir -p {bin_relpath}
for tool in $TOOLS; do $ln -sf ${{!tool}} {bin_relpath}; done
export PATH="$PWD/{bin_relpath}"
""").split("\n"))
return Process(
argv=(bash.path, "-c", boot_script + command),
description=f"Running {shell_command.alias} {shell_command.address}",
env=command_env,
input_digest=input_digest,
output_directories=output_directories,
output_files=output_files,
timeout_seconds=timeout,
working_directory=working_directory,
)
async def find_python(python_bootstrap: PythonBootstrap) -> PythonBinary:
# PEX files are compatible with bootstrapping via Python 2.7 or Python 3.5+, but we select 3.6+
# for maximum compatibility with internal scripts.
interpreter_search_paths = python_bootstrap.interpreter_search_paths()
all_python_binary_paths = await MultiGet(
Get(
BinaryPaths,
BinaryPathRequest(
search_path=interpreter_search_paths,
binary_name=binary_name,
check_file_entries=True,
test=BinaryPathTest(
args=[
"-c",
# N.B.: The following code snippet must be compatible with Python 3.6+.
#
# We hash the underlying Python interpreter executable to ensure we detect
# changes in the real interpreter that might otherwise be masked by Pyenv
# shim scripts found on the search path. Naively, just printing out the full
# version_info would be enough, but that does not account for supported abi
# changes (e.g.: a pyenv switch from a py27mu interpreter to a py27m
# interpreter.)
#
# When hashing, we pick 8192 for efficiency of reads and fingerprint updates
# (writes) since it's a common OS buffer size and an even multiple of the
# hash block size.
dedent("""\
import sys
major, minor = sys.version_info[:2]
if not (major == 3 and minor >= 6):
sys.exit(1)
import hashlib
hasher = hashlib.sha256()
with open(sys.executable, "rb") as fp:
for chunk in iter(lambda: fp.read(8192), b""):
hasher.update(chunk)
sys.stdout.write(hasher.hexdigest())
"""),
],
fingerprint_stdout=
False, # We already emit a usable fingerprint to stdout.
),
),
) for binary_name in python_bootstrap.interpreter_names)
for binary_paths in all_python_binary_paths:
path = binary_paths.first_path
if path:
return PythonBinary(
path=path.path,
fingerprint=path.fingerprint,
)
raise BinaryNotFoundError(
"Was not able to locate a Python interpreter to execute rule code.\n"
"Please ensure that Python is available in one of the locations identified by "
"`[python-bootstrap] search_path`, which currently expands to:\n"
f" {interpreter_search_paths}")
async def run_shell_command(
request: GenerateFilesFromShellCommandRequest,
shell_setup: ShellSetup,
bash: BashBinary,
) -> GeneratedSources:
shell_command = request.protocol_target
working_directory = shell_command.address.spec_path
command = shell_command[ShellCommandCommandField].value
tools = shell_command[ShellCommandToolsField].value
outputs = shell_command[ShellCommandOutputsField].value or ()
if not command:
raise ValueError(
f"Missing `command` line in `shell_command` target {shell_command.address}."
)
if not tools:
raise ValueError(
f"Must provide any `tools` used by the `shell_command` {shell_command.address}."
)
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_path = shell_setup.executable_search_path(env)
tool_requests = [
BinaryPathRequest(
binary_name=tool,
search_path=search_path,
) for tool in {*tools, *["mkdir", "ln"]}
if tool not in BASH_BUILTIN_COMMANDS
]
tool_paths = await MultiGet(
Get(BinaryPaths, BinaryPathRequest, request)
for request in tool_requests)
command_env = {
"TOOLS":
" ".join(shlex.quote(tool.binary_name) for tool in tool_requests),
}
for binary, tool_request in zip(tool_paths, tool_requests):
if binary.first_path:
command_env[tool_request.binary_name] = binary.first_path.path
else:
raise BinaryNotFoundError.from_request(
tool_request,
rationale=
f"execute experimental_shell_command {shell_command.address}",
)
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([shell_command.address]),
)
sources = await Get(
SourceFiles,
SourceFilesRequest(
sources_fields=[
tgt.get(Sources) for tgt in transitive_targets.dependencies
],
for_sources_types=(
Sources,
FilesSources,
),
enable_codegen=True,
),
)
output_files = [f for f in outputs if not f.endswith("/")]
output_directories = [d for d in outputs if d.endswith("/")]
if working_directory in sources.snapshot.dirs:
input_digest = sources.snapshot.digest
else:
work_dir = await Get(Digest,
CreateDigest([Directory(working_directory)]))
input_digest = await Get(
Digest, MergeDigests([sources.snapshot.digest, work_dir]))
# Setup bin_relpath dir with symlinks to all requested tools, so that we can use PATH.
bin_relpath = ".bin"
setup_tool_symlinks_script = ";".join(
dedent(f"""\
$mkdir -p {bin_relpath}
for tool in $TOOLS; do $ln -s ${{!tool}} {bin_relpath}/; done
export PATH="$PWD/{bin_relpath}"
""").split("\n"))
result = await Get(
ProcessResult,
Process(
argv=(bash.path, "-c", setup_tool_symlinks_script + command),
description=
f"Running experimental_shell_command {shell_command.address}",
env=command_env,
input_digest=input_digest,
output_directories=output_directories,
output_files=output_files,
working_directory=working_directory,
),
)
if shell_command[ShellCommandLogOutputField].value:
if result.stdout:
logger.info(result.stdout.decode())
if result.stderr:
logger.warning(result.stderr.decode())
output = await Get(Snapshot,
AddPrefix(result.output_digest, working_directory))
return GeneratedSources(output)
async def setup_goroot(golang_subsystem: GolangSubsystem) -> GoRoot:
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_paths = golang_subsystem.go_search_paths(env)
all_go_binary_paths = await Get(
BinaryPaths,
BinaryPathRequest(
search_path=search_paths,
binary_name="go",
test=BinaryPathTest(["version"]),
),
)
if not all_go_binary_paths.paths:
raise BinaryNotFoundError(
"Cannot find any `go` binaries using the option "
f"`[golang].go_search_paths`: {list(search_paths)}\n\n"
"To fix, please install Go (https://golang.org/doc/install) with the version "
f"{golang_subsystem.expected_version} (set by `[golang].expected_version`) and ensure "
"that it is discoverable via `[golang].go_search_paths`.")
# `go env GOVERSION` does not work in earlier Go versions (like 1.15), so we must run
# `go version` and `go env GOROOT` to calculate both the version and GOROOT.
version_results = await MultiGet(
Get(
ProcessResult,
Process(
(binary_path.path, "version"),
description=f"Determine Go version for {binary_path.path}",
level=LogLevel.DEBUG,
cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL,
),
) for binary_path in all_go_binary_paths.paths)
invalid_versions = []
for binary_path, version_result in zip(all_go_binary_paths.paths,
version_results):
try:
_raw_version = version_result.stdout.decode("utf-8").split()[
2] # e.g. go1.17 or go1.17.1
_version_components = _raw_version[2:].split(
".") # e.g. [1, 17] or [1, 17, 1]
version = f"{_version_components[0]}.{_version_components[1]}"
except IndexError:
raise AssertionError(
f"Failed to parse `go version` output for {binary_path}. Please open a bug at "
f"https://github.com/pantsbuild/pants/issues/new/choose with the below data."
f"\n\n"
f"{version_result}")
if version == golang_subsystem.expected_version:
env_result = await Get(
ProcessResult,
Process(
(binary_path.path, "env", "GOROOT"),
description=
f"Determine Go version and GOROOT for {binary_path.path}",
level=LogLevel.DEBUG,
cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL,
env={"GOPATH": "/does/not/matter"},
),
)
goroot = env_result.stdout.decode("utf-8").strip()
return GoRoot(goroot)
logger.debug(
f"Go binary at {binary_path.path} has version {version}, but this "
f"project is using {golang_subsystem.expected_version} "
"(set by `[golang].expected_version`). Ignoring.")
invalid_versions.append((binary_path.path, version))
invalid_versions_str = bullet_list(
f"{path}: {version}" for path, version in sorted(invalid_versions))
raise BinaryNotFoundError(
"Cannot find a `go` binary with the expected version of "
f"{golang_subsystem.expected_version} (set by `[golang].expected_version`).\n\n"
f"Found these `go` binaries, but they had different versions:\n\n"
f"{invalid_versions_str}\n\n"
"To fix, please install the expected version (https://golang.org/doc/install) and ensure "
"that it is discoverable via the option `[golang].go_search_paths`, or change "
"`[golang].expected_version`.")
