async def determine_shunit2_shell(request: Shunit2RunnerRequest,
shell_setup: ShellSetup) -> Shunit2Runner:
if request.shell_field.value is not None:
tgt_shell = Shunit2Shell(request.shell_field.value)
else:
parse_result = Shunit2Shell.parse_shebang(
request.test_file_content.content)
if parse_result is None:
raise ShellNotConfigured(
f"Could not determine which shell to use to run shunit2 on {request.address}.\n\n"
f"Please either specify the `{Shunit2ShellField.alias}` field or add a "
f"shebang to {request.test_file_content.path} with one of the supported shells in "
f"the format `!#/path/to/shell` or `!#/path/to/env shell`"
f"(run `./pants help {Shunit2Tests.alias}` for valid shells).")
tgt_shell = parse_result
env = await Get(Environment, EnvironmentRequest(["PATH"]))
path_request = BinaryPathRequest(
binary_name=tgt_shell.name,
search_path=shell_setup.executable_search_path(env),
test=tgt_shell.binary_path_test,
)
paths = await Get(BinaryPaths, BinaryPathRequest, path_request)
first_path = paths.first_path
if not first_path:
raise BinaryNotFoundError(
path_request, rationale=f"run shunit2 on {request.address}")
return Shunit2Runner(tgt_shell, first_path)
async def prepare_shell_command_process(request: ShellCommandProcessRequest,
shell_setup: ShellSetup,
bash: BashBinary) -> Process:
shell_command = request.target
interactive = shell_command.has_field(ShellCommandRunWorkdirField)
if interactive:
working_directory = shell_command[
ShellCommandRunWorkdirField].value or ""
else:
working_directory = shell_command.address.spec_path
command = shell_command[ShellCommandCommandField].value
timeout = shell_command.get(ShellCommandTimeoutField).value
tools = shell_command.get(ShellCommandToolsField,
default_raw_value=()).value
outputs = shell_command.get(ShellCommandOutputsField).value or ()
extra_env_vars = shell_command.get(
ShellCommandExtraEnvVarsField).value or ()
if not command:
raise ValueError(
f"Missing `command` line in `{shell_command.alias}` target {shell_command.address}."
)
if interactive:
command_env = {
"CHROOT": "{chroot}",
}
else:
if not tools:
raise ValueError(
f"Must provide any `tools` used by the `{shell_command.alias}` {shell_command.address}."
)
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_path = shell_setup.executable_search_path(env)
tool_requests = [
BinaryPathRequest(
binary_name=tool,
search_path=search_path,
) for tool in {*tools, *["mkdir", "ln"]}
if tool not in BASH_BUILTIN_COMMANDS
]
tool_paths = await MultiGet(
Get(BinaryPaths, BinaryPathRequest, request)
for request in tool_requests)
command_env = {
"TOOLS":
" ".join(
_shell_tool_safe_env_name(tool.binary_name)
for tool in tool_requests),
}
for binary, tool_request in zip(tool_paths, tool_requests):
if binary.first_path:
command_env[_shell_tool_safe_env_name(
tool_request.binary_name)] = binary.first_path.path
else:
raise BinaryNotFoundError.from_request(
tool_request,
rationale=
f"execute `{shell_command.alias}` {shell_command.address}",
)
extra_env = await Get(Environment, EnvironmentRequest(extra_env_vars))
command_env.update(extra_env)
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([shell_command.address]),
)
sources, pkgs_per_target = await MultiGet(
Get(
SourceFiles,
SourceFilesRequest(
sources_fields=[
tgt.get(SourcesField)
for tgt in transitive_targets.dependencies
],
for_sources_types=(SourcesField, FileSourceField),
enable_codegen=True,
),
),
Get(
FieldSetsPerTarget,
FieldSetsPerTargetRequest(PackageFieldSet,
transitive_targets.dependencies),
),
)
packages = await MultiGet(
Get(BuiltPackage, PackageFieldSet, field_set)
for field_set in pkgs_per_target.field_sets)
if interactive or not working_directory or working_directory in sources.snapshot.dirs:
work_dir = EMPTY_DIGEST
else:
work_dir = await Get(Digest,
CreateDigest([Directory(working_directory)]))
input_digest = await Get(
Digest,
MergeDigests([
sources.snapshot.digest, work_dir,
*(pkg.digest for pkg in packages)
]))
output_files = [f for f in outputs if not f.endswith("/")]
output_directories = [d for d in outputs if d.endswith("/")]
if interactive:
relpath = os.path.relpath(
working_directory or ".",
start="/" if os.path.isabs(working_directory) else ".")
boot_script = f"cd {shlex.quote(relpath)}; " if relpath != "." else ""
else:
# Setup bin_relpath dir with symlinks to all requested tools, so that we can use PATH, force
# symlinks to avoid issues with repeat runs using the __run.sh script in the sandbox.
bin_relpath = ".bin"
boot_script = ";".join(
dedent(f"""\
$mkdir -p {bin_relpath}
for tool in $TOOLS; do $ln -sf ${{!tool}} {bin_relpath}; done
export PATH="$PWD/{bin_relpath}"
""").split("\n"))
return Process(
argv=(bash.path, "-c", boot_script + command),
description=f"Running {shell_command.alias} {shell_command.address}",
env=command_env,
input_digest=input_digest,
output_directories=output_directories,
output_files=output_files,
timeout_seconds=timeout,
working_directory=working_directory,
)
async def setup_shunit2_for_target(
request: TestSetupRequest,
shell_setup: ShellSetup,
test_subsystem: TestSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
) -> TestSetup:
shunit2_download_file = DownloadFile(
"https://raw.githubusercontent.com/kward/shunit2/b9102bb763cc603b3115ed30a5648bf950548097/shunit2",
FileDigest(
"1f11477b7948150d1ca50cdd41d89be4ed2acd137e26d2e0fe23966d0e272cc5",
40987),
)
shunit2_script, transitive_targets, built_package_dependencies, env = await MultiGet(
Get(Digest, DownloadFile, shunit2_download_file),
Get(TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address])),
Get(
BuiltPackageDependencies,
BuildPackageDependenciesRequest(
request.field_set.runtime_package_dependencies),
),
Get(Environment, EnvironmentRequest(["PATH"])),
)
dependencies_source_files_request = Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.dependencies),
for_sources_types=(ShellSources, FilesSources, ResourcesSources),
enable_codegen=True,
),
)
dependencies_source_files, field_set_sources = await MultiGet(
dependencies_source_files_request,
Get(SourceFiles, SourceFilesRequest([request.field_set.sources])),
)
field_set_digest_content = await Get(DigestContents, Digest,
field_set_sources.snapshot.digest)
# Because a FieldSet corresponds to a file address, there should be exactly 1 file in the
# sources. This assumption allows us to simplify determining which shell to use via inspecting
# the shebang.
if len(field_set_digest_content) != 1:
raise AssertionError(
f"The file address {request.field_set.address} had sources != 1, which is unexpected: "
f"{field_set_sources.snapshot.files}. Please file a bug at "
"https://github.com/pantsbuild/pants/issues/new with this error message copied."
)
original_test_file_content = field_set_digest_content[0]
updated_test_file_content = add_source_shunit2(original_test_file_content)
updated_test_digest, runner = await MultiGet(
Get(Digest, CreateDigest([updated_test_file_content])),
Get(
Shunit2Runner,
Shunit2RunnerRequest(request.field_set.address,
original_test_file_content,
request.field_set.shell),
),
)
input_digest = await Get(
Digest,
MergeDigests((
shunit2_script,
updated_test_digest,
dependencies_source_files.snapshot.digest,
*(pkg.digest for pkg in built_package_dependencies),
)),
)
env_dict = {
"PATH": create_path_env_var(shell_setup.executable_search_path(env)),
"SHUNIT_COLOR": "always" if global_options.options.colors else "none",
**test_extra_env.env,
}
argv = (
# Zsh requires extra args. See https://github.com/kward/shunit2/#-zsh.
[
runner.binary_path.path, "-o", "shwordsplit", "--",
*field_set_sources.snapshot.files
] if runner.shell == Shunit2Shell.zsh else
[runner.binary_path.path, *field_set_sources.snapshot.files])
cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL
process = Process(
argv=argv,
input_digest=input_digest,
description=f"Run shunit2 for {request.field_set.address}.",
level=LogLevel.DEBUG,
env=env_dict,
timeout_seconds=request.field_set.timeout.value,
cache_scope=cache_scope,
)
return TestSetup(process)
async def run_shell_command(
request: GenerateFilesFromShellCommandRequest,
shell_setup: ShellSetup,
bash: BashBinary,
) -> GeneratedSources:
shell_command = request.protocol_target
working_directory = shell_command.address.spec_path
command = shell_command[ShellCommandCommandField].value
tools = shell_command[ShellCommandToolsField].value
outputs = shell_command[ShellCommandOutputsField].value or ()
if not command:
raise ValueError(
f"Missing `command` line in `shell_command` target {shell_command.address}."
)
if not tools:
raise ValueError(
f"Must provide any `tools` used by the `shell_command` {shell_command.address}."
)
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_path = shell_setup.executable_search_path(env)
tool_requests = [
BinaryPathRequest(
binary_name=tool,
search_path=search_path,
) for tool in {*tools, *["mkdir", "ln"]}
if tool not in BASH_BUILTIN_COMMANDS
]
tool_paths = await MultiGet(
Get(BinaryPaths, BinaryPathRequest, request)
for request in tool_requests)
command_env = {
"TOOLS":
" ".join(shlex.quote(tool.binary_name) for tool in tool_requests),
}
for binary, tool_request in zip(tool_paths, tool_requests):
if binary.first_path:
command_env[tool_request.binary_name] = binary.first_path.path
else:
raise BinaryNotFoundError.from_request(
tool_request,
rationale=
f"execute experimental_shell_command {shell_command.address}",
)
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([shell_command.address]),
)
sources = await Get(
SourceFiles,
SourceFilesRequest(
sources_fields=[
tgt.get(Sources) for tgt in transitive_targets.dependencies
],
for_sources_types=(
Sources,
FilesSources,
),
enable_codegen=True,
),
)
output_files = [f for f in outputs if not f.endswith("/")]
output_directories = [d for d in outputs if d.endswith("/")]
if working_directory in sources.snapshot.dirs:
input_digest = sources.snapshot.digest
else:
work_dir = await Get(Digest,
CreateDigest([Directory(working_directory)]))
input_digest = await Get(
Digest, MergeDigests([sources.snapshot.digest, work_dir]))
# Setup bin_relpath dir with symlinks to all requested tools, so that we can use PATH.
bin_relpath = ".bin"
setup_tool_symlinks_script = ";".join(
dedent(f"""\
$mkdir -p {bin_relpath}
for tool in $TOOLS; do $ln -s ${{!tool}} {bin_relpath}/; done
export PATH="$PWD/{bin_relpath}"
""").split("\n"))
result = await Get(
ProcessResult,
Process(
argv=(bash.path, "-c", setup_tool_symlinks_script + command),
description=
f"Running experimental_shell_command {shell_command.address}",
env=command_env,
input_digest=input_digest,
output_directories=output_directories,
output_files=output_files,
working_directory=working_directory,
),
)
if shell_command[ShellCommandLogOutputField].value:
if result.stdout:
logger.info(result.stdout.decode())
if result.stderr:
logger.warning(result.stderr.decode())
output = await Get(Snapshot,
AddPrefix(result.output_digest, working_directory))
return GeneratedSources(output)
async def setup_shunit2_for_target(
request: TestSetupRequest,
shell_setup: ShellSetup,
test_subsystem: TestSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
) -> TestSetup:
shunit2_download_file = DownloadFile(
"https://raw.githubusercontent.com/kward/shunit2/b9102bb763cc603b3115ed30a5648bf950548097/shunit2",
FileDigest(
"1f11477b7948150d1ca50cdd41d89be4ed2acd137e26d2e0fe23966d0e272cc5",
40987),
)
shunit2_script, transitive_targets, built_package_dependencies, env = await MultiGet(
Get(Digest, DownloadFile, shunit2_download_file),
Get(TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address])),
Get(
BuiltPackageDependencies,
BuildPackageDependenciesRequest(
request.field_set.runtime_package_dependencies),
),
Get(Environment, EnvironmentRequest(["PATH"])),
)
dependencies_source_files_request = Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(SourcesField) for tgt in transitive_targets.dependencies),
for_sources_types=(ShellSourceField, FileSourceField,
ResourceSourceField),
enable_codegen=True,
),
)
dependencies_source_files, field_set_sources = await MultiGet(
dependencies_source_files_request,
Get(SourceFiles, SourceFilesRequest([request.field_set.sources])),
)
field_set_digest_content = await Get(DigestContents, Digest,
field_set_sources.snapshot.digest)
# `ShellTestSourceField` validates that there's exactly one file.
test_file_content = field_set_digest_content[0]
updated_test_file_content = add_source_shunit2(test_file_content)
updated_test_digest, runner = await MultiGet(
Get(Digest, CreateDigest([updated_test_file_content])),
Get(
Shunit2Runner,
Shunit2RunnerRequest(request.field_set.address, test_file_content,
request.field_set.shell),
),
)
input_digest = await Get(
Digest,
MergeDigests((
shunit2_script,
updated_test_digest,
dependencies_source_files.snapshot.digest,
*(pkg.digest for pkg in built_package_dependencies),
)),
)
env_dict = {
"PATH": create_path_env_var(shell_setup.executable_search_path(env)),
"SHUNIT_COLOR": "always" if global_options.colors else "none",
**test_extra_env.env,
}
argv = (
# Zsh requires extra args. See https://github.com/kward/shunit2/#-zsh.
[
runner.binary_path.path, "-o", "shwordsplit", "--",
*field_set_sources.snapshot.files
] if runner.shell == Shunit2Shell.zsh else
[runner.binary_path.path, *field_set_sources.snapshot.files])
cache_scope = (ProcessCacheScope.PER_SESSION
if test_subsystem.force else ProcessCacheScope.SUCCESSFUL)
process = Process(
argv=argv,
input_digest=input_digest,
description=f"Run shunit2 for {request.field_set.address}.",
level=LogLevel.DEBUG,
env=env_dict,
timeout_seconds=request.field_set.timeout.value,
cache_scope=cache_scope,
)
return TestSetup(process)