async def find_open_program(request: OpenFilesRequest,
plat: Platform) -> OpenFiles:
open_program_name = "open" if plat == Platform.darwin else "xdg-open"
open_program_paths = await Get(
BinaryPaths,
BinaryPathRequest(binary_name=open_program_name,
search_path=("/bin", "/usr/bin")),
)
if not open_program_paths.first_path:
error = (
f"Could not find the program '{open_program_name}' on `/bin` or `/usr/bin`, so cannot "
f"open the files {sorted(request.files)}.")
if request.error_if_open_not_found:
raise OSError(error)
logger.error(error)
return OpenFiles(())
if plat == Platform.darwin:
processes = [
InteractiveProcess(
argv=(open_program_paths.first_path,
*(str(f) for f in request.files)),
run_in_workspace=True,
)
]
else:
processes = [
InteractiveProcess(argv=(open_program_paths.first_path, str(f)),
run_in_workspace=True) for f in request.files
]
return OpenFiles(tuple(processes))
async def debug_python_test(field_set: PythonTestFieldSet) -> TestDebugRequest:
if field_set.is_conftest_or_type_stub():
return TestDebugRequest(None)
setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=True))
return TestDebugRequest(
InteractiveProcess.from_process(setup.process, forward_signals_to_process=False)
)
async def push_docker_images(
request: PublishDockerImageRequest, docker: DockerBinary, options: DockerOptions
) -> PublishProcesses:
tags = tuple(
chain.from_iterable(
cast(BuiltDockerImage, image).tags
for pkg in request.packages
for image in pkg.artifacts
)
)
if request.field_set.skip_push.value:
return PublishProcesses(
[
PublishPackages(
names=tags,
description=f"(by `{request.field_set.skip_push.alias}` on {request.field_set.address})",
),
]
)
process = docker.push_image(tags)
return PublishProcesses(
[
PublishPackages(
names=tags,
process=InteractiveProcess.from_process(process) if process else None,
),
]
)
def make_interactive_process(self) -> InteractiveProcess:
digest = self.request_product(Digest, [
CreateDigest((FileContent(path="program.py",
content=b"def test(): pass"), ))
])
return InteractiveProcess(["/usr/bin/python", "program.py"],
input_digest=digest)
def test_running_interactive_process_in_workspace_cannot_have_input_files(
) -> None:
mock_digest = Digest(EMPTY_DIGEST.fingerprint, 1)
with pytest.raises(ValueError):
InteractiveProcess(argv=["/bin/echo"],
input_digest=mock_digest,
run_in_workspace=True)
async def setup_shunit2_debug_test(
field_set: Shunit2FieldSet) -> TestDebugRequest:
setup = await Get(TestSetup, TestSetupRequest(field_set))
return TestDebugRequest(
InteractiveProcess.from_process(setup.process,
forward_signals_to_process=False,
restartable=True))
async def push_docker_images(
request: PublishDockerImageRequest, docker: DockerBinary, options: DockerOptions
) -> PublishProcesses:
tags = tuple(
chain.from_iterable(
cast(BuiltDockerImage, image).tags
for pkg in request.packages
for image in pkg.artifacts
)
)
if request.field_set.skip_push.value:
return PublishProcesses(
[
PublishPackages(
names=tags,
description=f"(by `{request.field_set.skip_push.alias}` on {request.field_set.address})",
),
]
)
env = await Get(Environment, EnvironmentRequest(options.env_vars))
processes = zip(tags, docker.push_image(tags, env))
return PublishProcesses(
[
PublishPackages(
names=(tag,),
process=InteractiveProcess.from_process(process),
)
for tag, process in processes
]
)
async def run(
run_subsystem: RunSubsystem,
global_options: GlobalOptions,
workspace: Workspace,
build_root: BuildRoot,
complete_env: CompleteEnvironment,
) -> Run:
targets_to_valid_field_sets = await Get(
TargetRootsToFieldSets,
TargetRootsToFieldSetsRequest(
RunFieldSet,
goal_description="the `run` goal",
no_applicable_targets_behavior=NoApplicableTargetsBehavior.error,
expect_single_field_set=True,
),
)
field_set = targets_to_valid_field_sets.field_sets[0]
request = await Get(RunRequest, RunFieldSet, field_set)
wrapped_target = await Get(
WrappedTarget,
WrappedTargetRequest(field_set.address,
description_of_origin="<infallible>"))
restartable = wrapped_target.target.get(RestartableField).value
# Cleanup is the default, so we want to preserve the chroot if either option is off.
cleanup = run_subsystem.cleanup and global_options.process_cleanup
with temporary_dir(root_dir=global_options.pants_workdir,
cleanup=cleanup) as tmpdir:
if not cleanup:
logger.info(f"Preserving running binary chroot {tmpdir}")
workspace.write_digest(
request.digest,
path_prefix=PurePath(tmpdir).relative_to(
build_root.path).as_posix(),
# We don't want to influence whether the InteractiveProcess is able to restart. Because
# we're writing into a temp directory, we can safely mark this side_effecting=False.
side_effecting=False,
)
args = (arg.format(chroot=tmpdir) for arg in request.args)
env = {
**complete_env,
**{
k: v.format(chroot=tmpdir)
for k, v in request.extra_env.items()
}
}
result = await Effect(
InteractiveProcessResult,
InteractiveProcess(
argv=(*args, *run_subsystem.args),
env=env,
run_in_workspace=True,
restartable=restartable,
),
)
exit_code = result.exit_code
return Run(exit_code)
def mock_debug_request(_: TestFieldSet) -> TestDebugRequest:
digest = rule_runner.request(Digest, [
CreateDigest((FileContent(path="program.py",
content=b"def test(): pass"), ))
])
process = InteractiveProcess(["/usr/bin/python", "program.py"],
input_digest=digest)
return TestDebugRequest(process)
def debug_python_test(field_set: PythonTestFieldSet,
setup: TestTargetSetup) -> TestDebugRequest:
if field_set.is_conftest():
return TestDebugRequest(None)
process = InteractiveProcess(
argv=(setup.test_runner_pex.name, *setup.args),
input_digest=setup.input_digest,
)
return TestDebugRequest(process)
async def find_open_program(
request: OpenFilesRequest,
plat: Platform,
complete_env: CompleteEnvironment,
) -> OpenFiles:
open_program_name = "open" if plat.is_macos else "xdg-open"
open_program_paths = await Get(
BinaryPaths,
BinaryPathRequest(binary_name=open_program_name,
search_path=("/bin", "/usr/bin")),
)
if not open_program_paths.first_path:
error = (
f"Could not find the program '{open_program_name}' on `/bin` or `/usr/bin`, so cannot "
f"open the files {sorted(request.files)}.")
if request.error_if_open_not_found:
raise OSError(error)
logger.error(error)
return OpenFiles(())
if plat.is_macos:
processes = [
InteractiveProcess(
argv=(open_program_paths.first_path.path,
*(str(f) for f in request.files)),
run_in_workspace=True,
restartable=True,
)
]
else:
processes = [
InteractiveProcess(
argv=(open_program_paths.first_path.path, str(f)),
run_in_workspace=True,
# The xdg-open binary needs many environment variables to work properly. In addition
# to the various XDG_* environment variables, DISPLAY and other X11 variables are
# required. Instead of attempting to track all of these we just export the full user
# environment since this is not a cached process.
env=complete_env,
restartable=True,
) for f in request.files
]
return OpenFiles(tuple(processes))
async def setup_shunit2_debug_test(
field_set: Shunit2FieldSet) -> TestDebugRequest:
setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=True))
# We set up an InteractiveProcess, which will be executed in the `@goal_rule` in `test.py`.
return TestDebugRequest(
InteractiveProcess(
argv=setup.process.argv,
env=setup.process.env,
input_digest=setup.process.input_digest,
), )
def test_materialize_input_files(self) -> None:
program_text = b'#!/usr/bin/python\nprint("hello")'
binary = self.create_mock_run_request(program_text)
interactive_runner = InteractiveRunner(self.scheduler)
process = InteractiveProcess(
argv=("./program.py", ),
run_in_workspace=False,
input_digest=binary.digest,
)
result = interactive_runner.run(process)
assert result.exit_code == 0
def test_materialize_input_files(rule_runner: RuleRunner) -> None:
program_text = b'#!/usr/bin/python\nprint("hello")'
binary = create_mock_run_request(rule_runner, program_text)
with mock_console(rule_runner.options_bootstrapper):
result = rule_runner.run_interactive_process(
InteractiveProcess(
argv=("./program.py", ),
run_in_workspace=False,
input_digest=binary.digest,
))
assert result.exit_code == 0
async def run_repl(
console: Console,
workspace: Workspace,
repl_subsystem: ReplSubsystem,
all_specified_addresses: Addresses,
build_root: BuildRoot,
union_membership: UnionMembership,
global_options: GlobalOptions,
complete_env: CompleteEnvironment,
) -> Repl:
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest(all_specified_addresses))
# TODO: When we support multiple languages, detect the default repl to use based
# on the targets. For now we default to the python repl.
repl_shell_name = repl_subsystem.shell or "python"
implementations = {
impl.name: impl
for impl in union_membership[ReplImplementation]
}
repl_implementation_cls = implementations.get(repl_shell_name)
if repl_implementation_cls is None:
available = sorted(implementations.keys())
console.print_stderr(
f"{repr(repl_shell_name)} is not a registered REPL. Available REPLs (which may "
f"be specified through the option `--repl-shell`): {available}")
return Repl(-1)
with temporary_dir(root_dir=global_options.options.pants_workdir,
cleanup=False) as tmpdir:
repl_impl = repl_implementation_cls(targets=Targets(
transitive_targets.closure),
chroot=tmpdir)
request = await Get(ReplRequest, ReplImplementation, repl_impl)
workspace.write_digest(
request.digest,
path_prefix=PurePath(tmpdir).relative_to(
build_root.path).as_posix(),
# We don't want to influence whether the InteractiveProcess is able to restart. Because
# we're writing into a temp directory, we can safely mark this side_effecting=False.
side_effecting=False,
)
env = {**complete_env, **request.extra_env}
result = await Effect(
InteractiveProcessResult,
InteractiveProcess(
argv=request.args,
env=env,
run_in_workspace=True,
restartable=repl_subsystem.restartable,
),
)
return Repl(result.exit_code)
async def mock_publish(request: MockPublishRequest) -> PublishProcesses:
if not request.field_set.repositories.value:
return PublishProcesses()
return PublishProcesses(
PublishPackages(
names=tuple(artifact.relpath for pkg in request.packages
for artifact in pkg.artifacts if artifact.relpath),
process=None if repo ==
"skip" else InteractiveProcess(["echo", repo]),
description="(requested)" if repo == "skip" else repo,
) for repo in request.field_set.repositories.value)
async def run(
run_subsystem: RunSubsystem,
global_options: GlobalOptions,
console: Console,
interactive_runner: InteractiveRunner,
workspace: Workspace,
build_root: BuildRoot,
complete_env: CompleteEnvironment,
) -> Run:
targets_to_valid_field_sets = await Get(
TargetRootsToFieldSets,
TargetRootsToFieldSetsRequest(
RunFieldSet,
goal_description="the `run` goal",
no_applicable_targets_behavior=NoApplicableTargetsBehavior.error,
expect_single_field_set=True,
),
)
field_set = targets_to_valid_field_sets.field_sets[0]
request = await Get(RunRequest, RunFieldSet, field_set)
with temporary_dir(root_dir=global_options.options.pants_workdir,
cleanup=True) as tmpdir:
workspace.write_digest(request.digest,
path_prefix=PurePath(tmpdir).relative_to(
build_root.path).as_posix())
args = (arg.format(chroot=tmpdir) for arg in request.args)
env = {
**complete_env,
**{
k: v.format(chroot=tmpdir)
for k, v in request.extra_env.items()
}
}
try:
result = interactive_runner.run(
InteractiveProcess(
argv=(*args, *run_subsystem.args),
env=env,
run_in_workspace=True,
))
exit_code = result.exit_code
except Exception as e:
console.print_stderr(
f"Exception when attempting to run {field_set.address}: {e!r}")
exit_code = -1
return Run(exit_code)
async def run_repl(
console: Console,
workspace: Workspace,
interactive_runner: InteractiveRunner,
repl_subsystem: ReplSubsystem,
all_specified_addresses: Addresses,
build_root: BuildRoot,
union_membership: UnionMembership,
global_options: GlobalOptions,
) -> Repl:
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest(all_specified_addresses))
# TODO: When we support multiple languages, detect the default repl to use based
# on the targets. For now we default to the python repl.
repl_shell_name = repl_subsystem.shell or "python"
implementations: Dict[str, Type[ReplImplementation]] = {
impl.name: impl
for impl in union_membership[ReplImplementation]
}
repl_implementation_cls = implementations.get(repl_shell_name)
if repl_implementation_cls is None:
available = sorted(implementations.keys())
console.print_stderr(
f"{repr(repl_shell_name)} is not a registered REPL. Available REPLs (which may "
f"be specified through the option `--repl-shell`): {available}")
return Repl(-1)
with temporary_dir(root_dir=global_options.options.pants_workdir,
cleanup=False) as tmpdir:
repl_impl = repl_implementation_cls(targets=Targets(
transitive_targets.closure),
chroot=tmpdir)
request = await Get(ReplRequest, ReplImplementation, repl_impl)
workspace.write_digest(request.digest,
path_prefix=PurePath(tmpdir).relative_to(
build_root.path).as_posix())
result = interactive_runner.run(
InteractiveProcess(argv=request.args,
env=request.extra_env,
run_in_workspace=True,
hermetic_env=False))
return Repl(result.exit_code)
async def run_repl(
console: Console,
workspace: Workspace,
interactive_runner: InteractiveRunner,
repl_subsystem: ReplSubsystem,
transitive_targets: TransitiveTargets,
build_root: BuildRoot,
union_membership: UnionMembership,
global_options: GlobalOptions,
) -> Repl:
repl_shell_name = repl_subsystem.shell or "python"
implementations: Dict[str, Type[ReplImplementation]] = {
impl.name: impl
for impl in union_membership[ReplImplementation]
}
repl_implementation_cls = implementations.get(repl_shell_name)
if repl_implementation_cls is None:
available = sorted(implementations.keys())
console.print_stderr(
f"{repr(repl_shell_name)} is not a registered REPL. Available REPLs (which may "
f"be specified through the option `--repl-shell`): {available}")
return Repl(-1)
repl_impl = repl_implementation_cls(targets=Targets(
tgt for tgt in transitive_targets.closure
if repl_implementation_cls.is_valid(tgt)))
request = await Get(ReplRequest, ReplImplementation, repl_impl)
with temporary_dir(root_dir=global_options.options.pants_workdir,
cleanup=False) as tmpdir:
tmpdir_relative_path = PurePath(tmpdir).relative_to(
build_root.path).as_posix()
exe_path = PurePath(tmpdir, request.binary_name).as_posix()
workspace.write_digest(request.digest,
path_prefix=tmpdir_relative_path)
result = interactive_runner.run(
InteractiveProcess(argv=(exe_path, ),
env=request.env,
run_in_workspace=True))
return Repl(result.exit_code)
async def run(
run_subsystem: RunSubsystem,
global_options: GlobalOptions,
console: Console,
interactive_runner: InteractiveRunner,
workspace: Workspace,
build_root: BuildRoot,
) -> Run:
targets_to_valid_field_sets = await Get(
TargetsToValidFieldSets,
TargetsToValidFieldSetsRequest(
BinaryFieldSet,
goal_description="the `run` goal",
error_if_no_valid_targets=True,
expect_single_field_set=True,
),
)
field_set = targets_to_valid_field_sets.field_sets[0]
request = await Get(RunRequest, BinaryFieldSet, field_set)
with temporary_dir(root_dir=global_options.options.pants_workdir,
cleanup=True) as tmpdir:
tmpdir_relative_path = PurePath(tmpdir).relative_to(
build_root.path).as_posix()
workspace.write_digest(request.digest,
path_prefix=tmpdir_relative_path)
exe_path = PurePath(tmpdir, request.binary_name).as_posix()
process = InteractiveProcess(
argv=(exe_path, *request.prefix_args, *run_subsystem.args),
env=request.env,
run_in_workspace=True,
)
try:
result = interactive_runner.run(process)
exit_code = result.exit_code
except Exception as e:
console.print_stderr(
f"Exception when attempting to run {field_set.address}: {e!r}")
exit_code = -1
return Run(exit_code)
async def export(
console: Console,
targets: Targets,
workspace: Workspace,
union_membership: UnionMembership,
build_root: BuildRoot,
dist_dir: DistDir,
) -> Export:
request_types = cast("Iterable[type[ExportRequest]]",
union_membership.get(ExportRequest))
requests = tuple(request_type(targets) for request_type in request_types)
all_results = await MultiGet(
Get(ExportResults, ExportRequest, request) for request in requests)
flattened_results = [res for results in all_results for res in results]
prefixed_digests = await MultiGet(
Get(Digest, AddPrefix(result.digest, result.reldir))
for result in flattened_results)
output_dir = os.path.join(str(dist_dir.relpath), "export")
merged_digest = await Get(Digest, MergeDigests(prefixed_digests))
dist_digest = await Get(Digest, AddPrefix(merged_digest, output_dir))
workspace.write_digest(dist_digest)
environment = await Get(Environment, EnvironmentRequest(["PATH"]))
for result in flattened_results:
digest_root = os.path.join(build_root.path, output_dir, result.reldir)
for cmd in result.post_processing_cmds:
argv = tuple(
arg.format(digest_root=digest_root) for arg in cmd.argv)
ip = InteractiveProcess(
argv=argv,
env={
"PATH": environment.get("PATH", ""),
**cmd.extra_env
},
run_in_workspace=True,
)
await Effect(InteractiveProcessResult, InteractiveProcess, ip)
console.print_stdout(
f"Wrote {result.description} to {os.path.join(output_dir, result.reldir)}"
)
return Export(exit_code=0)
async def debug_python_test(field_set: PythonTestFieldSet) -> TestDebugRequest:
if field_set.is_conftest():
return TestDebugRequest(None)
setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=True))
return TestDebugRequest(InteractiveProcess.from_process(setup.process))
async def debug_python_test(field_set: PythonTestFieldSet) -> TestDebugRequest:
setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=True))
return TestDebugRequest(
InteractiveProcess.from_process(setup.process,
forward_signals_to_process=False,
restartable=True))
def mock_debug_request(_: TestFieldSet) -> TestDebugRequest:
return TestDebugRequest(
InteractiveProcess(["/bin/example"], input_digest=EMPTY_DIGEST))
def test_interactive_process_cannot_have_append_only_caches_and_workspace() -> None:
with pytest.raises(ValueError):
InteractiveProcess(
argv=["/bin/echo"], append_only_caches={"foo": "bar"}, run_in_workspace=True
)
def _iter_openers(self, files: Iterable[PurePath]) -> Iterator[InteractiveProcess]:
for f in files:
yield InteractiveProcess(argv=(self.program, str(f)), run_in_workspace=True)
async def twine_upload(request: PublishToPyPiRequest,
twine_subsystem: TwineSubsystem) -> PublishProcesses:
dists = tuple(artifact.relpath for pkg in request.packages
for artifact in pkg.artifacts if artifact.relpath)
if twine_subsystem.skip or not dists:
return PublishProcesses()
# Too verbose to provide feedback as to why some packages were skipped?
skip = None
if request.field_set.skip_twine.value:
skip = f"(by `{request.field_set.skip_twine.alias}` on {request.field_set.address})"
elif not request.field_set.repositories.value:
# I'd rather have used the opt_out mechanism on the field set, but that gives no hint as to
# why the target was not applicable..
skip = f"(no `{request.field_set.repositories.alias}` specified for {request.field_set.address})"
if skip:
return PublishProcesses([
PublishPackages(
names=dists,
description=skip,
),
])
twine_pex, packages_digest, config_files = await MultiGet(
Get(
VenvPex,
PexRequest(
output_filename="twine.pex",
internal_only=True,
requirements=twine_subsystem.pex_requirements(),
interpreter_constraints=twine_subsystem.
interpreter_constraints,
main=twine_subsystem.main,
),
),
Get(Digest, MergeDigests(pkg.digest for pkg in request.packages)),
Get(ConfigFiles, ConfigFilesRequest, twine_subsystem.config_request()),
)
input_digest = await Get(
Digest, MergeDigests((packages_digest, config_files.snapshot.digest)))
pex_proc_requests = []
twine_envs = await MultiGet(
Get(Environment, EnvironmentRequest, twine_env_request(repo))
for repo in request.field_set.repositories.value)
for repo, env in zip(request.field_set.repositories.value, twine_envs):
pex_proc_requests.append(
VenvPexProcess(
twine_pex,
argv=twine_upload_args(twine_subsystem, config_files, repo,
dists),
input_digest=input_digest,
extra_env=twine_env(env, repo),
description=repo,
))
processes = await MultiGet(
Get(Process, VenvPexProcess, request) for request in pex_proc_requests)
return PublishProcesses(
PublishPackages(
names=dists,
process=InteractiveProcess.from_process(process),
description=process.description,
data=PublishOutputData({"repository": process.description}),
) for process in processes)
async def debug_python_test(test_setup: TestTargetSetup) -> TestDebugRequest:
process = InteractiveProcess(
argv=(test_setup.test_runner_pex.output_filename, *test_setup.args),
input_digest=test_setup.input_digest,
)
return TestDebugRequest(process)
