async def find_open_program(request: OpenFilesRequest, plat: Platform) -> OpenFiles: open_program_name = "open" if plat == Platform.darwin else "xdg-open" open_program_paths = await Get( BinaryPaths, BinaryPathRequest(binary_name=open_program_name, search_path=("/bin", "/usr/bin")), ) if not open_program_paths.first_path: error = ( f"Could not find the program '{open_program_name}' on `/bin` or `/usr/bin`, so cannot " f"open the files {sorted(request.files)}.") if request.error_if_open_not_found: raise OSError(error) logger.error(error) return OpenFiles(()) if plat == Platform.darwin: processes = [ InteractiveProcess( argv=(open_program_paths.first_path, *(str(f) for f in request.files)), run_in_workspace=True, ) ] else: processes = [ InteractiveProcess(argv=(open_program_paths.first_path, str(f)), run_in_workspace=True) for f in request.files ] return OpenFiles(tuple(processes))
async def debug_python_test(field_set: PythonTestFieldSet) -> TestDebugRequest: if field_set.is_conftest_or_type_stub(): return TestDebugRequest(None) setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=True)) return TestDebugRequest( InteractiveProcess.from_process(setup.process, forward_signals_to_process=False) )
async def push_docker_images( request: PublishDockerImageRequest, docker: DockerBinary, options: DockerOptions ) -> PublishProcesses: tags = tuple( chain.from_iterable( cast(BuiltDockerImage, image).tags for pkg in request.packages for image in pkg.artifacts ) ) if request.field_set.skip_push.value: return PublishProcesses( [ PublishPackages( names=tags, description=f"(by `{request.field_set.skip_push.alias}` on {request.field_set.address})", ), ] ) process = docker.push_image(tags) return PublishProcesses( [ PublishPackages( names=tags, process=InteractiveProcess.from_process(process) if process else None, ), ] )
def make_interactive_process(self) -> InteractiveProcess: digest = self.request_product(Digest, [ CreateDigest((FileContent(path="program.py", content=b"def test(): pass"), )) ]) return InteractiveProcess(["/usr/bin/python", "program.py"], input_digest=digest)
def test_running_interactive_process_in_workspace_cannot_have_input_files( ) -> None: mock_digest = Digest(EMPTY_DIGEST.fingerprint, 1) with pytest.raises(ValueError): InteractiveProcess(argv=["/bin/echo"], input_digest=mock_digest, run_in_workspace=True)
async def setup_shunit2_debug_test( field_set: Shunit2FieldSet) -> TestDebugRequest: setup = await Get(TestSetup, TestSetupRequest(field_set)) return TestDebugRequest( InteractiveProcess.from_process(setup.process, forward_signals_to_process=False, restartable=True))
async def push_docker_images( request: PublishDockerImageRequest, docker: DockerBinary, options: DockerOptions ) -> PublishProcesses: tags = tuple( chain.from_iterable( cast(BuiltDockerImage, image).tags for pkg in request.packages for image in pkg.artifacts ) ) if request.field_set.skip_push.value: return PublishProcesses( [ PublishPackages( names=tags, description=f"(by `{request.field_set.skip_push.alias}` on {request.field_set.address})", ), ] ) env = await Get(Environment, EnvironmentRequest(options.env_vars)) processes = zip(tags, docker.push_image(tags, env)) return PublishProcesses( [ PublishPackages( names=(tag,), process=InteractiveProcess.from_process(process), ) for tag, process in processes ] )
async def run( run_subsystem: RunSubsystem, global_options: GlobalOptions, workspace: Workspace, build_root: BuildRoot, complete_env: CompleteEnvironment, ) -> Run: targets_to_valid_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( RunFieldSet, goal_description="the `run` goal", no_applicable_targets_behavior=NoApplicableTargetsBehavior.error, expect_single_field_set=True, ), ) field_set = targets_to_valid_field_sets.field_sets[0] request = await Get(RunRequest, RunFieldSet, field_set) wrapped_target = await Get( WrappedTarget, WrappedTargetRequest(field_set.address, description_of_origin="<infallible>")) restartable = wrapped_target.target.get(RestartableField).value # Cleanup is the default, so we want to preserve the chroot if either option is off. cleanup = run_subsystem.cleanup and global_options.process_cleanup with temporary_dir(root_dir=global_options.pants_workdir, cleanup=cleanup) as tmpdir: if not cleanup: logger.info(f"Preserving running binary chroot {tmpdir}") workspace.write_digest( request.digest, path_prefix=PurePath(tmpdir).relative_to( build_root.path).as_posix(), # We don't want to influence whether the InteractiveProcess is able to restart. Because # we're writing into a temp directory, we can safely mark this side_effecting=False. side_effecting=False, ) args = (arg.format(chroot=tmpdir) for arg in request.args) env = { **complete_env, **{ k: v.format(chroot=tmpdir) for k, v in request.extra_env.items() } } result = await Effect( InteractiveProcessResult, InteractiveProcess( argv=(*args, *run_subsystem.args), env=env, run_in_workspace=True, restartable=restartable, ), ) exit_code = result.exit_code return Run(exit_code)
def mock_debug_request(_: TestFieldSet) -> TestDebugRequest: digest = rule_runner.request(Digest, [ CreateDigest((FileContent(path="program.py", content=b"def test(): pass"), )) ]) process = InteractiveProcess(["/usr/bin/python", "program.py"], input_digest=digest) return TestDebugRequest(process)
def debug_python_test(field_set: PythonTestFieldSet, setup: TestTargetSetup) -> TestDebugRequest: if field_set.is_conftest(): return TestDebugRequest(None) process = InteractiveProcess( argv=(setup.test_runner_pex.name, *setup.args), input_digest=setup.input_digest, ) return TestDebugRequest(process)
async def find_open_program( request: OpenFilesRequest, plat: Platform, complete_env: CompleteEnvironment, ) -> OpenFiles: open_program_name = "open" if plat.is_macos else "xdg-open" open_program_paths = await Get( BinaryPaths, BinaryPathRequest(binary_name=open_program_name, search_path=("/bin", "/usr/bin")), ) if not open_program_paths.first_path: error = ( f"Could not find the program '{open_program_name}' on `/bin` or `/usr/bin`, so cannot " f"open the files {sorted(request.files)}.") if request.error_if_open_not_found: raise OSError(error) logger.error(error) return OpenFiles(()) if plat.is_macos: processes = [ InteractiveProcess( argv=(open_program_paths.first_path.path, *(str(f) for f in request.files)), run_in_workspace=True, restartable=True, ) ] else: processes = [ InteractiveProcess( argv=(open_program_paths.first_path.path, str(f)), run_in_workspace=True, # The xdg-open binary needs many environment variables to work properly. In addition # to the various XDG_* environment variables, DISPLAY and other X11 variables are # required. Instead of attempting to track all of these we just export the full user # environment since this is not a cached process. env=complete_env, restartable=True, ) for f in request.files ] return OpenFiles(tuple(processes))
async def setup_shunit2_debug_test( field_set: Shunit2FieldSet) -> TestDebugRequest: setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=True)) # We set up an InteractiveProcess, which will be executed in the `@goal_rule` in `test.py`. return TestDebugRequest( InteractiveProcess( argv=setup.process.argv, env=setup.process.env, input_digest=setup.process.input_digest, ), )
def test_materialize_input_files(self) -> None: program_text = b'#!/usr/bin/python\nprint("hello")' binary = self.create_mock_run_request(program_text) interactive_runner = InteractiveRunner(self.scheduler) process = InteractiveProcess( argv=("./program.py", ), run_in_workspace=False, input_digest=binary.digest, ) result = interactive_runner.run(process) assert result.exit_code == 0
def test_materialize_input_files(rule_runner: RuleRunner) -> None: program_text = b'#!/usr/bin/python\nprint("hello")' binary = create_mock_run_request(rule_runner, program_text) with mock_console(rule_runner.options_bootstrapper): result = rule_runner.run_interactive_process( InteractiveProcess( argv=("./program.py", ), run_in_workspace=False, input_digest=binary.digest, )) assert result.exit_code == 0
async def run_repl( console: Console, workspace: Workspace, repl_subsystem: ReplSubsystem, all_specified_addresses: Addresses, build_root: BuildRoot, union_membership: UnionMembership, global_options: GlobalOptions, complete_env: CompleteEnvironment, ) -> Repl: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(all_specified_addresses)) # TODO: When we support multiple languages, detect the default repl to use based # on the targets. For now we default to the python repl. repl_shell_name = repl_subsystem.shell or "python" implementations = { impl.name: impl for impl in union_membership[ReplImplementation] } repl_implementation_cls = implementations.get(repl_shell_name) if repl_implementation_cls is None: available = sorted(implementations.keys()) console.print_stderr( f"{repr(repl_shell_name)} is not a registered REPL. Available REPLs (which may " f"be specified through the option `--repl-shell`): {available}") return Repl(-1) with temporary_dir(root_dir=global_options.options.pants_workdir, cleanup=False) as tmpdir: repl_impl = repl_implementation_cls(targets=Targets( transitive_targets.closure), chroot=tmpdir) request = await Get(ReplRequest, ReplImplementation, repl_impl) workspace.write_digest( request.digest, path_prefix=PurePath(tmpdir).relative_to( build_root.path).as_posix(), # We don't want to influence whether the InteractiveProcess is able to restart. Because # we're writing into a temp directory, we can safely mark this side_effecting=False. side_effecting=False, ) env = {**complete_env, **request.extra_env} result = await Effect( InteractiveProcessResult, InteractiveProcess( argv=request.args, env=env, run_in_workspace=True, restartable=repl_subsystem.restartable, ), ) return Repl(result.exit_code)
async def mock_publish(request: MockPublishRequest) -> PublishProcesses: if not request.field_set.repositories.value: return PublishProcesses() return PublishProcesses( PublishPackages( names=tuple(artifact.relpath for pkg in request.packages for artifact in pkg.artifacts if artifact.relpath), process=None if repo == "skip" else InteractiveProcess(["echo", repo]), description="(requested)" if repo == "skip" else repo, ) for repo in request.field_set.repositories.value)
async def run( run_subsystem: RunSubsystem, global_options: GlobalOptions, console: Console, interactive_runner: InteractiveRunner, workspace: Workspace, build_root: BuildRoot, complete_env: CompleteEnvironment, ) -> Run: targets_to_valid_field_sets = await Get( TargetRootsToFieldSets, TargetRootsToFieldSetsRequest( RunFieldSet, goal_description="the `run` goal", no_applicable_targets_behavior=NoApplicableTargetsBehavior.error, expect_single_field_set=True, ), ) field_set = targets_to_valid_field_sets.field_sets[0] request = await Get(RunRequest, RunFieldSet, field_set) with temporary_dir(root_dir=global_options.options.pants_workdir, cleanup=True) as tmpdir: workspace.write_digest(request.digest, path_prefix=PurePath(tmpdir).relative_to( build_root.path).as_posix()) args = (arg.format(chroot=tmpdir) for arg in request.args) env = { **complete_env, **{ k: v.format(chroot=tmpdir) for k, v in request.extra_env.items() } } try: result = interactive_runner.run( InteractiveProcess( argv=(*args, *run_subsystem.args), env=env, run_in_workspace=True, )) exit_code = result.exit_code except Exception as e: console.print_stderr( f"Exception when attempting to run {field_set.address}: {e!r}") exit_code = -1 return Run(exit_code)
async def run_repl( console: Console, workspace: Workspace, interactive_runner: InteractiveRunner, repl_subsystem: ReplSubsystem, all_specified_addresses: Addresses, build_root: BuildRoot, union_membership: UnionMembership, global_options: GlobalOptions, ) -> Repl: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(all_specified_addresses)) # TODO: When we support multiple languages, detect the default repl to use based # on the targets. For now we default to the python repl. repl_shell_name = repl_subsystem.shell or "python" implementations: Dict[str, Type[ReplImplementation]] = { impl.name: impl for impl in union_membership[ReplImplementation] } repl_implementation_cls = implementations.get(repl_shell_name) if repl_implementation_cls is None: available = sorted(implementations.keys()) console.print_stderr( f"{repr(repl_shell_name)} is not a registered REPL. Available REPLs (which may " f"be specified through the option `--repl-shell`): {available}") return Repl(-1) with temporary_dir(root_dir=global_options.options.pants_workdir, cleanup=False) as tmpdir: repl_impl = repl_implementation_cls(targets=Targets( transitive_targets.closure), chroot=tmpdir) request = await Get(ReplRequest, ReplImplementation, repl_impl) workspace.write_digest(request.digest, path_prefix=PurePath(tmpdir).relative_to( build_root.path).as_posix()) result = interactive_runner.run( InteractiveProcess(argv=request.args, env=request.extra_env, run_in_workspace=True, hermetic_env=False)) return Repl(result.exit_code)
async def run_repl( console: Console, workspace: Workspace, interactive_runner: InteractiveRunner, repl_subsystem: ReplSubsystem, transitive_targets: TransitiveTargets, build_root: BuildRoot, union_membership: UnionMembership, global_options: GlobalOptions, ) -> Repl: repl_shell_name = repl_subsystem.shell or "python" implementations: Dict[str, Type[ReplImplementation]] = { impl.name: impl for impl in union_membership[ReplImplementation] } repl_implementation_cls = implementations.get(repl_shell_name) if repl_implementation_cls is None: available = sorted(implementations.keys()) console.print_stderr( f"{repr(repl_shell_name)} is not a registered REPL. Available REPLs (which may " f"be specified through the option `--repl-shell`): {available}") return Repl(-1) repl_impl = repl_implementation_cls(targets=Targets( tgt for tgt in transitive_targets.closure if repl_implementation_cls.is_valid(tgt))) request = await Get(ReplRequest, ReplImplementation, repl_impl) with temporary_dir(root_dir=global_options.options.pants_workdir, cleanup=False) as tmpdir: tmpdir_relative_path = PurePath(tmpdir).relative_to( build_root.path).as_posix() exe_path = PurePath(tmpdir, request.binary_name).as_posix() workspace.write_digest(request.digest, path_prefix=tmpdir_relative_path) result = interactive_runner.run( InteractiveProcess(argv=(exe_path, ), env=request.env, run_in_workspace=True)) return Repl(result.exit_code)
async def run( run_subsystem: RunSubsystem, global_options: GlobalOptions, console: Console, interactive_runner: InteractiveRunner, workspace: Workspace, build_root: BuildRoot, ) -> Run: targets_to_valid_field_sets = await Get( TargetsToValidFieldSets, TargetsToValidFieldSetsRequest( BinaryFieldSet, goal_description="the `run` goal", error_if_no_valid_targets=True, expect_single_field_set=True, ), ) field_set = targets_to_valid_field_sets.field_sets[0] request = await Get(RunRequest, BinaryFieldSet, field_set) with temporary_dir(root_dir=global_options.options.pants_workdir, cleanup=True) as tmpdir: tmpdir_relative_path = PurePath(tmpdir).relative_to( build_root.path).as_posix() workspace.write_digest(request.digest, path_prefix=tmpdir_relative_path) exe_path = PurePath(tmpdir, request.binary_name).as_posix() process = InteractiveProcess( argv=(exe_path, *request.prefix_args, *run_subsystem.args), env=request.env, run_in_workspace=True, ) try: result = interactive_runner.run(process) exit_code = result.exit_code except Exception as e: console.print_stderr( f"Exception when attempting to run {field_set.address}: {e!r}") exit_code = -1 return Run(exit_code)
async def export( console: Console, targets: Targets, workspace: Workspace, union_membership: UnionMembership, build_root: BuildRoot, dist_dir: DistDir, ) -> Export: request_types = cast("Iterable[type[ExportRequest]]", union_membership.get(ExportRequest)) requests = tuple(request_type(targets) for request_type in request_types) all_results = await MultiGet( Get(ExportResults, ExportRequest, request) for request in requests) flattened_results = [res for results in all_results for res in results] prefixed_digests = await MultiGet( Get(Digest, AddPrefix(result.digest, result.reldir)) for result in flattened_results) output_dir = os.path.join(str(dist_dir.relpath), "export") merged_digest = await Get(Digest, MergeDigests(prefixed_digests)) dist_digest = await Get(Digest, AddPrefix(merged_digest, output_dir)) workspace.write_digest(dist_digest) environment = await Get(Environment, EnvironmentRequest(["PATH"])) for result in flattened_results: digest_root = os.path.join(build_root.path, output_dir, result.reldir) for cmd in result.post_processing_cmds: argv = tuple( arg.format(digest_root=digest_root) for arg in cmd.argv) ip = InteractiveProcess( argv=argv, env={ "PATH": environment.get("PATH", ""), **cmd.extra_env }, run_in_workspace=True, ) await Effect(InteractiveProcessResult, InteractiveProcess, ip) console.print_stdout( f"Wrote {result.description} to {os.path.join(output_dir, result.reldir)}" ) return Export(exit_code=0)
async def debug_python_test(field_set: PythonTestFieldSet) -> TestDebugRequest: if field_set.is_conftest(): return TestDebugRequest(None) setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=True)) return TestDebugRequest(InteractiveProcess.from_process(setup.process))
async def debug_python_test(field_set: PythonTestFieldSet) -> TestDebugRequest: setup = await Get(TestSetup, TestSetupRequest(field_set, is_debug=True)) return TestDebugRequest( InteractiveProcess.from_process(setup.process, forward_signals_to_process=False, restartable=True))
def mock_debug_request(_: TestFieldSet) -> TestDebugRequest: return TestDebugRequest( InteractiveProcess(["/bin/example"], input_digest=EMPTY_DIGEST))
def test_interactive_process_cannot_have_append_only_caches_and_workspace() -> None: with pytest.raises(ValueError): InteractiveProcess( argv=["/bin/echo"], append_only_caches={"foo": "bar"}, run_in_workspace=True )
def _iter_openers(self, files: Iterable[PurePath]) -> Iterator[InteractiveProcess]: for f in files: yield InteractiveProcess(argv=(self.program, str(f)), run_in_workspace=True)
async def twine_upload(request: PublishToPyPiRequest, twine_subsystem: TwineSubsystem) -> PublishProcesses: dists = tuple(artifact.relpath for pkg in request.packages for artifact in pkg.artifacts if artifact.relpath) if twine_subsystem.skip or not dists: return PublishProcesses() # Too verbose to provide feedback as to why some packages were skipped? skip = None if request.field_set.skip_twine.value: skip = f"(by `{request.field_set.skip_twine.alias}` on {request.field_set.address})" elif not request.field_set.repositories.value: # I'd rather have used the opt_out mechanism on the field set, but that gives no hint as to # why the target was not applicable.. skip = f"(no `{request.field_set.repositories.alias}` specified for {request.field_set.address})" if skip: return PublishProcesses([ PublishPackages( names=dists, description=skip, ), ]) twine_pex, packages_digest, config_files = await MultiGet( Get( VenvPex, PexRequest( output_filename="twine.pex", internal_only=True, requirements=twine_subsystem.pex_requirements(), interpreter_constraints=twine_subsystem. interpreter_constraints, main=twine_subsystem.main, ), ), Get(Digest, MergeDigests(pkg.digest for pkg in request.packages)), Get(ConfigFiles, ConfigFilesRequest, twine_subsystem.config_request()), ) input_digest = await Get( Digest, MergeDigests((packages_digest, config_files.snapshot.digest))) pex_proc_requests = [] twine_envs = await MultiGet( Get(Environment, EnvironmentRequest, twine_env_request(repo)) for repo in request.field_set.repositories.value) for repo, env in zip(request.field_set.repositories.value, twine_envs): pex_proc_requests.append( VenvPexProcess( twine_pex, argv=twine_upload_args(twine_subsystem, config_files, repo, dists), input_digest=input_digest, extra_env=twine_env(env, repo), description=repo, )) processes = await MultiGet( Get(Process, VenvPexProcess, request) for request in pex_proc_requests) return PublishProcesses( PublishPackages( names=dists, process=InteractiveProcess.from_process(process), description=process.description, data=PublishOutputData({"repository": process.description}), ) for process in processes)
async def debug_python_test(test_setup: TestTargetSetup) -> TestDebugRequest: process = InteractiveProcess( argv=(test_setup.test_runner_pex.output_filename, *test_setup.args), input_digest=test_setup.input_digest, ) return TestDebugRequest(process)