def _nailgunned_stdio(self, sock): """Redirects stdio to the connected socket speaking the nailgun protocol.""" # Determine output tty capabilities from the environment. stdin_isatty, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env( self._env) # If all stdio is a tty, there's only one logical I/O device (the tty device). This happens to # be addressable as a file in OSX and Linux, so we take advantage of that and directly open the # character device for output redirection - eliminating the need to directly marshall any # interactive stdio back/forth across the socket and permitting full, correct tty control with # no middle-man. if all((stdin_isatty, stdout_isatty, stderr_isatty)): stdin_ttyname, stdout_ttyname, stderr_ttyname = NailgunProtocol.ttynames_from_env( self._env) assert stdin_ttyname == stdout_ttyname == stderr_ttyname, ( 'expected all stdio ttys to be the same, but instead got: {}\n' 'please file a bug at http://github.com/pantsbuild/pants'. format([stdin_ttyname, stdout_ttyname, stderr_ttyname])) with open(stdin_ttyname, 'rb+wb', 0) as tty: tty_fileno = tty.fileno() with stdio_as(stdin_fd=tty_fileno, stdout_fd=tty_fileno, stderr_fd=tty_fileno): def finalizer(): termios.tcdrain(tty_fileno) yield finalizer else: stdio_writers = ((ChunkType.STDOUT, stdout_isatty), (ChunkType.STDERR, stderr_isatty)) types, ttys = zip(*(stdio_writers)) with NailgunStreamStdinReader.open(sock, stdin_isatty) as stdin_fd,\ NailgunStreamWriter.open_multi(sock, types, ttys) as ((stdout_fd, stderr_fd), writer),\ stdio_as(stdout_fd=stdout_fd, stderr_fd=stderr_fd, stdin_fd=stdin_fd): # N.B. This will be passed to and called by the `DaemonExiter` prior to sending an # exit chunk, to avoid any socket shutdown vs write races. stdout, stderr = sys.stdout, sys.stderr def finalizer(): try: stdout.flush() stderr.flush() finally: time.sleep( .001 ) # HACK: Sleep 1ms in the main thread to free the GIL. writer.stop() writer.join() stdout.close() stderr.close() yield finalizer
def test_close_stdio(mock_close): mock_options = unittest.mock.Mock() mock_options_values = unittest.mock.Mock() mock_options.for_global_scope.return_value = mock_options_values mock_options_values.pants_subprocessdir = "non_existent_dir" mock_server = unittest.mock.Mock() def create_services(bootstrap_options, legacy_graph_scheduler): return PantsServices() pantsd = PantsDaemon( native=Native(), work_dir="test_work_dir", log_level=logging.INFO, server=mock_server, core=PantsDaemonCore(create_services), metadata_base_dir="/tmp/pants_test_metadata_dir", bootstrap_options=mock_options, ) with stdio_as(-1, -1, -1): handles = (sys.stdin, sys.stdout, sys.stderr) fds = [h.fileno() for h in handles] pantsd._close_stdio() mock_close.assert_has_calls(unittest.mock.call(x) for x in fds) for handle in handles: assert handle.closed is True
def assert_checker(self, relpath, contents, expected_code=0, expected_message=""): with temporary_dir() as td: with safe_open(os.path.join(td, relpath), "w") as fp: fp.write(contents) args = ["--root-dir={}".format(td)] for plugin_type in checker.plugins(): opts = {"skip": False, "max_length": self._MAX_LENGTH, "ignore": ["E111"]} args.append("--{}-options={}".format(plugin_type.name(), json.dumps(opts))) args.append(relpath) with open(os.path.join(td, "stdout"), "w+") as stdout: with open(os.path.join(td, "stderr"), "w+") as stderr: with stdio_as( stdout_fd=stdout.fileno(), stderr_fd=stderr.fileno(), stdin_fd=-1 ): with self.assertRaises(SystemExit) as error: checker.main(args=args) def read_stdio(fp): fp.flush() fp.seek(0) return fp.read() self.assertEqual( expected_code, error.exception.code, "STDOUT:\n{}\nSTDERR:\n{}".format(read_stdio(stdout), read_stdio(stderr)), ) self.assertEqual(expected_message, read_stdio(stdout).strip()) self.assertEqual("", read_stdio(stderr))
def _pantsd_logging(self) -> Iterator[None]: """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( f"pantsd logging cannot initialize while stdio is open: {fd}" ) except OSError: pass # Redirect stdio to /dev/null for the rest of the run to reserve those file descriptors. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. global_options = self._bootstrap_options.for_global_scope() setup_logging(global_options, stderr_logging=False) log_dir = os.path.join(self._work_dir, self.name) setup_logging_to_file(global_options.level, log_dir=log_dir, log_filename=self.LOG_NAME) self._logger.debug("Logging reinitialized in pantsd context") yield
def _nailgunned_stdio(self, sock): """Redirects stdio to the connected socket speaking the nailgun protocol.""" # Determine output tty capabilities from the environment. stdin_isatty, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env( self._env) # Launch a thread to read stdin data from the socket (the only messages expected from the client # for the remainder of the protocol), and threads to copy from stdout/stderr pipes onto the # socket. with NailgunStreamWriter.open_multi( sock, (ChunkType.STDOUT, ChunkType.STDERR), None, (stdout_isatty, stderr_isatty) ) as ((stdout_fd, stderr_fd), writer),\ NailgunStreamStdinReader.open(sock, stdin_isatty) as stdin_fd,\ stdio_as(stdout_fd=stdout_fd, stderr_fd=stderr_fd, stdin_fd=stdin_fd): # N.B. This will be passed to and called by the `DaemonExiter` prior to sending an # exit chunk, to avoid any socket shutdown vs write races. stdout, stderr = sys.stdout, sys.stderr def finalizer(): try: stdout.flush() stderr.flush() finally: time.sleep( .001 ) # HACK: Sleep 1ms in the main thread to free the GIL. writer.stop() writer.join() stdout.close() stderr.close() yield finalizer
def _pantsd_logging(self): """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( 'pantsd logging cannot initialize while stdio is open: {}'. format(fd)) except OSError: pass # Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors # for further forks. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. result = setup_logging(self._log_level, log_dir=self._log_dir, log_name=self.LOG_NAME) # Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`. # TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements # for `1,2`, and allow them to be used via `stdio_as`. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_handler) sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_handler) self._logger.debug('logging initialized') yield result.log_handler.stream
def _pantsd_logging(self): """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( 'pantsd logging cannot initialize while stdio is open: {}'.format(fd)) except OSError: pass # Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors # for further forks. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. init_rust_logger(self._log_level, self._log_show_rust_3rdparty) result = setup_logging(self._log_level, log_dir=self._log_dir, log_name=self.LOG_NAME, native=self._native) self._native.override_thread_logging_destination_to_just_pantsd() # Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`. # TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements # for `1,2`, and allow them to be used via `stdio_as`. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_handler) sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_handler) self._logger.debug('logging initialized') yield (result.log_handler.stream, result.log_handler.native_filename)
def __call__( self, command: str, args: Tuple[str, ...], env: Dict[str, str], working_directory: bytes, cancellation_latch: PySessionCancellationLatch, stdin_fd: int, stdout_fd: int, stderr_fd: int, ) -> ExitCode: request_timeout = float(env.get("PANTSD_REQUEST_TIMEOUT_LIMIT", -1)) # NB: Order matters: we acquire a lock before mutating either `sys.std*`, `os.environ`, etc. with self._one_run_at_a_time( stderr_fd, cancellation_latch=cancellation_latch, timeout=request_timeout, ), stdio_as(stdin_fd=stdin_fd, stdout_fd=stdout_fd, stderr_fd=stderr_fd), hermetic_environment_as( **env), argv_as((command, ) + args): # NB: Run implements exception handling, so only the most primitive errors will escape # this function, where they will be logged to the pantsd.log by the server. logger.info(f"handling request: `{' '.join(args)}`") try: return self.single_daemonized_run(working_directory.decode(), cancellation_latch) finally: logger.info(f"request completed: `{' '.join(args)}`")
def _pipe_stdio(self, sock, stdin_isatty, stdout_isatty, stderr_isatty): """Handles stdio redirection in the case of pipes and/or mixed pipes and ttys.""" stdio_writers = ((ChunkType.STDOUT, stdout_isatty), (ChunkType.STDERR, stderr_isatty)) types, ttys = zip(*(stdio_writers)) with NailgunStreamStdinReader.open(sock, stdin_isatty) as stdin_fd,\ NailgunStreamWriter.open_multi(sock, types, ttys) as ((stdout_fd, stderr_fd), writer),\ stdio_as(stdout_fd=stdout_fd, stderr_fd=stderr_fd, stdin_fd=stdin_fd): # N.B. This will be passed to and called by the `DaemonExiter` prior to sending an # exit chunk, to avoid any socket shutdown vs write races. stdout, stderr = sys.stdout, sys.stderr def finalizer(): try: stdout.flush() stderr.flush() finally: time.sleep( .001 ) # HACK: Sleep 1ms in the main thread to free the GIL. writer.stop() writer.join() stdout.close() stderr.close() yield finalizer
def assert_checker(self, relpath, contents, expected_code=0, expected_message=''): with temporary_dir() as td: with safe_open(os.path.join(td, relpath), 'w') as fp: fp.write(contents) args=['--root-dir={}'.format(td)] for plugin_type in checker.plugins(): opts = {'skip': False, 'max_length': self._MAX_LENGTH, 'ignore': ['E111']} args.append('--{}-options={}'.format(plugin_type.name(), json.dumps(opts))) args.append(relpath) with open(os.path.join(td, 'stdout'), 'w+') as stdout: with open(os.path.join(td, 'stderr'), 'w+') as stderr: with stdio_as(stdout_fd=stdout.fileno(), stderr_fd=stderr.fileno(), stdin_fd=-1): with self.assertRaises(SystemExit) as error: checker.main(args=args) def read_stdio(fp): fp.flush() fp.seek(0) return fp.read() self.assertEqual(expected_code, error.exception.code, 'STDOUT:\n{}\nSTDERR:\n{}'.format(read_stdio(stdout), read_stdio(stderr))) self.assertEqual(expected_message, read_stdio(stdout).strip()) self.assertEqual('', read_stdio(stderr))
def test_close_stdio(self, mock_close): with stdio_as(-1, -1, -1): handles = (sys.stdin, sys.stdout, sys.stderr) fds = [h.fileno() for h in handles] self.pantsd._close_stdio() mock_close.assert_has_calls(mock.call(x) for x in fds) for handle in handles: self.assertTrue(handle.closed, '{} was not closed'.format(handle))
def test_close_stdio(self, mock_close): with stdio_as(-1, -1, -1): handles = (sys.stdin, sys.stdout, sys.stderr) fds = [h.fileno() for h in handles] self.pantsd._close_stdio() mock_close.assert_has_calls(unittest.mock.call(x) for x in fds) for handle in handles: self.assertTrue(handle.closed, f"{handle} was not closed")
def test_close_fds(self, mock_close): mock_fd = mock.Mock() mock_fd.fileno.side_effect = [0, 1, 2] with stdio_as(mock_fd, mock_fd, mock_fd): self.pantsd._close_fds() self.assertEquals(mock_fd.close.call_count, 3) mock_close.assert_has_calls(mock.call(x) for x in [0, 1, 2])
def test_stdio_as_dev_null(self): # Capture output to tempfiles. with self._stdio_as_tempfiles(): # Read/write from/to `/dev/null`, which will be validated by the harness as not # affecting the tempfiles. with stdio_as(stdout_fd=-1, stderr_fd=-1, stdin_fd=-1): self.assertEquals(b'', sys.stdin.read()) print('garbage', file=sys.stdout) print('garbage', file=sys.stderr)
def test_stdio_as_dev_null(self) -> None: # Capture output to tempfiles. with self._stdio_as_tempfiles(): # Read/write from/to `/dev/null`, which will be validated by the harness as not # affecting the tempfiles. with stdio_as(stdout_fd=-1, stderr_fd=-1, stdin_fd=-1): self.assertEqual("", sys.stdin.read()) print("garbage", file=sys.stdout) print("garbage", file=sys.stderr)
def test_close_fds(self): mock_stdout, mock_stderr, mock_stdin = mock.Mock(), mock.Mock(), mock.Mock() with stdio_as(mock_stdout, mock_stderr, mock_stdin): self.pantsd._close_fds() mock_stdout.close.assert_called_once_with() mock_stderr.close.assert_called_once_with() mock_stdin.close.assert_called_once_with()
def new_io(self, stdin_data): with temporary_dir() as iodir: stdin = os.path.join(iodir, 'stdin') stdout = os.path.join(iodir, 'stdout') stderr = os.path.join(iodir, 'stderr') with open(stdin, 'w') as fp: fp.write(stdin_data) with open(stdin, 'r') as inp, open(stdout, 'w') as out, open(stderr, 'w') as err: with stdio_as(stdin_fd=inp.fileno(), stdout_fd=out.fileno(), stderr_fd=err.fileno()): yield (stdin, stdout, stderr)
def test_stdio_as_stdin_default(self): old_stdout, old_stderr, old_stdin = sys.stdout, sys.stderr, sys.stdin with stdio_as(stdout=1, stderr=2): self.assertEquals(sys.stdout, 1) self.assertEquals(sys.stderr, 2) self.assertEquals(sys.stdin, old_stdin) self.assertEquals(sys.stdout, old_stdout) self.assertEquals(sys.stderr, old_stderr)
def test_close_fds(self): mock_stdout, mock_stderr, mock_stdin = mock.Mock(), mock.Mock( ), mock.Mock() with stdio_as(mock_stdout, mock_stderr, mock_stdin): self.pantsd._close_fds() mock_stdout.close.assert_called_once_with() mock_stderr.close.assert_called_once_with() mock_stdin.close.assert_called_once_with()
def _pantsd_logging(self) -> Iterator[IO[str]]: """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( f"pantsd logging cannot initialize while stdio is open: {fd}" ) except OSError: pass # Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors # for further forks. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. init_rust_logger(self._log_level, self._log_show_rust_3rdparty) # We can't statically prove it, but we won't execute `launch()` (which # calls `run_sync` which calls `_pantsd_logging`) unless PantsDaemon # is launched with full_init=True. If PantsdDaemon is launched with # full_init=True, we can guarantee self._native and self._bootstrap_options # are non-None. native = cast(Native, self._native) bootstrap_options = cast(OptionValueContainer, self._bootstrap_options) level = self._log_level ignores = bootstrap_options.for_global_scope( ).ignore_pants_warnings clear_previous_loggers() setup_logging_to_stderr(level, warnings_filter_regexes=ignores) log_dir = os.path.join(self._work_dir, self.name) log_handler = setup_logging_to_file( level, log_dir=log_dir, log_filename=self.LOG_NAME, warnings_filter_regexes=ignores) native.override_thread_logging_destination_to_just_pantsd() # Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`. # TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements # for `1,2`, and allow them to be used via `stdio_as`. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, log_handler) # type: ignore[assignment] sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, log_handler) # type: ignore[assignment] self._logger.debug("logging initialized") yield log_handler.stream
def _nailgunned_stdio(self, sock): """Redirects stdio to the connected socket speaking the nailgun protocol.""" # Determine output tty capabilities from the environment. stdin_isatty, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env( self._env) # Launch a thread to read stdin data from the socket (the only messages expected from the client # for the remainder of the protocol), and threads to copy from stdout/stderr pipes onto the # socket. with NailgunStreamStdinReader.open(sock, isatty=stdin_isatty) as stdin,\ NailgunStreamWriter.open(sock, ChunkType.STDOUT, None, isatty=stdout_isatty) as stdout,\ NailgunStreamWriter.open(sock, ChunkType.STDERR, None, isatty=stderr_isatty) as stderr: with stdio_as(stdout=stdout, stderr=stderr, stdin=stdin): yield
def _pantsd_logging(self) -> Iterator[IO[str]]: """A context manager that runs with pantsd logging. Asserts that stdio (represented by file handles 0, 1, 2) is closed to ensure that we can safely reuse those fd numbers. """ # Ensure that stdio is closed so that we can safely reuse those file descriptors. for fd in (0, 1, 2): try: os.fdopen(fd) raise AssertionError( f"pantsd logging cannot initialize while stdio is open: {fd}" ) except OSError: pass # Redirect stdio to /dev/null for the rest of the run, to reserve those file descriptors # for further forks. with stdio_as(stdin_fd=-1, stdout_fd=-1, stderr_fd=-1): # Reinitialize logging for the daemon context. use_color = self._bootstrap_options.for_global_scope().colors init_rust_logger(self._log_level, self._log_show_rust_3rdparty, use_color=use_color) level = self._log_level ignores = self._bootstrap_options.for_global_scope( ).ignore_pants_warnings clear_logging_handlers() log_dir = os.path.join(self._work_dir, self.name) log_handler = setup_logging_to_file( level, log_dir=log_dir, log_filename=self.LOG_NAME, warnings_filter_regexes=ignores) self._native.override_thread_logging_destination_to_just_pantsd() # Do a python-level redirect of stdout/stderr, which will not disturb `0,1,2`. # TODO: Consider giving these pipes/actual fds, in order to make them "deep" replacements # for `1,2`, and allow them to be used via `stdio_as`. sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, log_handler) # type: ignore[assignment] sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, log_handler) # type: ignore[assignment] self._logger.debug("logging initialized") yield log_handler.stream
def test_isort_check_only(self): isort_task = self._create_task(target_roots=[self.a_library], passthru_args=['--check-only']) with temporary_dir() as output_dir: with open(os.path.join(output_dir, 'stdout'), 'w+') as stdout: with stdio_as(stdout_fd=stdout.fileno(), stderr_fd=stdout.fileno(), stdin_fd=-1): try: isort_task.execute() except TaskError: stdout.flush() stdout.seek(0) output = stdout.read() self.assertIn("a_1.py Imports are incorrectly sorted.", output) self.assertIn("a_2.py Imports are incorrectly sorted.", output) else: fail("--check-only test for {} is supposed to fail, but passed.".format(self.a_library))
def _nailgunned_stdio(self, sock): """Redirects stdio to the connected socket speaking the nailgun protocol.""" # Determine output tty capabilities from the environment. _, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env(self._env) # TODO(kwlzn): Implement remote input reading and fix the non-fork()-safe sys.stdin reference # in NailgunClient to enable support for interactive goals like `repl` etc. # Construct StreamWriters for stdout, stderr. streams = ( NailgunStreamWriter(sock, ChunkType.STDOUT, isatty=stdout_isatty), NailgunStreamWriter(sock, ChunkType.STDERR, isatty=stderr_isatty) ) # Launch the stdin StreamReader and redirect stdio. with stdio_as(*streams): yield
def _pipe_stdio(cls, maybe_shutdown_socket, stdin_isatty, stdout_isatty, stderr_isatty, handle_stdin): """Handles stdio redirection in the case of pipes and/or mixed pipes and ttys.""" stdio_writers = ((ChunkType.STDOUT, stdout_isatty), (ChunkType.STDERR, stderr_isatty)) types, ttys = zip(*(stdio_writers)) @contextmanager def maybe_handle_stdin(want): if want: with NailgunStreamStdinReader.open(maybe_shutdown_socket, stdin_isatty) as fd: yield fd else: with open("/dev/null", "rb") as fh: yield fh.fileno() # TODO https://github.com/pantsbuild/pants/issues/7653 with maybe_handle_stdin( handle_stdin) as stdin_fd, PipedNailgunStreamWriter.open_multi( maybe_shutdown_socket.socket, types, ttys) as ((stdout_pipe, stderr_pipe), writer), stdio_as(stdout_fd=stdout_pipe.write_fd, stderr_fd=stderr_pipe.write_fd, stdin_fd=stdin_fd): # N.B. This will be passed to and called by the `DaemonExiter` prior to sending an # exit chunk, to avoid any socket shutdown vs write races. stdout, stderr = sys.stdout, sys.stderr def finalizer(): try: stdout.flush() stderr.flush() finally: time.sleep( 0.001 ) # HACK: Sleep 1ms in the main thread to free the GIL. stdout_pipe.stop_writing() stderr_pipe.stop_writing() writer.join(timeout=60) if writer.isAlive(): raise NailgunStreamWriterError( "pantsd timed out while waiting for the stdout/err to finish writing to the socket." ) yield finalizer
def _nailgunned_stdio(self, sock): """Redirects stdio to the connected socket speaking the nailgun protocol.""" # Determine output tty capabilities from the environment. _, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env(self._env) # Construct a StreamReader for stdin. stdin_reader = NailgunStreamReader(sys.stdin, sock) # Construct StreamWriters for stdout, stderr. streams = ( NailgunStreamWriter(sock, ChunkType.STDOUT, isatty=stdout_isatty), NailgunStreamWriter(sock, ChunkType.STDERR, isatty=stderr_isatty), stdin_reader ) # Launch the stdin StreamReader and redirect stdio. with stdin_reader.running(), stdio_as(*streams): yield
def _tty_stdio(self): """Handles stdio redirection in the case of all stdio descriptors being the same tty.""" # If all stdio is a tty, there's only one logical I/O device (the tty device). This happens to # be addressable as a file in OSX and Linux, so we take advantage of that and directly open the # character device for output redirection - eliminating the need to directly marshall any # interactive stdio back/forth across the socket and permitting full, correct tty control with # no middle-man. stdin_ttyname, stdout_ttyname, stderr_ttyname = NailgunProtocol.ttynames_from_env(self._env) assert stdin_ttyname == stdout_ttyname == stderr_ttyname, ( 'expected all stdio ttys to be the same, but instead got: {}\n' 'please file a bug at http://github.com/pantsbuild/pants' .format([stdin_ttyname, stdout_ttyname, stderr_ttyname]) ) with open(stdin_ttyname, 'rb+wb', 0) as tty: tty_fileno = tty.fileno() with stdio_as(stdin_fd=tty_fileno, stdout_fd=tty_fileno, stderr_fd=tty_fileno): def finalizer(): termios.tcdrain(tty_fileno) yield finalizer
def _tty_stdio(cls, env): """Handles stdio redirection in the case of all stdio descriptors being the same tty.""" # If all stdio is a tty, there's only one logical I/O device (the tty device). This happens to # be addressable as a file in OSX and Linux, so we take advantage of that and directly open the # character device for output redirection - eliminating the need to directly marshall any # interactive stdio back/forth across the socket and permitting full, correct tty control with # no middle-man. stdin_ttyname, stdout_ttyname, stderr_ttyname = NailgunProtocol.ttynames_from_env(env) assert stdin_ttyname == stdout_ttyname == stderr_ttyname, ( 'expected all stdio ttys to be the same, but instead got: {}\n' 'please file a bug at http://github.com/pantsbuild/pants' .format([stdin_ttyname, stdout_ttyname, stderr_ttyname]) ) with open(stdin_ttyname, 'rb+', 0) as tty: tty_fileno = tty.fileno() with stdio_as(stdin_fd=tty_fileno, stdout_fd=tty_fileno, stderr_fd=tty_fileno): def finalizer(): termios.tcdrain(tty_fileno) yield finalizer
def _stdio_as_tempfiles(self) -> Iterator[None]: """Harness to replace `sys.std*` with tempfiles. Validates that all files are read/written/flushed correctly, and acts as a contextmanager to allow for recursive tests. """ # Prefix contents written within this instance with a unique string to differentiate # them from other instances. uuid_str = str(uuid.uuid4()) def u(string: str) -> str: return f"{uuid_str}#{string}" stdin_data = u("stdio") stdout_data = u("stdout") stderr_data = u("stderr") with temporary_file(binary_mode=False) as tmp_stdin, temporary_file( binary_mode=False) as tmp_stdout, temporary_file( binary_mode=False) as tmp_stderr: print(stdin_data, file=tmp_stdin) tmp_stdin.seek(0) # Read prepared content from stdin, and write content to stdout/stderr. with stdio_as( stdout_fd=tmp_stdout.fileno(), stderr_fd=tmp_stderr.fileno(), stdin_fd=tmp_stdin.fileno(), ): self.assertEqual(sys.stdin.fileno(), 0) self.assertEqual(sys.stdout.fileno(), 1) self.assertEqual(sys.stderr.fileno(), 2) self.assertEqual(stdin_data, sys.stdin.read().strip()) print(stdout_data, file=sys.stdout) yield print(stderr_data, file=sys.stderr) tmp_stdout.seek(0) tmp_stderr.seek(0) self.assertEqual(stdout_data, tmp_stdout.read().strip()) self.assertEqual(stderr_data, tmp_stderr.read().strip())
def _nailgunned_stdio(self, sock): """Redirects stdio to the connected socket speaking the nailgun protocol.""" # Determine output tty capabilities from the environment. _, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env( self._env) # Construct a StreamReader for stdin. stdin_reader = NailgunStreamReader(sys.stdin, sock) # Construct StreamWriters for stdout, stderr. streams = (NailgunStreamWriter(sock, ChunkType.STDOUT, isatty=stdout_isatty), NailgunStreamWriter(sock, ChunkType.STDERR, isatty=stderr_isatty), stdin_reader) # Launch the stdin StreamReader and redirect stdio. with stdin_reader.running(), stdio_as(*streams): yield
def _pipe_stdio(cls, sock, stdin_isatty, stdout_isatty, stderr_isatty, handle_stdin): """Handles stdio redirection in the case of pipes and/or mixed pipes and ttys.""" stdio_writers = ((ChunkType.STDOUT, stdout_isatty), (ChunkType.STDERR, stderr_isatty)) types, ttys = zip(*(stdio_writers)) @contextmanager def maybe_handle_stdin(want): if want: # TODO: Launching this thread pre-fork to handle @rule input currently results # in an unhandled SIGILL in `src/python/pants/engine/scheduler.py, line 313 in pre_fork`. # More work to be done here in https://github.com/pantsbuild/pants/issues/6005 with NailgunStreamStdinReader.open(sock, stdin_isatty) as fd: yield fd else: with open('/dev/null', 'rb') as fh: yield fh.fileno() with maybe_handle_stdin(handle_stdin) as stdin_fd,\ NailgunStreamWriter.open_multi(sock, types, ttys) as ((stdout_fd, stderr_fd), writer),\ stdio_as(stdout_fd=stdout_fd, stderr_fd=stderr_fd, stdin_fd=stdin_fd): # N.B. This will be passed to and called by the `DaemonExiter` prior to sending an # exit chunk, to avoid any socket shutdown vs write races. stdout, stderr = sys.stdout, sys.stderr def finalizer(): try: stdout.flush() stderr.flush() finally: time.sleep( .001 ) # HACK: Sleep 1ms in the main thread to free the GIL. writer.stop() writer.join() stdout.close() stderr.close() yield finalizer
def _stdio_as_tempfiles(self): """Harness to replace `sys.std*` with tempfiles. Validates that all files are read/written/flushed correctly, and acts as a contextmanager to allow for recursive tests. """ # Prefix contents written within this instance with a unique string to differentiate # them from other instances. uuid_str = str(uuid.uuid4()) def u(string): return '{}#{}'.format(uuid_str, string) stdin_data = u('stdio') stdout_data = u('stdout') stderr_data = u('stderr') with temporary_file() as tmp_stdin,\ temporary_file() as tmp_stdout,\ temporary_file() as tmp_stderr: print(stdin_data, file=tmp_stdin) tmp_stdin.seek(0) # Read prepared content from stdin, and write content to stdout/stderr. with stdio_as(stdout_fd=tmp_stdout.fileno(), stderr_fd=tmp_stderr.fileno(), stdin_fd=tmp_stdin.fileno()): self.assertEquals(sys.stdin.fileno(), 0) self.assertEquals(sys.stdout.fileno(), 1) self.assertEquals(sys.stderr.fileno(), 2) self.assertEquals(stdin_data, sys.stdin.read().strip()) print(stdout_data, file=sys.stdout) yield print(stderr_data, file=sys.stderr) tmp_stdout.seek(0) tmp_stderr.seek(0) self.assertEquals(stdout_data, tmp_stdout.read().strip()) self.assertEquals(stderr_data, tmp_stderr.read().strip())
def _pipe_stdio(cls, sock, stdin_isatty, stdout_isatty, stderr_isatty, handle_stdin): """Handles stdio redirection in the case of pipes and/or mixed pipes and ttys.""" stdio_writers = ( (ChunkType.STDOUT, stdout_isatty), (ChunkType.STDERR, stderr_isatty) ) types, ttys = zip(*(stdio_writers)) @contextmanager def maybe_handle_stdin(want): if want: # TODO: Launching this thread pre-fork to handle @rule input currently results # in an unhandled SIGILL in `src/python/pants/engine/scheduler.py, line 313 in pre_fork`. # More work to be done here in https://github.com/pantsbuild/pants/issues/6005 with NailgunStreamStdinReader.open(sock, stdin_isatty) as fd: yield fd else: with open('/dev/null', 'rb') as fh: yield fh.fileno() with maybe_handle_stdin(handle_stdin) as stdin_fd,\ NailgunStreamWriter.open_multi(sock, types, ttys) as ((stdout_fd, stderr_fd), writer),\ stdio_as(stdout_fd=stdout_fd, stderr_fd=stderr_fd, stdin_fd=stdin_fd): # N.B. This will be passed to and called by the `DaemonExiter` prior to sending an # exit chunk, to avoid any socket shutdown vs write races. stdout, stderr = sys.stdout, sys.stderr def finalizer(): try: stdout.flush() stderr.flush() finally: time.sleep(.001) # HACK: Sleep 1ms in the main thread to free the GIL. writer.stop() writer.join() stdout.close() stderr.close() yield finalizer
def _pipe_stdio(self, sock, stdin_isatty, stdout_isatty, stderr_isatty): """Handles stdio redirection in the case of pipes and/or mixed pipes and ttys.""" stdio_writers = ( (ChunkType.STDOUT, stdout_isatty), (ChunkType.STDERR, stderr_isatty) ) types, ttys = zip(*(stdio_writers)) with NailgunStreamStdinReader.open(sock, stdin_isatty) as stdin_fd,\ NailgunStreamWriter.open_multi(sock, types, ttys) as ((stdout_fd, stderr_fd), writer),\ stdio_as(stdout_fd=stdout_fd, stderr_fd=stderr_fd, stdin_fd=stdin_fd): # N.B. This will be passed to and called by the `DaemonExiter` prior to sending an # exit chunk, to avoid any socket shutdown vs write races. stdout, stderr = sys.stdout, sys.stderr def finalizer(): try: stdout.flush() stderr.flush() finally: time.sleep(.001) # HACK: Sleep 1ms in the main thread to free the GIL. writer.stop() writer.join() stdout.close() stderr.close() yield finalizer