def test_send_exit_default(self): NailgunProtocol.send_exit(self.server_sock) chunk_type, payload = NailgunProtocol.read_chunk(self.client_sock) self.assertEqual( (chunk_type, payload), (ChunkType.EXIT, self.EMPTY_PAYLOAD) )
def test_send_start_reading_input(self): NailgunProtocol.send_start_reading_input(self.server_sock) chunk_type, payload = NailgunProtocol.read_chunk(self.client_sock) self.assertEqual( (chunk_type, payload), (ChunkType.START_READING_INPUT, self.EMPTY_PAYLOAD) )
def run(self): # Ensure anything referencing sys.argv inherits the Pailgun'd args. sys.argv = self._args # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so # they can send signals (e.g. SIGINT) to all processes in the runners process group. with self._maybe_shutdown_socket.lock: NailgunProtocol.send_pid(self._maybe_shutdown_socket.socket, os.getpid()) NailgunProtocol.send_pgrp(self._maybe_shutdown_socket.socket, os.getpgrp() * -1) # Invoke a Pants run with stdio redirected and a proxied environment. with self.nailgunned_stdio( self._maybe_shutdown_socket, self._env) as finalizer, DaemonExiter.override_global_exiter( self._maybe_shutdown_socket, finalizer), hermetic_environment_as( **self._env), encapsulated_global_logger(): try: # Clean global state. clean_global_runtime_state(reset_subsystem=True) options, _, options_bootstrapper = LocalPantsRunner.parse_options( self._args, self._env) graph_helper, specs, exit_code = self._scheduler_service.prepare_v1_graph_run_v2( options, options_bootstrapper, ) self.exit_code = exit_code # Otherwise, conduct a normal run. with ExceptionSink.exiter_as_until_exception( lambda _: PantsRunFailCheckerExiter()): runner = LocalPantsRunner.create( self._args, self._env, specs, graph_helper, options_bootstrapper, ) runner.set_start_time( self._maybe_get_client_start_time_from_env(self._env)) runner.run() except KeyboardInterrupt: self._exiter.exit_and_fail("Interrupted by user.\n") except _PantsRunFinishedWithFailureException as e: ExceptionSink.log_exception( "Pants run failed with exception: {}; exiting".format(e)) self._exiter.exit(e.exit_code) except Exception as e: # TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That # excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't # happening here, so something is probably overriding the excepthook. By catching Exception # and calling this method, we emulate the normal, expected sys.excepthook override. ExceptionSink._log_unhandled_exception_and_exit(exc=e) else: self._exiter.exit(self.exit_code if self. exit_code else PANTS_SUCCEEDED_EXIT_CODE)
def test_send_exit(self): NailgunProtocol.send_exit(self.server_sock, self.TEST_OUTPUT) chunk_type, payload = NailgunProtocol.read_chunk(self.client_sock) self.assertEqual( (chunk_type, payload), (ChunkType.EXIT, self.TEST_OUTPUT) )
def run(self): while not self.is_stopped: readable, _, errored = select.select([self._in_file], [], [self._in_file], self._select_timeout) if self._in_file in errored: self.stop() return if not self.is_stopped and self._in_file in readable: data = os.read(self._in_file.fileno(), self._buf_size) if not self.is_stopped: if data: NailgunProtocol.write_chunk(self._socket, self._chunk_type, data) else: try: if self._chunk_eof_type is not None: NailgunProtocol.write_chunk( self._socket, self._chunk_eof_type) self._socket.shutdown( socket.SHUT_WR) # Shutdown socket sends. except socket.error: # Can happen if response is quick. pass finally: self.stop()
def test_send_exit_with_code(self): return_code = 1 NailgunProtocol.send_exit_with_code(self.server_sock, return_code) chunk_type, payload = NailgunProtocol.read_chunk(self.client_sock, return_bytes=True) self.assertEqual( (chunk_type, payload), (ChunkType.EXIT, NailgunProtocol.encode_int(return_code)) )
def run(self): while not self.is_stopped: readable, _, errored = select.select([self._stdin], [], [self._stdin], self._select_timeout) if self._stdin in errored: self.stop() return if not self.is_stopped and self._stdin in readable: data = os.read(self._stdin.fileno(), self._buf_size) if not self.is_stopped: if data: NailgunProtocol.write_chunk(self._socket, ChunkType.STDIN, data) else: NailgunProtocol.write_chunk(self._socket, ChunkType.STDIN_EOF) try: self._socket.shutdown( socket.SHUT_WR) # Shutdown socket sends. except socket.error: # Can happen if response is quick. pass finally: self.stop()
def open(cls, sock, isatty=False): with _pipe(isatty) as (read_fd, write_fd): reader = NailgunStreamStdinReader(sock, os.fdopen(write_fd, 'wb')) with reader.running(): # Instruct the thin client to begin reading and sending stdin. NailgunProtocol.send_start_reading_input(sock) yield read_fd
def post_fork_child(self): """Post-fork child process callback executed via ProcessManager.daemonize().""" # Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception # hook with socket-specific behavior. self._exiter.set_except_hook() # Set context in the process title. set_process_title('pantsd-runner [{}]'.format(' '.join(self._args))) # Broadcast our pid to the remote client so they can send us signals (i.e. SIGINT). NailgunProtocol.write_chunk(self._socket, ChunkType.PID, bytes(os.getpid())) # Setup a SIGINT signal handler. self._setup_sigint_handler() # Invoke a Pants run with stdio redirected. with self._nailgunned_stdio(self._socket): try: # Clean global state. clean_global_runtime_state(reset_subsystem=True) # Re-raise any deferred exceptions, if present. self._raise_deferred_exc() # Otherwise, conduct a normal run. LocalPantsRunner(self._exiter, self._args, self._env, self._graph_helper).run() except KeyboardInterrupt: self._exiter.exit(1, msg='Interrupted by user.\n') except Exception: self._exiter.handle_unhandled_exception(add_newline=True) else: self._exiter.exit(0)
def test_send_and_parse_request_bad_chunktype(self): INVALID_CHUNK_TYPE = b";" NailgunProtocol.write_chunk(self.client_sock, INVALID_CHUNK_TYPE, "1729") with self.assertRaises(NailgunProtocol.ProtocolError): NailgunProtocol.parse_request(self.server_sock)
def test_send_unicode_chunk(self): NailgunProtocol.send_stdout(self.server_sock, self.TEST_UNICODE_PAYLOAD) chunk_type, payload = NailgunProtocol.read_chunk(self.client_sock, return_bytes=True) self.assertEqual((chunk_type, payload), (ChunkType.STDOUT, self.TEST_UNICODE_PAYLOAD))
def test_send_pid(self): test_pid = 1 NailgunProtocol.send_pid(self.server_sock, test_pid) chunk_type, payload = NailgunProtocol.read_chunk(self.client_sock, return_bytes=True) self.assertEqual( (chunk_type, payload), (ChunkType.PID, NailgunProtocol.encode_int(test_pid)) )
def test_send_unicode_chunk(self): NailgunProtocol.send_stdout(self.server_sock, self.TEST_UNICODE_PAYLOAD) chunk_type, payload = NailgunProtocol.read_chunk(self.client_sock, return_bytes=True) self.assertEqual( (chunk_type, payload), (ChunkType.STDOUT, self.TEST_UNICODE_PAYLOAD) )
def run(self): while self._in_fds and not self.is_stopped: readable, _, errored = select.select(self._in_fds, [], self._in_fds, self._select_timeout) if readable: for fileno in readable: data = os.read(fileno, self._buf_size) if not data: # We've reached EOF. try: if self._chunk_eof_type is not None: NailgunProtocol.write_chunk( self._socket, self._chunk_eof_type) finally: try: os.close(fileno) finally: self._in_fds.remove(fileno) else: NailgunProtocol.write_chunk( self._socket, self._fileno_chunk_type_map[fileno], data) if errored: for fileno in errored: self._in_fds.remove(fileno)
def run(self): while self._in_fds and not self.is_stopped: readable, _, errored = select.select(self._in_fds, [], self._in_fds, self._select_timeout) if readable: for fileno in readable: data = os.read(fileno, self._buf_size) if not data: # We've reached EOF. try: if self._chunk_eof_type is not None: NailgunProtocol.write_chunk(self._socket, self._chunk_eof_type) finally: try: os.close(fileno) finally: self._in_fds.remove(fileno) else: NailgunProtocol.write_chunk( self._socket, self._fileno_chunk_type_map[fileno], data ) if errored: for fileno in errored: self._in_fds.remove(fileno)
def test_read_and_write_chunk(self): # Write a command chunk to the server socket. NailgunProtocol.write_chunk(self.server_sock, ChunkType.COMMAND, self.TEST_COMMAND) # Read the chunk from the client socket. chunk_type, payload = NailgunProtocol.read_chunk(self.client_sock) self.assertEqual((chunk_type, payload), (ChunkType.COMMAND, self.TEST_COMMAND))
def test_read_chunk_truncated_before_payload(self): """Construct a chunk and send exactly the header (first 5 bytes) and truncate the remainder.""" truncated_chunk = NailgunProtocol.construct_chunk(ChunkType.STDOUT, self.TEST_OUTPUT)[:5] self.server_sock.sendall(truncated_chunk) self.server_sock.close() with self.assertRaises(NailgunProtocol.TruncatedPayloadError): NailgunProtocol.read_chunk(self.client_sock)
def test_read_chunk_truncated_during_payload(self): """Construct a chunk and truncate the last 3 bytes of the payload ([:-3]).""" truncated_chunk = NailgunProtocol.construct_chunk(ChunkType.STDOUT, self.TEST_OUTPUT)[:-3] self.server_sock.sendall(truncated_chunk) self.server_sock.close() with self.assertRaises(NailgunProtocol.TruncatedPayloadError): NailgunProtocol.read_chunk(self.client_sock)
def test_read_chunk_truncated_during_header(self): """Construct a chunk and truncate to the first 3 bytes ([:3]), an incomplete header.""" truncated_chunk = NailgunProtocol.construct_chunk(ChunkType.STDOUT, self.TEST_OUTPUT)[:3] self.server_sock.sendall(truncated_chunk) self.server_sock.close() with self.assertRaises(NailgunProtocol.TruncatedHeaderError): NailgunProtocol.read_chunk(self.client_sock)
def post_fork_child(self): """Post-fork child process callback executed via ProcessManager.daemonize().""" # Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception # hook with socket-specific behavior. Note that this intentionally points the faulthandler # trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to # the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner # process that the stacktrace output lands deterministically in a known place vs to a stray # terminal window. # TODO: test the above! ExceptionSink.reset_exiter(self._exiter) ExceptionSink.reset_interactive_output_stream( sys.stderr.buffer if PY3 else sys.stderr) # Ensure anything referencing sys.argv inherits the Pailgun'd args. sys.argv = self._args # Set context in the process title. set_process_title('pantsd-runner [{}]'.format(' '.join(self._args))) # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so # they can send signals (e.g. SIGINT) to all processes in the runners process group. NailgunProtocol.send_pid(self._socket, os.getpid()) NailgunProtocol.send_pgrp(self._socket, os.getpgrp() * -1) # Stop the services that were paused pre-fork. for service in self._services.services: service.terminate() # Invoke a Pants run with stdio redirected and a proxied environment. with self.nailgunned_stdio(self._socket, self._env) as finalizer,\ hermetic_environment_as(**self._env): try: # Setup the Exiter's finalizer. self._exiter.set_finalizer(finalizer) # Clean global state. clean_global_runtime_state(reset_subsystem=True) # Re-raise any deferred exceptions, if present. self._raise_deferred_exc() # Otherwise, conduct a normal run. runner = LocalPantsRunner.create(self._exiter, self._args, self._env, self._target_roots, self._graph_helper, self._options_bootstrapper) runner.set_start_time( self._maybe_get_client_start_time_from_env(self._env)) runner.run() except KeyboardInterrupt: self._exiter.exit_and_fail('Interrupted by user.\n') except Exception: ExceptionSink._log_unhandled_exception_and_exit() else: self._exiter.exit(0)
def _handle_closed_input_stream(self, fileno): # We've reached EOF. try: if self._chunk_eof_type is not None: NailgunProtocol.write_chunk(self._socket, self._chunk_eof_type) finally: try: os.close(fileno) finally: self.stop_reading_from_fd(fileno)
def run(self): # Ensure anything referencing sys.argv inherits the Pailgun'd args. sys.argv = self._args # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so # they can send signals (e.g. SIGINT) to all processes in the runners process group. with self._maybe_shutdown_socket.lock: NailgunProtocol.send_pid(self._maybe_shutdown_socket.socket, os.getpid()) NailgunProtocol.send_pgrp(self._maybe_shutdown_socket.socket, os.getpgrp() * -1) # Invoke a Pants run with stdio redirected and a proxied environment. with self.nailgunned_stdio(self._maybe_shutdown_socket, self._env) as finalizer, \ hermetic_environment_as(**self._env), \ encapsulated_global_logger(): try: # Raise any exceptions we may have found when precomputing products. # NB: We raise it here as opposed to earlier because we have setup logging and stdio. if self._exception is not None: raise self._exception # Clean global state. clean_global_runtime_state(reset_subsystem=True) # Setup the Exiter's finalizer. self._exiter.set_finalizer(finalizer) # Otherwise, conduct a normal run. runner = LocalPantsRunner.create( PantsRunFailCheckerExiter(), self._args, self._env, self._target_roots, self._graph_helper, self._options_bootstrapper, ) runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env)) runner.run() except KeyboardInterrupt: self._exiter.exit_and_fail('Interrupted by user.\n') except _PantsRunFinishedWithFailureException as e: ExceptionSink.log_exception( 'Pants run failed with exception: {}; exiting'.format(e)) self._exiter.exit(e.exit_code) except _PantsProductPrecomputeFailed as e: ExceptionSink.log_exception(repr(e)) self._exiter.exit(e.exit_code) except Exception as e: # TODO: We override sys.excepthook above when we call ExceptionSink.set_exiter(). That # excepthook catches `SignalHandledNonLocalExit`s from signal handlers, which isn't # happening here, so something is probably overriding the excepthook. By catching Exception # and calling this method, we emulate the normal, expected sys.excepthook override. ExceptionSink._log_unhandled_exception_and_exit(exc=e) else: self._exiter.exit(PANTS_SUCCEEDED_EXIT_CODE)
def test_process_session_bad_chunk(self, mock_psutil_process): mock_psutil_process.cmdline.return_value = ["mock", "process"] NailgunProtocol.write_chunk(self.server_sock, ChunkType.START_READING_INPUT) NailgunProtocol.write_chunk(self.server_sock, self.BAD_CHUNK_TYPE, "") with self.assertRaises(NailgunClientSession.ProtocolError): self.nailgun_client_session._process_session() self.mock_stdin_reader.start.assert_called_once_with() self.mock_stdin_reader.stop.assert_called_once_with()
def write(self, payload): try: NailgunProtocol.write_chunk(self._socket, self._chunk_type, payload) except IOError as e: # If the remote client disconnects and we try to perform a write (e.g. socket.send/sendall), # an 'error: [Errno 32] Broken pipe' exception can be thrown. Setting mask_broken_pipe=True # safeguards against this case (which is unexpected for most writers of sys.stdout etc) so # that we don't awkwardly interrupt the runtime by throwing this exception on writes to # stdout/stderr. if e.errno == errno.EPIPE and not self._mask_broken_pipe: raise
def post_fork_child(self): """Post-fork child process callback executed via ProcessManager.daemonize().""" # Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception # hook with socket-specific behavior. Note that this intentionally points the faulthandler # trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to # the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner # process that the stacktrace output lands deterministically in a known place vs to a stray # terminal window. self._exiter.set_except_hook(sys.stderr) # Ensure anything referencing sys.argv inherits the Pailgun'd args. sys.argv = self._args # Set context in the process title. set_process_title('pantsd-runner [{}]'.format(' '.join(self._args))) # Setup a SIGINT signal handler. self._setup_sigint_handler() # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so # they can send signals (e.g. SIGINT) to all processes in the runners process group. pid = str(os.getpgrp() * -1).encode('ascii') NailgunProtocol.send_pid(self._socket, pid) # Invoke a Pants run with stdio redirected and a proxied environment. with self.nailgunned_stdio(self._socket, self._env) as finalizer,\ hermetic_environment_as(**self._env): try: # Setup the Exiter's finalizer. self._exiter.set_finalizer(finalizer) # Clean global state. clean_global_runtime_state(reset_subsystem=True) # Re-raise any deferred exceptions, if present. self._raise_deferred_exc() # Otherwise, conduct a normal run. runner = LocalPantsRunner.create( self._exiter, self._args, self._env, self._target_roots, self._graph_helper, self._options_bootstrapper ) runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env)) runner.run() except KeyboardInterrupt: self._exiter.exit(1, msg='Interrupted by user.\n') except Exception: self._exiter.handle_unhandled_exception(add_newline=True) else: self._exiter.exit(0)
def test_read_and_write_chunk(self): # Write a command chunk to the server socket. NailgunProtocol.write_chunk(self.server_sock, ChunkType.COMMAND, self.TEST_COMMAND) # Read the chunk from the client socket. chunk_type, payload = NailgunProtocol.read_chunk(self.client_sock) self.assertEqual( (chunk_type, payload), (ChunkType.COMMAND, self.TEST_COMMAND) )
def ensure_request_is_exclusive(self, environment, request): """ Ensure that this is the only pants running. We currently don't allow parallel pants runs, so this function blocks a request thread until there are no more requests being handled. """ # TODO add `did_poll` to pantsd metrics timeout = float(environment['PANTSD_REQUEST_TIMEOUT_LIMIT']) @contextmanager def yield_and_release(time_waited): try: self.logger.debug("request lock acquired {}.".format( "on the first try" if time_waited == 0 else "in {} seconds".format(time_waited))) yield finally: self.free_to_handle_request_lock.release() self.logger.debug("released request lock.") time_polled = 0.0 user_notification_interval = 1.0 # Stop polling to notify the user every second. self.logger.debug( "request {} is trying to aquire the request lock.".format(request)) # NB: Optimistically try to acquire the lock without blocking, in case we are the only request being handled. # This could be merged into the `while` loop below, but separating this special case for logging helps. if self.free_to_handle_request_lock.acquire(timeout=0): with yield_and_release(time_polled): yield else: # We have to wait for another request to finish being handled. NailgunProtocol.send_stderr( request, "Another pants invocation is running. Will wait {} for it to finish before giving up.\n" .format("forever" if self._should_poll_forever(timeout) else "up to {} seconds".format(timeout))) while not self.free_to_handle_request_lock.acquire( timeout=user_notification_interval): time_polled += user_notification_interval if self._should_keep_polling(timeout, time_polled): NailgunProtocol.send_stderr( request, "Waiting for invocation to finish (waited for {}s so far)...\n" .format(time_polled)) else: # We have timed out. raise ExclusiveRequestTimeout( "Timed out while waiting for another pants invocation to finish." ) with yield_and_release(time_polled): yield
def _nailgunned_stdio(self, sock): """Redirects stdio to the connected socket speaking the nailgun protocol.""" # Determine output tty capabilities from the environment. stdin_isatty, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env( self._env) # If all stdio is a tty, there's only one logical I/O device (the tty device). This happens to # be addressable as a file in OSX and Linux, so we take advantage of that and directly open the # character device for output redirection - eliminating the need to directly marshall any # interactive stdio back/forth across the socket and permitting full, correct tty control with # no middle-man. if all((stdin_isatty, stdout_isatty, stderr_isatty)): stdin_ttyname, stdout_ttyname, stderr_ttyname = NailgunProtocol.ttynames_from_env( self._env) assert stdin_ttyname == stdout_ttyname == stderr_ttyname, ( 'expected all stdio ttys to be the same, but instead got: {}\n' 'please file a bug at http://github.com/pantsbuild/pants'. format([stdin_ttyname, stdout_ttyname, stderr_ttyname])) with open(stdin_ttyname, 'rb+wb', 0) as tty: tty_fileno = tty.fileno() with stdio_as(stdin_fd=tty_fileno, stdout_fd=tty_fileno, stderr_fd=tty_fileno): def finalizer(): termios.tcdrain(tty_fileno) yield finalizer else: stdio_writers = ((ChunkType.STDOUT, stdout_isatty), (ChunkType.STDERR, stderr_isatty)) types, ttys = zip(*(stdio_writers)) with NailgunStreamStdinReader.open(sock, stdin_isatty) as stdin_fd,\ NailgunStreamWriter.open_multi(sock, types, ttys) as ((stdout_fd, stderr_fd), writer),\ stdio_as(stdout_fd=stdout_fd, stderr_fd=stderr_fd, stdin_fd=stdin_fd): # N.B. This will be passed to and called by the `DaemonExiter` prior to sending an # exit chunk, to avoid any socket shutdown vs write races. stdout, stderr = sys.stdout, sys.stderr def finalizer(): try: stdout.flush() stderr.flush() finally: time.sleep( .001 ) # HACK: Sleep 1ms in the main thread to free the GIL. writer.stop() writer.join() stdout.close() stderr.close() yield finalizer
def test_send_and_parse_request(self): # Send a test request over the client socket. NailgunProtocol.send_request(self.client_sock, self.TEST_WORKING_DIR, self.TEST_COMMAND, *self.TEST_ARGUMENTS, **self.TEST_ENVIRON) # Receive the request from the server-side context. working_dir, command, arguments, environment = NailgunProtocol.parse_request( self.server_sock) self.assertEqual(working_dir, self.TEST_WORKING_DIR) self.assertEqual(command, self.TEST_COMMAND) self.assertEqual(arguments, self.TEST_ARGUMENTS) self.assertEqual(environment, self.TEST_ENVIRON)
def test_iter_chunks(self): expected_chunks = [ (ChunkType.COMMAND, self.TEST_COMMAND), (ChunkType.STDOUT, self.TEST_OUTPUT), (ChunkType.STDERR, self.TEST_OUTPUT), (ChunkType.EXIT, self.EMPTY_PAYLOAD) # N.B. without an EXIT chunk here (or socket failure), this test will deadlock in iter_chunks. ] for chunk_type, payload in expected_chunks: NailgunProtocol.write_chunk(self.server_sock, chunk_type, payload) for i, chunk in enumerate(NailgunProtocol.iter_chunks(self.client_sock)): self.assertEqual(chunk, expected_chunks[i])
def run(self): for chunk_type, payload in NailgunProtocol.iter_chunks( self._socket, return_bytes=True): if chunk_type == ChunkType.STDIN: self._write_handle.write(payload) self._write_handle.flush() elif chunk_type == ChunkType.STDIN_EOF: self._write_handle.close() break else: self._try_close() raise NailgunProtocol.ProtocolError( 'received unexpected chunk {} -> {}: closing.'.format( chunk_type, payload))
def do_run(self, readable_fds, errored_fds): """Represents one iteration of the infinite reading cycle.""" if readable_fds: for fileno in readable_fds: data = os.read(fileno, self._buf_size) if not data: self._handle_closed_input_stream(fileno) else: NailgunProtocol.write_chunk( self._socket, self._fileno_chunk_type_map[fileno], data) if errored_fds: for fileno in errored_fds: self._stop_reading_from_fd(fileno)
def exit(self, result=0, msg=None): """Exit the runtime.""" try: # Write a final message to stderr if present. if msg: NailgunProtocol.write_chunk(self._socket, ChunkType.STDERR, msg) # Send an Exit chunk with the result. NailgunProtocol.write_chunk(self._socket, ChunkType.EXIT, str(result).encode('ascii')) # Shutdown the connected socket. self._shutdown_socket() finally: # N.B. Assuming a fork()'d child, os._exit(0) here to avoid the routine sys.exit() behavior. os._exit(0)
def _connect_and_execute(self, port): # Merge the nailgun TTY capability environment variables with the passed environment dict. ng_env = NailgunProtocol.isatty_to_env(self._stdin, self._stdout, self._stderr) modified_env = combined_dict(self._env, ng_env) modified_env['PANTSD_RUNTRACKER_CLIENT_START_TIME'] = str( self._start_time) assert isinstance(port, int), 'port {} is not an integer!'.format(port) # Instantiate a NailgunClient. client = NailgunClient(port=port, ins=self._stdin, out=self._stdout, err=self._stderr, exit_on_broken_pipe=True, expects_pid=True) with self._trapped_signals(client), STTYSettings.preserved(): # Execute the command on the pailgun. result = client.execute(self.PANTS_COMMAND, *self._args, **modified_env) # Exit. self._exiter.exit(result)
def exit(self, result=0, msg=None): """Exit the runtime.""" try: # Write a final message to stderr if present. if msg: NailgunProtocol.write_chunk(self._socket, ChunkType.STDERR, msg) # Send an Exit chunk with the result. NailgunProtocol.write_chunk(self._socket, ChunkType.EXIT, str(result).encode('ascii')) # Shutdown the connected socket. self._shutdown_socket() finally: # N.B. Assuming a fork()'d child, cause os._exit to be called here to avoid the routine # sys.exit behavior (via `pants.util.contextutil.hard_exit_handler()`). raise HardSystemExit()
def test_isatty_from_env_mixed(self): self.assertEquals( NailgunProtocol.isatty_from_env({ 'NAILGUN_TTY_0': '0', 'NAILGUN_TTY_1': '1' }), (False, True, False) )
def test_send_and_parse_request(self): # Send a test request over the client socket. NailgunProtocol.send_request( self.client_sock, self.TEST_WORKING_DIR, self.TEST_COMMAND, *self.TEST_ARGUMENTS, **self.TEST_ENVIRON ) # Receive the request from the server-side context. working_dir, command, arguments, environment = NailgunProtocol.parse_request(self.server_sock) self.assertEqual(working_dir, self.TEST_WORKING_DIR) self.assertEqual(command, self.TEST_COMMAND) self.assertEqual(arguments, self.TEST_ARGUMENTS) self.assertEqual(environment, self.TEST_ENVIRON)
def open(cls, maybe_shutdown_socket, isatty=False): # We use a plain pipe here (as opposed to a self-closing pipe), because # NailgunStreamStdinReader will close the file descriptor it's writing to when it's done. # Therefore, when _self_closing_pipe tries to clean up, it will try to close an already closed fd. # The alternative is passing an os.dup(write_fd) to NSSR, but then we have the problem where # _self_closing_pipe doens't close the write_fd until the pants run is done, and that generates # issues around piping stdin to interactive processes such as REPLs. with _pipe(isatty) as (read_fd, write_fd): reader = NailgunStreamStdinReader(maybe_shutdown_socket, os.fdopen(write_fd, 'wb')) with reader.running(): # Instruct the thin client to begin reading and sending stdin. with maybe_shutdown_socket.lock: NailgunProtocol.send_start_reading_input(maybe_shutdown_socket.socket) try: yield read_fd finally: os.close(read_fd)
def _nailgunned_stdio(self, sock): """Redirects stdio to the connected socket speaking the nailgun protocol.""" # Determine output tty capabilities from the environment. stdin_isatty, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env(self._env) if all((stdin_isatty, stdout_isatty, stderr_isatty)): with self._tty_stdio() as finalizer: yield finalizer else: with self._pipe_stdio(sock, stdin_isatty, stdout_isatty, stderr_isatty) as finalizer: yield finalizer
def handle(self): """Request handler for a single Pailgun request.""" # Parse the Nailgun request portion. _, _, arguments, environment = NailgunProtocol.parse_request(self.request) # N.B. the first and second nailgun request arguments (working_dir and command) are currently # ignored in favor of a get_buildroot() call within LocalPantsRunner.run() and an assumption # that anyone connecting to this nailgun server always intends to run pants itself. # Prepend the command to our arguments so it aligns with the expected sys.argv format of python # (e.g. [list', '::'] -> ['./pants', 'list', '::']). arguments.insert(0, './pants') self.logger.info('handling pailgun request: `{}`'.format(' '.join(arguments))) self.logger.debug('pailgun request environment: %s', environment) # Instruct the client to send stdin (if applicable). NailgunProtocol.send_start_reading_input(self.request) # Execute the requested command. self._run_pants(self.request, arguments, environment)
def test_isatty_to_env_without_tty(self): mock_stdin = self._make_mock_stream(False, 0) mock_stdout = self._make_mock_stream(False, 1) mock_stderr = self._make_mock_stream(False, 2) self.assertEqual( NailgunProtocol.isatty_to_env(mock_stdin, mock_stdout, mock_stderr), { 'NAILGUN_TTY_0': b'0', 'NAILGUN_TTY_1': b'0', 'NAILGUN_TTY_2': b'0', })
def exit(self, result=0, msg=None): """Exit the runtime.""" if self._finalizer: try: self._finalizer() except Exception as e: try: NailgunProtocol.send_stderr( self._socket, '\nUnexpected exception in finalizer: {!r}\n'.format(e) ) except Exception: pass try: # Write a final message to stderr if present. if msg: NailgunProtocol.send_stderr(self._socket, msg) # Send an Exit chunk with the result. NailgunProtocol.send_exit(self._socket, str(result).encode('ascii')) # Shutdown the connected socket. teardown_socket(self._socket) finally: # N.B. Assuming a fork()'d child, cause os._exit to be called here to avoid the routine # sys.exit behavior (via `pants.util.contextutil.hard_exit_handler()`). raise HardSystemExit()
def exit(self, result=0, msg=None, *args, **kwargs): """Exit the runtime.""" if self._finalizer: try: self._finalizer() except Exception as e: try: NailgunProtocol.send_stderr( self._socket, '\nUnexpected exception in finalizer: {!r}\n'.format(e) ) except Exception: pass try: # Write a final message to stderr if present. if msg: NailgunProtocol.send_stderr(self._socket, msg) # Send an Exit chunk with the result. NailgunProtocol.send_exit_with_code(self._socket, result) # Shutdown the connected socket. teardown_socket(self._socket) finally: super(DaemonExiter, self).exit(result=result, *args, **kwargs)
def run(self): while not self.is_stopped: readable, _, errored = select.select([self._stdin], [], [self._stdin], self._select_timeout) if self._stdin in errored: self.stop() return if not self.is_stopped and self._stdin in readable: data = os.read(self._stdin.fileno(), self._buf_size) if not self.is_stopped: if data: NailgunProtocol.write_chunk(self._socket, ChunkType.STDIN, data) else: NailgunProtocol.write_chunk(self._socket, ChunkType.STDIN_EOF) try: self._socket.shutdown(socket.SHUT_WR) # Shutdown socket sends. except socket.error: # Can happen if response is quick. pass finally: self.stop()
def run(self, args=None): # Merge the nailgun TTY capability environment variables with the passed environment dict. ng_env = NailgunProtocol.isatty_to_env(self._stdin, self._stdout, self._stderr) modified_env = self._combine_dicts(self._env, ng_env) # Instantiate a NailgunClient. client = NailgunClient(port=self._port, ins=self._stdin, out=self._stdout, err=self._stderr) with self._trapped_control_c(client): # Execute the command on the pailgun. result = client.execute(self.PANTS_COMMAND, *self._args, **modified_env) # Exit. self._exiter.exit(result)
def test_process_session_bad_chunk(self): NailgunProtocol.write_chunk(self.server_sock, ChunkType.PID, b'31337') NailgunProtocol.write_chunk(self.server_sock, ChunkType.START_READING_INPUT) NailgunProtocol.write_chunk(self.server_sock, self.BAD_CHUNK_TYPE, '') with self.assertRaises(NailgunClientSession.ProtocolError): self.nailgun_client_session._process_session() self.mock_stdin_reader.start.assert_called_once_with() self.mock_stdin_reader.stop.assert_called_once_with()
def test_isatty_to_env_with_mock_tty(self, mock_ttyname): mock_ttyname.return_value = self._fake_ttyname mock_stdin = self._make_mock_stream(True, 0) mock_stdout = self._make_mock_stream(True, 1) mock_stderr = self._make_mock_stream(True, 2) self.assertEqual( NailgunProtocol.isatty_to_env(mock_stdin, mock_stdout, mock_stderr), { 'NAILGUN_TTY_0': b'1', 'NAILGUN_TTY_1': b'1', 'NAILGUN_TTY_2': b'1', 'NAILGUN_TTY_PATH_0': self._fake_ttyname, 'NAILGUN_TTY_PATH_1': self._fake_ttyname, 'NAILGUN_TTY_PATH_2': self._fake_ttyname, })