def create(cls, sock, args, env, services, scheduler_service):
maybe_shutdown_socket = MaybeShutdownSocket(sock)
try:
# N.B. This will redirect stdio in the daemon's context to the nailgun session.
with cls.nailgunned_stdio(maybe_shutdown_socket,
env,
handle_stdin=False) as finalizer:
options, _, options_bootstrapper = LocalPantsRunner.parse_options(
args, env)
subprocess_dir = options.for_global_scope().pants_subprocessdir
graph_helper, target_roots, exit_code = scheduler_service.prefork(
options, options_bootstrapper)
finalizer()
except Exception:
graph_helper = None
target_roots = None
options_bootstrapper = None
# TODO: this should no longer be necessary, remove the creation of subprocess_dir
subprocess_dir = os.path.join(get_buildroot(), '.pids')
exit_code = 1
# TODO This used to raise the _GracefulTerminationException, and maybe it should again, or notify in some way that the prefork has failed.
return cls(maybe_shutdown_socket, args, env, graph_helper,
target_roots, services, subprocess_dir,
options_bootstrapper, exit_code)
def create(cls, sock, args, env, services, scheduler_service):
return cls(maybe_shutdown_socket=MaybeShutdownSocket(sock),
args=args,
env=env,
services=services,
exit_code=0,
scheduler_service=scheduler_service)
def test_handle_error(self):
self.handler.handle_error()
maybe_shutdown_socket = MaybeShutdownSocket(self.client_sock)
last_chunk_type, last_payload = list(
NailgunProtocol.iter_chunks(maybe_shutdown_socket))[-1]
self.assertEqual(last_chunk_type, ChunkType.EXIT)
self.assertEqual(last_payload, '1')
def _process_session(self):
"""Process the outputs of the nailgun session.
:raises: :class:`NailgunProtocol.ProcessStreamTimeout` if a timeout set from a signal handler
with .set_exit_timeout() completes.
:raises: :class:`Exception` if the session completes before the timeout, the `reason` argument
to .set_exit_timeout() will be raised."""
try:
for chunk_type, payload in self.iter_chunks(
MaybeShutdownSocket(self._sock),
return_bytes=True,
timeout_object=self,
):
# TODO(#6579): assert that we have at this point received all the chunk types in
# ChunkType.REQUEST_TYPES, then require PID and PGRP (exactly once?), and then allow any of
# ChunkType.EXECUTION_TYPES.
if chunk_type == ChunkType.STDOUT:
self._write_flush(self._stdout, payload)
elif chunk_type == ChunkType.STDERR:
self._write_flush(self._stderr, payload)
elif chunk_type == ChunkType.EXIT:
self._write_flush(self._stdout)
self._write_flush(self._stderr)
return int(payload)
elif chunk_type == ChunkType.PID:
self.remote_pid = int(payload)
self.remote_process_cmdline = psutil.Process(self.remote_pid).cmdline()
if self._remote_pid_callback:
self._remote_pid_callback(self.remote_pid)
elif chunk_type == ChunkType.PGRP:
self.remote_pgrp = int(payload)
if self._remote_pgrp_callback:
self._remote_pgrp_callback(self.remote_pgrp)
elif chunk_type == ChunkType.START_READING_INPUT:
self._maybe_start_input_writer()
else:
raise self.ProtocolError('received unexpected chunk {} -> {}'.format(chunk_type, payload))
except NailgunProtocol.ProcessStreamTimeout as e:
assert(self.remote_pid is not None)
# NB: We overwrite the process title in the pantsd process, which causes it to have an
# argv with lots of empty spaces for some reason. We filter those out and pretty-print the
# rest here.
filtered_remote_cmdline = safe_shlex_join(
arg for arg in self.remote_process_cmdline if arg != '')
logger.warning(
"timed out when attempting to gracefully shut down the remote client executing \"{}\". "
"sending SIGKILL to the remote client at pid: {}. message: {}"
.format(filtered_remote_cmdline, self.remote_pid, e))
finally:
# Bad chunk types received from the server can throw NailgunProtocol.ProtocolError in
# NailgunProtocol.iter_chunks(). This ensures the NailgunStreamWriter is always stopped.
self._maybe_stop_input_writer()
# If an asynchronous error was set at any point (such as in a signal handler), we want to make
# sure we clean up the remote process before exiting with error.
if self._exit_reason:
if self.remote_pgrp:
safe_kill(self.remote_pgrp, signal.SIGKILL)
if self.remote_pid:
safe_kill(self.remote_pid, signal.SIGKILL)
raise self._exit_reason
def test_iter_chunks(self):
expected_chunks = [
(ChunkType.COMMAND, self.TEST_COMMAND),
(ChunkType.STDOUT, self.TEST_OUTPUT),
(ChunkType.STDERR, self.TEST_OUTPUT),
(ChunkType.EXIT, self.EMPTY_PAYLOAD)
# N.B. without an EXIT chunk here (or socket failure), this test will deadlock in iter_chunks.
]
for chunk_type, payload in expected_chunks:
NailgunProtocol.write_chunk(self.server_sock, chunk_type, payload)
for i, chunk in enumerate(
NailgunProtocol.iter_chunks(
MaybeShutdownSocket(self.client_sock))):
self.assertEqual(chunk, expected_chunks[i])
def _process_session(self):
"""Process the outputs of the nailgun session.
:raises: :class:`NailgunProtocol.ProcessStreamTimeout` if a timeout set from a signal handler
with .set_exit_timeout() completes.
:raises: :class:`Exception` if the session completes before the timeout, the `reason` argument
to .set_exit_timeout() will be raised.
"""
try:
for chunk_type, payload in self.iter_chunks(
MaybeShutdownSocket(self._sock),
return_bytes=True,
timeout_object=self,
):
# TODO(#6579): assert that we have at this point received all the chunk types in
# ChunkType.REQUEST_TYPES, and then allow any of ChunkType.EXECUTION_TYPES.
if chunk_type == ChunkType.STDOUT:
self._write_flush(self._stdout, payload)
elif chunk_type == ChunkType.STDERR:
self._write_flush(self._stderr, payload)
elif chunk_type == ChunkType.EXIT:
self._write_flush(self._stdout)
self._write_flush(self._stderr)
return int(payload)
elif chunk_type == ChunkType.START_READING_INPUT:
self._maybe_start_input_writer()
else:
raise self.ProtocolError(
"received unexpected chunk {} -> {}".format(
chunk_type, payload))
except NailgunProtocol.ProcessStreamTimeout as e:
logger.warning(
"timed out when attempting to gracefully shut down the remote run. Sending SIGKILL"
"message: {}".format(e))
finally:
# Bad chunk types received from the server can throw NailgunProtocol.ProtocolError in
# NailgunProtocol.iter_chunks(). This ensures the NailgunStreamWriter is always stopped.
self._maybe_stop_input_writer()
# If an asynchronous error was set at any point (such as in a signal handler), we want to make
# sure we clean up the remote process before exiting with error.
if self._exit_reason:
raise self._exit_reason
def create(cls, sock, args, env, services, scheduler_service):
maybe_shutdown_socket = MaybeShutdownSocket(sock)
exception = None
exit_code = PANTS_SUCCEEDED_EXIT_CODE
# TODO(#8002) This can probably be moved to the try:except block in DaemonPantsRunner.run() function,
# Making exception handling a lot easier.
try:
# N.B. This will redirect stdio in the daemon's context to the nailgun session.
with cls.nailgunned_stdio(maybe_shutdown_socket, env, handle_stdin=False) as finalizer:
options, _, options_bootstrapper = LocalPantsRunner.parse_options(args, env)
subprocess_dir = options.for_global_scope().pants_subprocessdir
graph_helper, target_roots, exit_code = scheduler_service.prefork(options, options_bootstrapper)
finalizer()
except Exception as e:
graph_helper = None
target_roots = None
options_bootstrapper = None
# TODO: this should no longer be necessary, remove the creation of subprocess_dir
subprocess_dir = os.path.join(get_buildroot(), '.pids')
exception = _PantsProductPrecomputeFailed(e)
# NB: If a scheduler_service.prefork finishes with a non-0 exit code but doesn't raise an exception
# (e.g. ./pants list-and-die-for-testing ...). We still want to know about it.
if exception is None and exit_code != PANTS_SUCCEEDED_EXIT_CODE:
exception = _PantsProductPrecomputeFailed(
_PantsRunFinishedWithFailureException(exit_code=exit_code)
)
return cls(
maybe_shutdown_socket,
args,
env,
graph_helper,
target_roots,
services,
subprocess_dir,
options_bootstrapper,
exception
)
