def wait_for_exit(runner: Runner, main_process: Popen) -> None:
"""
Monitor main process and background items until done
"""
runner.write("Everything launched. Waiting to exit...")
main_command = str_command(str(arg) for arg in main_process.args)
span = runner.span()
while True:
sleep(0.1)
main_code = main_process.poll()
if main_code is not None:
# Shell exited, we're done. Automatic shutdown cleanup will kill
# subprocesses.
runner.write(
"Main process ({})\n exited with code {}.".format(
main_command, main_code
)
)
span.end()
runner.set_success(True)
raise SystemExit(main_code)
if runner.tracked:
dead_bg = runner.tracked.which_dead()
else:
dead_bg = None
if dead_bg:
# Unfortunately torsocks doesn't deal well with connections
# being lost, so best we can do is shut down.
# FIXME: Look at bg.critical and do something smarter
runner.show(
"Proxy to Kubernetes exited. This is typically due to"
" a lost connection."
)
span.end()
raise runner.fail("Exiting...", code=3)
def wait_for_exit(self, main_process: Popen) -> None:
"""
Monitor main process and background items until done
"""
self.write("Everything launched. Waiting to exit...")
main_code = None
span = self.span()
while not self.quitting and main_code is None:
sleep(0.1)
main_code = main_process.poll()
span.end()
if main_code is not None:
# Shell exited, we're done. Automatic shutdown cleanup
# will kill subprocesses.
main_command = str_command(str(arg) for arg in main_process.args)
self.write("Main process ({})".format(main_command))
self.write(" exited with code {}.".format(main_code))
raise self.exit()
# Something else exited, setting the quitting flag.
# Unfortunately torsocks doesn't deal well with connections
# being lost, so best we can do is shut down.
if self.ended:
self.show("\n")
self.show_raw(self.ended[0])
self.show("\n")
message = ("Proxy to Kubernetes exited. This is typically due to"
" a lost connection.")
raise self.fail(message, code=3)
def __init__(self, logfile_path: str) -> None:
"""
Create output handle
:param logfile_path: Path or string file path or "-" for stdout
"""
if logfile_path == "-":
self.logfile = sys.stdout
else:
# Log file path should be absolute since some processes may run in
# different directories:
logfile_path = os.path.abspath(logfile_path)
self.logfile = _open_logfile(logfile_path)
self.logfile_path = logfile_path
self.start_time = curtime()
self.logtail = deque(maxlen=25) # type: deque # keep last 25 lines
self.write("Telepresence {} launched at {}".format(
__version__, ctime()))
self.write(" {}".format(str_command(sys.argv)))
if version_override:
self.write(" TELEPRESENCE_VERSION is {}".format(image_version))
elif image_version != __version__:
self.write(" Using images version {} (dev)".format(image_version))
def __init__(self, logfile_path: str) -> None:
"""
Create output handle
:param logfile_path: Path or string file path or "-" for stdout
"""
# Fail if current working directory does not exist so we don't crash in
# standard library path-handling code.
try:
os.getcwd()
except OSError:
exit("T: Error: Current working directory does not exist.")
if logfile_path == "-":
self.logfile = sys.stdout
else:
# Log file path should be absolute since some processes may run in
# different directories:
logfile_path = os.path.abspath(logfile_path)
self.logfile = _open_logfile(logfile_path)
self.logfile_path = logfile_path
self.start_time = curtime()
self.logtail = deque(maxlen=25) # type: deque # keep last 25 lines
self.write("Telepresence {} launched at {}".format(
__version__, ctime()))
self.write(" {}".format(str_command(sys.argv)))
if version_override:
self.write(" TELEPRESENCE_VERSION is {}".format(image_version))
elif image_version != __version__:
self.write(" Using images version {} (dev)".format(image_version))
def _run_command(self, track, msg1, msg2, out_cb, err_cb, args, **kwargs):
"""Run a command synchronously"""
self.output.write("[{}] {}: {}".format(track, msg1, str_command(args)))
span = self.span("{} {}".format(track, str_command(args))[:80],
False,
verbose=False)
process = self._launch_command(track, out_cb, err_cb, args, **kwargs)
process.wait()
spent = span.end()
retcode = process.poll()
if retcode:
self.output.write("[{}] exit {} in {:0.2f} secs.".format(
track, retcode, spent))
raise CalledProcessError(retcode, args)
if spent > 1:
self.output.write("[{}] {} in {:0.2f} secs.".format(
track, msg2, spent))
def done(proc):
retcode = proc.wait()
self.output.write("[{}] exit {}".format(track, retcode))
self.quitting = True
recent_lines = [str(line) for line in capture if line is not None]
recent = " ".join(recent_lines).strip()
if recent:
recent = "\nRecent output was:\n {}".format(recent)
message = ("Background process ({}) exited with return code {}. "
"Command was:\n {}\n{}").format(
name, retcode, str_command(args), recent)
self.ended.append(message)
def wait_for_process(p: "Popen[str]") -> None:
"""Wait for process and set main_code and self.quitting flag
Note that main_code is defined in the parent function,
so it is declared as nonlocal
See https://github.com/telepresenceio/telepresence/issues/1003
"""
nonlocal main_code
main_code = p.wait()
main_command = str_command(str(arg) for arg in main_process.args)
self.write("Main process ({})".format(main_command))
self.write(" exited with code {}.".format(main_code))
self.quitting = True
def wait_for_exit(self, main_process: "Popen[str]") -> None:
"""
Monitor main process and background items until done
"""
main_code = None
def wait_for_process(p: "Popen[str]") -> None:
"""Wait for process and set main_code and self.quitting flag
Note that main_code is defined in the parent function,
so it is declared as nonlocal
See https://github.com/telepresenceio/telepresence/issues/1003
"""
nonlocal main_code
main_code = p.wait()
self.quitting = True
self.write("Everything launched. Waiting to exit...")
span = self.span()
Thread(target=wait_for_process, args=(main_process, )).start()
while not self.quitting:
sleep(0.1)
span.end()
if main_code is not None:
# User process exited, we're done. Automatic shutdown cleanup
# will kill subprocesses.
main_command = str_command(str(arg) for arg in main_process.args)
self.write("Main process ({})".format(main_command))
self.write(" exited with code {}.".format(main_code))
message = "Your process "
if main_code:
message += "exited with return code {}.".format(main_code)
else:
message += "has exited."
self.show(message)
raise self.exit(main_code)
# Something else exited, setting the quitting flag.
# Unfortunately torsocks doesn't deal well with connections
# being lost, so best we can do is shut down.
if self.ended:
self.show("\n")
self.show_raw(self.ended[0])
self.show("\n")
message = ("Proxy to Kubernetes exited. This is typically due to"
" a lost connection.")
raise self.fail(message)
def popen(self, args, **kwargs) -> Popen:
"""Return Popen object."""
self.counter = track = self.counter + 1
out_cb = err_cb = self.make_logger(track)
def done(proc):
self._popen_done(track, proc)
self.output.write(
"[{}] Launching: {}".format(track, str_command(args))
)
process = self.launch_command(
track, out_cb, err_cb, args, done=done, **kwargs
)
return process
def done(proc: "Popen[str]") -> None:
retcode = proc.wait()
self.output.write("[{}] {}: exit {}".format(track, name, retcode))
recent = "\n ".join(out_logger.get_captured().split("\n"))
if recent:
recent = "\nRecent output was:\n {}".format(recent)
message = ("Background process ({}) exited with return code {}. "
"Command was:\n {}\n{}").format(
name, retcode, str_command(args), recent)
self.ended.append(message)
if is_critical:
# End the program because this is a critical subprocess
self.quitting = True
else:
# Record the failure but don't quit
self.output.write(message)
def _popen(self, name: str, args, **kwargs) -> typing.Tuple[int, Popen]:
"""Return Popen object."""
self.counter = track = self.counter + 1
out_cb = err_cb = self._make_logger(track)
def done(proc):
retcode = proc.wait()
self.output.write("[{}] exit {}".format(track, retcode))
self.output.write(
"[{}] Launching {}: {}".format(track, name, str_command(args))
)
process = self._launch_command(
track, out_cb, err_cb, args, done=done, **kwargs
)
return track, process
def launch(self,
name: str,
args,
killer=None,
keep_session=False,
**kwargs) -> None:
if not keep_session:
# This prevents signals from getting forwarded, but breaks sudo
# if it is configured to ask for a password.
kwargs["start_new_session"] = True
assert "stderr" not in kwargs
kwargs["stderr"] = STDOUT
self.counter = track = self.counter + 1
capture = deque(maxlen=10) # type: typing.MutableSequence[str]
out_cb = err_cb = self._make_logger(track, capture=capture)
def done(proc):
retcode = proc.wait()
self.output.write("[{}] exit {}".format(track, retcode))
self.quitting = True
recent_lines = [str(line) for line in capture if line is not None]
recent = " ".join(recent_lines).strip()
if recent:
recent = "\nRecent output was:\n {}".format(recent)
message = ("Background process ({}) exited with return code {}. "
"Command was:\n {}\n{}").format(
name, retcode, str_command(args), recent)
self.ended.append(message)
self.output.write("[{}] Launching {}: {}".format(
track, name, str_command(args)))
try:
process = _launch_command(args,
out_cb,
err_cb,
done=done,
**kwargs)
except OSError as exc:
self.output.write("[{}] {}".format(track, exc))
raise
self.add_cleanup(
"Kill BG process [{}] {}".format(track, name),
killer if killer else partial(kill_process, process),
)
def report_subprocess_failure(
self, exc: typing.Union[CalledProcessError, TimeoutExpired]
) -> None:
if isinstance(exc, TimeoutExpired):
command = exc.cmd
message = "Timed out after {:.2f} seconds)".format(exc.timeout)
if exc.output:
output = exc.output
else:
output = "[no output]"
elif isinstance(exc, CalledProcessError):
command = exc.cmd
message = "Exited with return code {}".format(exc.returncode)
if exc.output:
output = exc.output
else:
output = "[no output]"
else:
raise exc # i.e. crash
indent = " "
output = indent + ("\n" + indent).join(output.splitlines())
self.show("{}$ {}".format(indent, str_command(command)))
self.show_raw(output)
self.show(indent + "--> " + message)
def command_span(self, track, args):
return self.span(
"{} {}".format(track, str_command(args))[:80],
False,
verbose=False
)
def launch(
self,
name: str,
args: typing.List[str],
killer: typing.Optional[typing.Callable[[], None]] = None,
notify: bool = False,
keep_session: bool = False,
bufsize: int = -1,
is_critical: bool = True,
) -> None:
"""Asyncrounously run a process.
:param name: A human-friendly name to describe the process.
:param args: The command to run.
:param killer: How to signal to the process that it should
stop. The default is to call Popen.terminate(), which on
POSIX OSs sends SIGTERM.
:param notify: Whether to synchronously wait for the process
to send "READY=1" via the ``sd_notify(3)`` interface before
returning.
:param keep_session: Whether to run the process in the current
session (as in ``setsid()``), or in a new session. The
default is to run in a new session, in order to prevent
keyboard signals from getting forwarded. However, running in
a new session breaks sudo if it is configured to ask for a
password.
:parmam bufsize: See ``subprocess.Popen()`.
:param is_critical: Whether this process quitting should end this
Telepresence session. Default is True because that used to be the
only supported behavior.
:return: ``None``.
"""
self.counter = track = self.counter + 1
out_logger = self._make_logger(track, True, True, 10)
def done(proc: "Popen[str]") -> None:
retcode = proc.wait()
self.output.write("[{}] {}: exit {}".format(track, name, retcode))
recent = "\n ".join(out_logger.get_captured().split("\n"))
if recent:
recent = "\nRecent output was:\n {}".format(recent)
message = ("Background process ({}) exited with return code {}. "
"Command was:\n {}\n{}").format(
name, retcode, str_command(args), recent)
self.ended.append(message)
if is_critical:
# End the program because this is a critical subprocess
self.quitting = True
else:
# Record the failure but don't quit
self.output.write(message)
self.output.write("[{}] Launching {}: {}".format(
track, name, str_command(args)))
env = os.environ.copy()
if notify:
sockname = str(self.temp / "notify-{}".format(track))
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
sock.bind(sockname)
env["NOTIFY_SOCKET"] = sockname
try:
process = _launch_command(
args,
out_logger,
out_logger, # Won't be used
done=done,
# kwargs
start_new_session=not keep_session,
stderr=STDOUT,
bufsize=bufsize,
env=env)
except OSError as exc:
self.output.write("[{}] {}".format(track, exc))
raise
if killer is None:
killer = partial(kill_process, process)
self.add_cleanup("Kill BG process [{}] {}".format(track, name), killer)
if notify:
# We need a select()able notification of death in case the
# process dies before sending READY=1. In C, I'd do this
# same pipe trick, but close the pipe from a SIGCHLD
# handler, which is lighter than a thread. But I fear
# that a SIGCHLD handler would interfere with the Python
# runtime? We're already using several threads per
# launched process, so what's the harm in one more?
pr, pw = os.pipe()
def pipewait() -> None:
process.wait()
os.close(pw)
Thread(target=pipewait, daemon=True).start()
# Block until either the process exits or we get a READY=1
# line on the socket.
while process.poll() is None:
r, _, x = select.select([pr, sock], [], [pr, sock])
if sock in r or sock in x:
lines = sock.recv(4096).decode("utf-8").split("\n")
if "READY=1" in lines:
break
os.close(pr)
sock.close()
def _run_command_sync(
self,
messages: typing.Tuple[str, str],
log_stdout: bool,
stderr_to_stdout: bool,
args: typing.List[str],
capture_limit: int,
timeout: typing.Optional[float],
input: typing.Optional[bytes],
env: typing.Optional[typing.Dict[str, str]],
) -> str:
"""
Run a command synchronously. Log stdout (optionally) and stderr (if not
redirected to stdout). Capture stdout and stderr, at least for
exceptions. Return output.
"""
self.counter = track = self.counter + 1
self.output.write("[{}] {}: {}".format(track, messages[0],
str_command(args)))
span = self.span("{} {}".format(track, str_command(args))[:80],
False,
verbose=False)
kwargs = {} # type: typing.Dict[str, typing.Any]
if env is not None:
kwargs["env"] = env
if input is not None:
kwargs["input"] = input
# Set up capture/logging
out_logger = self._make_logger(track, log_stdout or self.verbose, True,
capture_limit)
if stderr_to_stdout:
# This logger won't be used
err_logger = self._make_logger(track, False, False, capture_limit)
kwargs["stderr"] = STDOUT
else:
err_logger = self._make_logger(track, True, True, capture_limit)
# Launch the process and wait for it to finish
try:
process = _launch_command(args, out_logger, err_logger, **kwargs)
except OSError as exc:
# Failed to launch, so no need to wrap up capture stuff.
self.output.write("[{}] {}".format(track, exc))
raise
TIMED_OUT_RETCODE = -999
try:
retcode = process.wait(timeout)
except TimeoutExpired:
retcode = TIMED_OUT_RETCODE # sentinal for timeout
process.terminate()
try:
process.wait(timeout=1)
except TimeoutExpired:
process.kill()
process.wait()
output = out_logger.get_captured()
spent = span.end()
if retcode == TIMED_OUT_RETCODE:
# Command timed out. Need to raise TE.
self.output.write("[{}] timed out after {:0.2f} secs.".format(
track, spent))
assert timeout is not None
raise TimeoutExpired(
args,
timeout,
output,
None if stderr_to_stdout else err_logger.get_captured(),
)
if retcode:
# Command failed. Need to raise CPE.
self.output.write("[{}] exit {} in {:0.2f} secs.".format(
track, retcode, spent))
raise CalledProcessError(
retcode,
args,
output,
None if stderr_to_stdout else err_logger.get_captured(),
)
# Command succeeded. Just return the output
self.output.write("[{}] {} in {:0.2f} secs.".format(
track, messages[1], spent))
return output
