def hard_timeout_handler(self):
now = datetime.datetime.now
while True:
if self.started:
if (now() - self.started).total_seconds() > self.hard_timeout:
LOGGER.warning("Request (hard) Timeout => SIGKILL")
# Self-Kill
mfutil.kill_process_and_children(os.getpid())
time.sleep(1)
def _main(args, remaining_args):
p = subprocess.Popen(remaining_args)
pid = p.pid
for signum in (signal.SIGTERM, signal.SIGINT):
signal.signal(signum,
functools.partial(on_signal,
args.unix_socket, pid, args.signal,
args.timeout))
x = threading.Thread(target=socket_up_after,
args=(args.unix_socket, args.socket_up_after))
x.daemon = True
x.start()
while True:
try:
if six.PY2:
# emulate python3 subprocess python3 behaviour
if p.poll() is None:
time.sleep(1)
raise TIMEOUT_EXCEPTION_CLASS
else:
p.wait(timeout=1)
if SMART_SHUTDOWN:
LOGGER.info("process: %i (smart) stopped" % pid)
elif HARD_SHUTDOWN:
pass
else:
LOGGER.warning("process: %i stopped for an unknown reason "
"=> let's shutdown the corresponding socket "
"before restart" % pid)
socket_down(args.unix_socket, False, 10)
# wait a little bit to be sure that all nginx workers
# record that the socket is closed
time.sleep(0.5)
break
except TIMEOUT_EXCEPTION_CLASS:
pass
except Exception:
LOGGER.exception("Exception during process waiting => exiting")
break
if SIG_SHUTDOWN is not None and args.timeout_after_signal > 0:
now = datetime.datetime.now()
delta = now - SIG_SHUTDOWN
if delta.total_seconds() > args.timeout_after_signal:
LOGGER.warning("sending SIGKILL to %i and its children", pid)
kill_process_and_children(pid)
try:
os.remove(args.unix_socket)
except Exception:
pass
def send_signal_when_no_connection(pid, unix_socket, sig, timeout):
global SIG_SHUTDOWN, HARD_SHUTDOWN
LOGGER.info("stopping new connections and waiting for no connection "
"left on %s (timeout: %i seconds)" %
(unix_socket, timeout))
if socket_down(unix_socket, True, timeout):
SIG_SHUTDOWN = datetime.datetime.now()
if six.PY2:
LOGGER.info("sending %i to %i", sig, pid)
else:
LOGGER.info("sending %s to %i",
signal.Signals(sig).name, pid) # pylint: disable=E1101
os.kill(pid, sig)
else:
HARD_SHUTDOWN = datetime.datetime.now()
LOGGER.warning("sending SIGKILL to %i and its children", pid)
kill_process_and_children(pid)
def _kill(self, signal: int):
if self.process is None:
return
if self.cmd.recursive_sigkill and signal == 9:
self.logger.info("Sending signal %i to %i (and children)" %
(signal, self.process.pid))
try:
kill_process_and_children(self.process.pid)
except Exception:
self.logger.warning("can't recursively kill %i", exc_info=True)
else:
self.logger.info("Sending signal %i to %i" %
(signal, self.process.pid))
try:
os.kill(self.process.pid, signal)
except ProcessLookupError:
pass
except Exception:
self.logger.warning("can't kill %i", exc_info=True)
def on_signal(unix_socket, pid, signal, timeout, signum, frame):
global SMART_SHUTDOWN, HARD_SHUTDOWN
if signum == 15:
if SMART_SHUTDOWN is not None:
LOGGER.info("received SIGTERM but SMART_SHUTDOWN already in "
"progress => ignoring")
return
LOGGER.info("received SIGTERM => smart closing...")
SMART_SHUTDOWN = datetime.datetime.now()
x = threading.Thread(target=send_signal_when_no_connection,
args=(pid, unix_socket, signal, timeout))
x.daemon = True
x.start()
elif signum == 2:
if HARD_SHUTDOWN:
return
HARD_SHUTDOWN = datetime.datetime.now()
LOGGER.warning("received SIGINT => (non smart) stopping...")
LOGGER.warning("sending SIGKILL to %i and its children", pid)
kill_process_and_children(pid)
"the number of remaining processes"
argparser.add_argument("--silent", action="store_true", help=silent_doc)
args = argparser.parse_args()
processes_to_kill = get_processes_to_kill()
first_count = len(processes_to_kill)
for pid in processes_to_kill:
if not args.silent:
try:
proc = psutil.Process(pid)
LOG.info("killing remaining process (and children): "
"pid:%i, cmdline: %s" % (proc.pid,
" ".join(proc.cmdline())))
except Exception:
# we can have some exceptions here is some edge cases
pass
kill_process_and_children(pid)
processes_to_kill = get_processes_to_kill()
second_count = len(processes_to_kill)
for pid in processes_to_kill:
if not args.silent:
try:
proc = psutil.Process(pid)
LOG.warning("remaining process not killed: "
"pid:%i, cmdline: %s" % (proc.pid,
" ".join(proc.cmdline())))
except Exception:
# we can have some exceptions here is some edge cases
pass
if args.silent:
processes_to_kill = get_processes_to_kill()
first_count = len(processes_to_kill)
with MFProgress() as progress:
t = progress.add_task("- Killing remainging processes (if any)...",
total=(first_count + 1))
for pid in processes_to_kill:
if args.debug:
try:
proc = psutil.Process(pid)
LOG.info("killing remaining process (and children): "
"pid:%i, cmdline: %s" %
(proc.pid, " ".join(proc.cmdline())))
except Exception:
# we can have some exceptions here is some edge cases
pass
kill_process_and_children(pid)
progress.update(t, advance=1)
processes_to_kill = get_processes_to_kill()
second_count = len(processes_to_kill)
if second_count == 0:
if first_count > 0:
progress.complete_task_warning(t, "%i killed" % first_count)
else:
progress.complete_task(t)
else:
for pid in processes_to_kill:
progress.complete_task_nok(t, "%i remaining" % second_count)
if args.debug:
try:
proc = psutil.Process(pid)
def main():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument('pid', type=int,
help='root process pid (to kill)')
args = parser.parse_args()
kill_process_and_children(args.pid)
def main():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument('pid', type=int, help='root process pid (to kill)')
args = parser.parse_args()
kill_process_and_children(args.pid)