def hard_timeout_handler(self): now = datetime.datetime.now while True: if self.started: if (now() - self.started).total_seconds() > self.hard_timeout: LOGGER.warning("Request (hard) Timeout => SIGKILL") # Self-Kill mfutil.kill_process_and_children(os.getpid()) time.sleep(1)
def _main(args, remaining_args): p = subprocess.Popen(remaining_args) pid = p.pid for signum in (signal.SIGTERM, signal.SIGINT): signal.signal(signum, functools.partial(on_signal, args.unix_socket, pid, args.signal, args.timeout)) x = threading.Thread(target=socket_up_after, args=(args.unix_socket, args.socket_up_after)) x.daemon = True x.start() while True: try: if six.PY2: # emulate python3 subprocess python3 behaviour if p.poll() is None: time.sleep(1) raise TIMEOUT_EXCEPTION_CLASS else: p.wait(timeout=1) if SMART_SHUTDOWN: LOGGER.info("process: %i (smart) stopped" % pid) elif HARD_SHUTDOWN: pass else: LOGGER.warning("process: %i stopped for an unknown reason " "=> let's shutdown the corresponding socket " "before restart" % pid) socket_down(args.unix_socket, False, 10) # wait a little bit to be sure that all nginx workers # record that the socket is closed time.sleep(0.5) break except TIMEOUT_EXCEPTION_CLASS: pass except Exception: LOGGER.exception("Exception during process waiting => exiting") break if SIG_SHUTDOWN is not None and args.timeout_after_signal > 0: now = datetime.datetime.now() delta = now - SIG_SHUTDOWN if delta.total_seconds() > args.timeout_after_signal: LOGGER.warning("sending SIGKILL to %i and its children", pid) kill_process_and_children(pid) try: os.remove(args.unix_socket) except Exception: pass
def send_signal_when_no_connection(pid, unix_socket, sig, timeout): global SIG_SHUTDOWN, HARD_SHUTDOWN LOGGER.info("stopping new connections and waiting for no connection " "left on %s (timeout: %i seconds)" % (unix_socket, timeout)) if socket_down(unix_socket, True, timeout): SIG_SHUTDOWN = datetime.datetime.now() if six.PY2: LOGGER.info("sending %i to %i", sig, pid) else: LOGGER.info("sending %s to %i", signal.Signals(sig).name, pid) # pylint: disable=E1101 os.kill(pid, sig) else: HARD_SHUTDOWN = datetime.datetime.now() LOGGER.warning("sending SIGKILL to %i and its children", pid) kill_process_and_children(pid)
def _kill(self, signal: int): if self.process is None: return if self.cmd.recursive_sigkill and signal == 9: self.logger.info("Sending signal %i to %i (and children)" % (signal, self.process.pid)) try: kill_process_and_children(self.process.pid) except Exception: self.logger.warning("can't recursively kill %i", exc_info=True) else: self.logger.info("Sending signal %i to %i" % (signal, self.process.pid)) try: os.kill(self.process.pid, signal) except ProcessLookupError: pass except Exception: self.logger.warning("can't kill %i", exc_info=True)
def on_signal(unix_socket, pid, signal, timeout, signum, frame): global SMART_SHUTDOWN, HARD_SHUTDOWN if signum == 15: if SMART_SHUTDOWN is not None: LOGGER.info("received SIGTERM but SMART_SHUTDOWN already in " "progress => ignoring") return LOGGER.info("received SIGTERM => smart closing...") SMART_SHUTDOWN = datetime.datetime.now() x = threading.Thread(target=send_signal_when_no_connection, args=(pid, unix_socket, signal, timeout)) x.daemon = True x.start() elif signum == 2: if HARD_SHUTDOWN: return HARD_SHUTDOWN = datetime.datetime.now() LOGGER.warning("received SIGINT => (non smart) stopping...") LOGGER.warning("sending SIGKILL to %i and its children", pid) kill_process_and_children(pid)
"the number of remaining processes" argparser.add_argument("--silent", action="store_true", help=silent_doc) args = argparser.parse_args() processes_to_kill = get_processes_to_kill() first_count = len(processes_to_kill) for pid in processes_to_kill: if not args.silent: try: proc = psutil.Process(pid) LOG.info("killing remaining process (and children): " "pid:%i, cmdline: %s" % (proc.pid, " ".join(proc.cmdline()))) except Exception: # we can have some exceptions here is some edge cases pass kill_process_and_children(pid) processes_to_kill = get_processes_to_kill() second_count = len(processes_to_kill) for pid in processes_to_kill: if not args.silent: try: proc = psutil.Process(pid) LOG.warning("remaining process not killed: " "pid:%i, cmdline: %s" % (proc.pid, " ".join(proc.cmdline()))) except Exception: # we can have some exceptions here is some edge cases pass if args.silent:
processes_to_kill = get_processes_to_kill() first_count = len(processes_to_kill) with MFProgress() as progress: t = progress.add_task("- Killing remainging processes (if any)...", total=(first_count + 1)) for pid in processes_to_kill: if args.debug: try: proc = psutil.Process(pid) LOG.info("killing remaining process (and children): " "pid:%i, cmdline: %s" % (proc.pid, " ".join(proc.cmdline()))) except Exception: # we can have some exceptions here is some edge cases pass kill_process_and_children(pid) progress.update(t, advance=1) processes_to_kill = get_processes_to_kill() second_count = len(processes_to_kill) if second_count == 0: if first_count > 0: progress.complete_task_warning(t, "%i killed" % first_count) else: progress.complete_task(t) else: for pid in processes_to_kill: progress.complete_task_nok(t, "%i remaining" % second_count) if args.debug: try: proc = psutil.Process(pid)
def main(): parser = argparse.ArgumentParser(description=DESCRIPTION) parser.add_argument('pid', type=int, help='root process pid (to kill)') args = parser.parse_args() kill_process_and_children(args.pid)
def main(): parser = argparse.ArgumentParser(description=DESCRIPTION) parser.add_argument('pid', type=int, help='root process pid (to kill)') args = parser.parse_args() kill_process_and_children(args.pid)