def __init__(self, till=None, timeout=None, seconds=None): global next_ping now = time() if till != None: if not isinstance(till, (float, int)): from mo_logs import Log Log.error("Date objects for Till are no longer allowed") timeout = till elif seconds != None: timeout = now + seconds elif timeout != None: if not isinstance(timeout, (float, int)): from mo_logs import Log Log.error("Duration objects for Till are no longer allowed") timeout = now + timeout Signal.__init__(self, name=unicode(timeout)) with _till_locker: if timeout != None: next_ping = min(next_ping, timeout) Till.new_timers.append((timeout, self))
def __init__(self, till=None, timeout=None, seconds=None): now = time() if till != None: if not isinstance(till, (float, int)): from mo_logs import Log Log.error("Date objects for Till are no longer allowed") timeout = till elif seconds != None: timeout = now + seconds elif timeout != None: if not isinstance(timeout, (float, int)): from mo_logs import Log Log.error("Duration objects for Till are no longer allowed") timeout = now + timeout else: from mo_logs import Log Log.error("Should not happen") Signal.__init__(self, name=text_type(timeout)) with Till.locker: if timeout != None: Till.next_ping = min(Till.next_ping, timeout) Till.new_timers.append((timeout, ref(self)))
def __init__(self, name, target, *args, **kwargs): self.id = -1 self.name = name self.target = target self.end_of_thread = None self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.kwargs["please_stop"] = self.kwargs.get( "please_stop", Signal("please_stop for " + self.name)) self.please_stop = self.kwargs["please_stop"] self.thread = None self.stopped = Signal("stopped signal for " + self.name) self.cprofiler = Null self.children = [] if "parent_thread" in kwargs: del self.kwargs["parent_thread"] self.parent = kwargs["parent_thread"] else: self.parent = Thread.current() self.parent.add_child(self)
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): self.name = name self.service_stopped = Signal("stopped signal for " + strings.quote(name)) self.stdin = Queue("stdin for process " + strings.quote(name), silent=True) self.stdout = Queue("stdout for process " + strings.quote(name), silent=True) self.stderr = Queue("stderr for process " + strings.quote(name), silent=True) try: self.debug = debug or DEBUG self.service = service = subprocess.Popen( [str(p) for p in params], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd if isinstance(cwd, (str, NullType, none_type)) else cwd.abspath, env={str(k): str(v) for k, v in set_default(env, os.environ).items()}, shell=shell ) self.please_stop = Signal() self.please_stop.on_go(self._kill) self.thread_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception as e: Log.error("Can not call", e) self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): self.name = name self.service_stopped = Signal("stopped signal for " + strings.quote(name)) self.stdin = Queue("stdin for process " + strings.quote(name), silent=True) self.stdout = Queue("stdout for process " + strings.quote(name), silent=True) self.stderr = Queue("stderr for process " + strings.quote(name), silent=True) try: self.debug = debug or DEBUG self.service = service = subprocess.Popen( params, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath, env=unwrap(set_default(env, os.environ)), shell=shell ) self.please_stop = Signal() self.please_stop.on_go(self._kill) self.thread_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception as e: Log.error("Can not call", e) if self.debug: Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self): self.name = "Main Thread" self.id = get_ident() self.please_stop = Signal() self.children = [] self.stop_logging = Log.stop self.timers = None self.cprofiler = Null
def __init__(self, name, max=None, silent=False, unique=False, allow_add_after_close=False): """ max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK silent - COMPLAIN IF THE READERS ARE TOO SLOW unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME """ self.name = name self.max = coalesce(max, 2 ** 10) self.silent = silent self.allow_add_after_close=allow_add_after_close self.unique = unique self.please_stop = Signal("stop signal for " + name) self.lock = Lock("lock for queue " + name) self.queue = deque() self.next_warning = time() # FOR DEBUGGING
def wait(self, till=None): """ THE ASSUMPTION IS wait() WILL ALWAYS RETURN WITH THE LOCK ACQUIRED :param till: WHEN TO GIVE UP WAITING FOR ANOTHER THREAD TO SIGNAL :return: True IF SIGNALED TO GO, False IF TIMEOUT HAPPENED """ waiter = Signal() if self.waiting: if DEBUG: _Log.note("{{name}} waiting with others", name=self.name) self.waiting.insert(0, waiter) else: self.waiting = [waiter] try: self.lock.release() (waiter | till).wait() if DEBUG: trace = _extract_stack(0)[2] _Log.note( "{{name|quote}} out of lock waiting till {{till|quote}}\n{{trace}} ", till=till.name if till else "", name=self.name, trace=trace) except Exception, e: if not _Log: _late_import() _Log.warning("problem", cause=e)
def wait_for_shutdown_signal( please_stop=False, # ASSIGN SIGNAL TO STOP EARLY allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False -> IF NO CHILD THREADS LEFT, THEN EXIT ): """ FOR USE BY PROCESSES NOT EXPECTED TO EVER COMPLETE UNTIL EXTERNAL SHUTDOWN IS REQUESTED SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit" :param please_stop: :param allow_exit: :param wait_forever:: Assume all needed threads have been launched. When done :return: """ if not isinstance(please_stop, Signal): please_stop = Signal() please_stop.on_go( lambda: thread.start_new_thread(_stop_main_thread, ())) self_thread = Thread.current() if self_thread != MAIN_THREAD: Log.error( "Only the main thread can sleep forever (waiting for KeyboardInterrupt)" ) if not wait_forever: # TRIGGER SIGNAL WHEN ALL EXITING THREADS ARE DONE pending = copy(self_thread.children) all = AndSignals(please_stop, len(pending)) for p in pending: p.stopped.on_go(all.done) try: if allow_exit: _wait_for_exit(please_stop) else: _wait_for_interrupt(please_stop) except (KeyboardInterrupt, SystemExit), _: Log.alert("SIGINT Detected! Stopping...")
def __init__(self, name, target, *args, **kwargs): BaseThread.__init__(self, -1) self.name = coalesce(name, "thread_" + text_type(object.__hash__(self))) self.target = target self.end_of_thread = Data() self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.kwargs["please_stop"] = self.kwargs.get("please_stop", Signal("please_stop for " + self.name)) self.please_stop = self.kwargs["please_stop"] self.thread = None self.stopped = Signal("stopped signal for " + self.name) if "parent_thread" in kwargs: del self.kwargs["parent_thread"] self.parent = kwargs["parent_thread"] else: self.parent = Thread.current() self.parent.add_child(self)
def __init__(self, till=None, seconds=None): """ ONE OF THESE PARAMETERS IS REQUIRED :param till: UNIX TIMESTAMP OF WHEN TO SIGNAL :param seconds: PREFERRED OVER timeout """ now = time() if till != None: if not isinstance(till, (float, int)): from mo_logs import Log Log.error("Date objects for Till are no longer allowed") timeout = till elif seconds != None: timeout = now + seconds else: from mo_logs import Log raise Log.error("Should not happen") Signal.__init__(self, name=text_type(timeout)) with Till.locker: if timeout != None: Till.next_ping = min(Till.next_ping, timeout) Till.new_timers.append(TodoItem(timeout, ref(self)))
def __init__(self, till=None, seconds=None): """ ONE OF THESE PARAMETERS IS REQUIRED :param till: UNIX TIMESTAMP OF WHEN TO SIGNAL :param seconds: PREFERRED OVER timeout """ now = time() if till != None: if not isinstance(till, (float, int)): from mo_logs import Log Log.error("Date objects for Till are no longer allowed") timeout = till elif seconds != None: timeout = now + seconds else: raise Log.error("Should not happen") Signal.__init__(self, name=text_type(timeout)) with Till.locker: if timeout != None: Till.next_ping = min(Till.next_ping, timeout) Till.new_timers.append(TodoItem(timeout, ref(self)))
def __init__(self, till=None, timeout=None, seconds=None): now = time() if till != None: if not isinstance(till, (float, int)): from mo_logs import Log Log.error("Date objects for Till are no longer allowed") timeout = till elif seconds != None: timeout = now + seconds elif timeout != None: if not isinstance(timeout, (float, int)): from mo_logs import Log Log.error("Duration objects for Till are no longer allowed") timeout = now + timeout Signal.__init__(self, name=unicode(timeout)) with Till.locker: if timeout != None: Till.next_ping = min(Till.next_ping, timeout) Till.new_timers.append((timeout, self))
class Till(Signal): """ TIMEOUT AS A SIGNAL """ __slots__ = [] locker = _allocate_lock() next_ping = time() done = Signal("Timers shutdown") enabled = False new_timers = [] def __new__(cls, till=None, timeout=None, seconds=None): if not Till.enabled: return Till.done elif till == None and timeout == None and seconds == None: return None else: return object.__new__(cls) def __init__(self, till=None, timeout=None, seconds=None): now = time() if till != None: if not isinstance(till, (float, int)): from mo_logs import Log Log.error("Date objects for Till are no longer allowed") timeout = till elif seconds != None: timeout = now + seconds elif timeout != None: if not isinstance(timeout, (float, int)): from mo_logs import Log Log.error("Duration objects for Till are no longer allowed") timeout = now + timeout else: from mo_logs import Log Log.error("Should not happen") Signal.__init__(self, name=text_type(timeout)) with Till.locker: if timeout != None: Till.next_ping = min(Till.next_ping, timeout) Till.new_timers.append((timeout, ref(self)))
def wait(self, till=None): """ THE ASSUMPTION IS wait() WILL ALWAYS RETURN WITH THE LOCK ACQUIRED :param till: WHEN TO GIVE UP WAITING FOR ANOTHER THREAD TO SIGNAL :return: True IF SIGNALED TO GO, False IF TIMEOUT HAPPENED """ waiter = Signal() if self.waiting: if DEBUG: _Log.note("waiting with {{num}} others on {{name|quote}}", num=len(self.waiting), name=self.name) self.waiting.insert(0, waiter) else: if DEBUG: _Log.note("waiting by self on {{name|quote}}", name=self.name) self.waiting = [waiter] try: self.lock.release() if DEBUG: _Log.note("out of lock {{name|quote}}", name=self.name) (waiter | till).wait() if DEBUG: _Log.note("done minimum wait (for signal {{till|quote}})", till=till.name if till else "", name=self.name) except Exception as e: if not _Log: _late_import() _Log.warning("problem", cause=e) finally: self.lock.acquire() if DEBUG: _Log.note("re-acquired lock {{name|quote}}", name=self.name) try: self.waiting.remove(waiter) if DEBUG: _Log.note("removed own signal from {{name|quote}}", name=self.name) except Exception: pass return bool(waiter)
def __init__(self, name, target, *args, **kwargs): BaseThread.__init__(self, -1) self.name = name self.target = target self.end_of_thread = Data() self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.kwargs["please_stop"] = self.kwargs.get("please_stop", Signal("please_stop for " + self.name)) self.please_stop = self.kwargs["please_stop"] self.thread = None self.stopped = Signal("stopped signal for " + self.name) if "parent_thread" in kwargs: del self.kwargs["parent_thread"] self.parent = kwargs["parent_thread"] else: self.parent = Thread.current() self.parent.add_child(self)
class Thread(object): """ join() ENHANCED TO ALLOW CAPTURE OF CTRL-C, AND RETURN POSSIBLE THREAD EXCEPTIONS run() ENHANCED TO CAPTURE EXCEPTIONS """ num_threads = 0 def __init__(self, name, target, *args, **kwargs): self.id = -1 self.name = name self.target = target self.end_of_thread = None self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.kwargs["please_stop"] = self.kwargs.get( "please_stop", Signal("please_stop for " + self.name)) self.please_stop = self.kwargs["please_stop"] self.thread = None self.stopped = Signal("stopped signal for " + self.name) self.cprofiler = None self.children = [] if "parent_thread" in kwargs: del self.kwargs["parent_thread"] self.parent = kwargs["parent_thread"] else: self.parent = Thread.current() self.parent.add_child(self) def __enter__(self): return self def __exit__(self, type, value, traceback): if isinstance(type, BaseException): self.please_stop.go() # TODO: AFTER A WHILE START KILLING THREAD self.join() self.args = None self.kwargs = None def start(self): try: self.thread = thread.start_new_thread(Thread._run, (self, )) return self except Exception as e: Log.error("Can not start thread", e) def stop(self): for c in copy(self.children): if DEBUG and c.name: Log.note("Stopping thread {{name|quote}}", name=c.name) c.stop() self.please_stop.go() if DEBUG: Log.note("Thread {{name|quote}} got request to stop", name=self.name) def add_child(self, child): self.children.append(child) def remove_child(self, child): try: self.children.remove(child) except Exception as e: # happens when multiple joins on same thread pass def _run(self): with CProfiler(): self.id = thread.get_ident() with ALL_LOCK: ALL[self.id] = self try: if self.target is not None: a, k, self.args, self.kwargs = self.args, self.kwargs, None, None response = self.target(*a, **k) with self.synch_lock: self.end_of_thread = Data(response=response) else: with self.synch_lock: self.end_of_thread = Null except Exception as e: e = Except.wrap(e) with self.synch_lock: self.end_of_thread = Data(exception=e) if self not in self.parent.children: # THREAD FAILURES ARE A PROBLEM ONLY IF NO ONE WILL BE JOINING WITH IT try: Log.fatal("Problem in thread {{name|quote}}", name=self.name, cause=e) except Exception: sys.stderr.write(b"ERROR in thread: " + str(self.name) + b" " + str(e) + b"\n") finally: try: children = copy(self.children) for c in children: try: if DEBUG: sys.stdout.write(b"Stopping thread " + str(c.name) + b"\n") c.stop() except Exception as e: Log.warning("Problem stopping thread {{thread}}", thread=c.name, cause=e) for c in children: try: if DEBUG: sys.stdout.write(b"Joining on thread " + str(c.name) + b"\n") c.join() except Exception as e: Log.warning("Problem joining thread {{thread}}", thread=c.name, cause=e) finally: if DEBUG: sys.stdout.write(b"Joined on thread " + str(c.name) + b"\n") self.stopped.go() if DEBUG: Log.note("thread {{name|quote}} stopping", name=self.name) del self.target, self.args, self.kwargs with ALL_LOCK: del ALL[self.id] except Exception as e: if DEBUG: Log.warning("problem with thread {{name|quote}}", cause=e, name=self.name) finally: self.stopped.go() if DEBUG: Log.note("thread {{name|quote}} is done", name=self.name) def is_alive(self): return not self.stopped def join(self, till=None): """ RETURN THE RESULT {"response":r, "exception":e} OF THE THREAD EXECUTION (INCLUDING EXCEPTION, IF EXISTS) """ if self is Thread: Log.error("Thread.join() is not a valid call, use t.join()") children = copy(self.children) for c in children: c.join(till=till) if DEBUG: Log.note("{{parent|quote}} waiting on thread {{child|quote}}", parent=Thread.current().name, child=self.name) (self.stopped | till).wait() if self.stopped: self.parent.remove_child(self) if not self.end_of_thread.exception: return self.end_of_thread.response else: Log.error("Thread {{name|quote}} did not end well", name=self.name, cause=self.end_of_thread.exception) else: raise Except(type=THREAD_TIMEOUT) @staticmethod def run(name, target, *args, **kwargs): # ENSURE target HAS please_stop ARGUMENT if "please_stop" not in target.__code__.co_varnames: Log.error( "function must have please_stop argument for signalling emergency shutdown" ) Thread.num_threads += 1 output = Thread(name, target, *args, **kwargs) output.start() return output @staticmethod def wait_for_shutdown_signal( please_stop=False, # ASSIGN SIGNAL TO STOP EARLY allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False -> IF NO CHILD THREADS LEFT, THEN EXIT ): """ FOR USE BY PROCESSES NOT EXPECTED TO EVER COMPLETE UNTIL EXTERNAL SHUTDOWN IS REQUESTED SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit" :param please_stop: :param allow_exit: :param wait_forever:: Assume all needed threads have been launched. When done :return: """ if not isinstance(please_stop, Signal): please_stop = Signal() please_stop.on_go( lambda: thread.start_new_thread(_stop_main_thread, ())) self_thread = Thread.current() if self_thread != MAIN_THREAD: Log.error( "Only the main thread can sleep forever (waiting for KeyboardInterrupt)" ) if not wait_forever: # TRIGGER SIGNAL WHEN ALL EXITING THREADS ARE DONE pending = copy(self_thread.children) all = AndSignals(please_stop, len(pending)) for p in pending: p.stopped.on_go(all.done) try: if allow_exit: _wait_for_exit(please_stop) else: _wait_for_interrupt(please_stop) except (KeyboardInterrupt, SystemExit), _: Log.alert("SIGINT Detected! Stopping...") finally:
class Process(object): def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): self.name = name self.service_stopped = Signal("stopped signal for " + strings.quote(name)) self.stdin = Queue("stdin for process " + strings.quote(name), silent=True) self.stdout = Queue("stdout for process " + strings.quote(name), silent=True) self.stderr = Queue("stderr for process " + strings.quote(name), silent=True) try: self.debug = debug or DEBUG self.service = service = subprocess.Popen( params, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath, env=unwrap(set_default(env, os.environ)), shell=shell ) self.please_stop = Signal() self.please_stop.on_go(self._kill) self.thread_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception as e: Log.error("Can not call", e) if self.debug: Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params))) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.join(raise_on_error=True) def stop(self): self.stdin.add("exit") # ONE MORE SEND self.please_stop.go() def join(self, raise_on_error=False): self.service_stopped.wait() with self.thread_locker: child_threads, self.children = self.children, [] for c in child_threads: c.join() if raise_on_error and self.returncode != 0: Log.error( "{{process}} FAIL: returncode={{code}}\n{{stderr}}", process=self.name, code=self.service.returncode, stderr=list(self.stderr) ) return self def remove_child(self, child): with self.thread_locker: try: self.children.remove(child) except Exception: pass @property def pid(self): return self.service.pid @property def returncode(self): return self.service.returncode def _monitor(self, please_stop): self.service.wait() if self.debug: Log.note("{{process}} STOP: returncode={{returncode}}", process=self.name, returncode=self.service.returncode) self.service_stopped.go() please_stop.go() def _reader(self, name, pipe, recieve, please_stop): try: line = "dummy" while not please_stop and self.service.returncode is None and line: line = pipe.readline().rstrip() if line: recieve.add(line) if self.debug: Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line) # GRAB A FEW MORE LINES max = 100 while max: try: line = pipe.readline().rstrip() if line: max = 100 recieve.add(line) if self.debug: Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line) else: max -= 1 except Exception: break finally: pipe.close() recieve.add(THREAD_STOP) def _writer(self, pipe, send, please_stop): while not please_stop: line = send.pop(till=please_stop) if line == THREAD_STOP: please_stop.go() break if line: if self.debug: Log.note("{{process}} (stdin): {{line}}", process=self.name, line=line.rstrip()) pipe.write(line + b"\n") pipe.close() def _kill(self): try: self.service.kill() except Exception as e: ee = Except.wrap(e) if 'The operation completed successfully' in ee: return if 'No such process' in ee: return Log.warning("Failure to kill process {{process|quote}}", process=self.name, cause=ee)
from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals from thread import allocate_lock as _allocate_lock from time import sleep, time from mo_threads.signal import Signal DEBUG = False INTERVAL = 0.1 _till_locker = _allocate_lock() next_ping = time() done = Signal("Timers shutdown") done.go() class Till(Signal): """ TIMEOUT AS A SIGNAL """ enabled = False new_timers = [] def __new__(cls, till=None, timeout=None, seconds=None): if not Till.enabled: return done elif till is None and timeout is None and seconds is None: return None
def __init__(self): BaseThread.__init__(self, get_ident()) self.name = "Main Thread" self.please_stop = Signal() self.stop_logging = Log.stop self.timers = None
class MainThread(BaseThread): def __init__(self): BaseThread.__init__(self, get_ident()) self.name = "Main Thread" self.please_stop = Signal() self.stop_logging = Log.stop self.timers = None def stop(self): """ BLOCKS UNTIL ALL THREADS HAVE STOPPED THEN RUNS sys.exit(0) """ global DEBUG self_thread = Thread.current() if self_thread != MAIN_THREAD or self_thread != self: Log.error("Only the main thread can call stop() on main thread") DEBUG = True self.please_stop.go() join_errors = [] with self.child_lock: children = copy(self.children) for c in reversed(children): DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name) try: c.stop() except Exception as e: join_errors.append(e) for c in children: DEBUG and c.name and Log.note("Joining on thread {{name|quote}}", name=c.name) try: c.join() except Exception as e: join_errors.append(e) DEBUG and c.name and Log.note("Done join on thread {{name|quote}}", name=c.name) if join_errors: Log.error("Problem while stopping {{name|quote}}", name=self.name, cause=unwraplist(join_errors)) self.stop_logging() self.timers.stop() self.timers.join() write_profiles(self.cprofiler) DEBUG and Log.note("Thread {{name|quote}} now stopped", name=self.name) sys.exit() def wait_for_shutdown_signal( self, please_stop=False, # ASSIGN SIGNAL TO STOP EARLY allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False => IF NO CHILD THREADS LEFT, THEN EXIT ): """ FOR USE BY PROCESSES THAT NEVER DIE UNLESS EXTERNAL SHUTDOWN IS REQUESTED CALLING THREAD WILL SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit" :param please_stop: :param allow_exit: :param wait_forever:: Assume all needed threads have been launched. When done :return: """ self_thread = Thread.current() if self_thread != MAIN_THREAD or self_thread != self: Log.error("Only the main thread can sleep forever (waiting for KeyboardInterrupt)") if isinstance(please_stop, Signal): # MUTUAL SIGNALING MAKES THESE TWO EFFECTIVELY THE SAME SIGNAL self.please_stop.on_go(please_stop.go) please_stop.on_go(self.please_stop.go) else: please_stop = self.please_stop if not wait_forever: # TRIGGER SIGNAL WHEN ALL CHILDREN THEADS ARE DONE with self_thread.child_lock: pending = copy(self_thread.children) children_done = AndSignals(please_stop, len(pending)) children_done.signal.on_go(self.please_stop.go) for p in pending: p.stopped.on_go(children_done.done) try: if allow_exit: _wait_for_exit(please_stop) else: _wait_for_interrupt(please_stop) except KeyboardInterrupt as _: Log.alert("SIGINT Detected! Stopping...") except SystemExit as _: Log.alert("SIGTERM Detected! Stopping...") finally: self.stop()
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): self.name = name self.service_stopped = Signal("stopped signal for " + string2quote(name)) self.stdin = Queue("stdin for process " + string2quote(name), silent=True) self.stdout = Queue("stdout for process " + string2quote(name), silent=True) self.stderr = Queue("stderr for process " + string2quote(name), silent=True) try: self.debug = debug or DEBUG self.service = service = subprocess.Popen( params, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath, env=unwrap(set_default(env, os.environ)), shell=shell) self.please_stop = Signal() self.please_stop.on_go(self._kill) self.thread_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception, e: Log.error("Can not call", e)
class Thread(BaseThread): """ join() ENHANCED TO ALLOW CAPTURE OF CTRL-C, AND RETURN POSSIBLE THREAD EXCEPTIONS run() ENHANCED TO CAPTURE EXCEPTIONS """ num_threads = 0 def __init__(self, name, target, *args, **kwargs): BaseThread.__init__(self, -1) self.name = coalesce(name, "thread_" + text_type(object.__hash__(self))) self.target = target self.end_of_thread = Data() self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.kwargs["please_stop"] = self.kwargs.get("please_stop", Signal("please_stop for " + self.name)) self.please_stop = self.kwargs["please_stop"] self.thread = None self.stopped = Signal("stopped signal for " + self.name) if "parent_thread" in kwargs: del self.kwargs["parent_thread"] self.parent = kwargs["parent_thread"] else: self.parent = Thread.current() self.parent.add_child(self) def __enter__(self): return self def __exit__(self, type, value, traceback): if isinstance(type, BaseException): self.please_stop.go() # TODO: AFTER A WHILE START KILLING THREAD self.join() self.args = None self.kwargs = None def start(self): try: self.thread = start_new_thread(Thread._run, (self,)) return self except Exception as e: Log.error("Can not start thread", e) def stop(self): """ SEND STOP SIGNAL, DO NOT BLOCK """ with self.child_lock: children = copy(self.children) for c in children: DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name) c.stop() self.please_stop.go() DEBUG and Log.note("Thread {{name|quote}} got request to stop", name=self.name) def _run(self): self.id = get_ident() with RegisterThread(self): try: if self.target is not None: a, k, self.args, self.kwargs = self.args, self.kwargs, None, None self.end_of_thread.response = self.target(*a, **k) self.parent.remove_child(self) # IF THREAD ENDS OK, THEN FORGET ABOUT IT except Exception as e: e = Except.wrap(e) with self.synch_lock: self.end_of_thread.exception = e with self.parent.child_lock: emit_problem = self not in self.parent.children if emit_problem: # THREAD FAILURES ARE A PROBLEM ONLY IF NO ONE WILL BE JOINING WITH IT try: Log.fatal("Problem in thread {{name|quote}}", name=self.name, cause=e) except Exception: sys.stderr.write(str("ERROR in thread: " + self.name + " " + text_type(e) + "\n")) finally: try: with self.child_lock: children = copy(self.children) for c in children: try: DEBUG and sys.stdout.write(str("Stopping thread " + c.name + "\n")) c.stop() except Exception as e: Log.warning("Problem stopping thread {{thread}}", thread=c.name, cause=e) for c in children: try: DEBUG and sys.stdout.write(str("Joining on thread " + c.name + "\n")) c.join() except Exception as e: Log.warning("Problem joining thread {{thread}}", thread=c.name, cause=e) finally: DEBUG and sys.stdout.write(str("Joined on thread " + c.name + "\n")) del self.target, self.args, self.kwargs DEBUG and Log.note("thread {{name|quote}} stopping", name=self.name) except Exception as e: DEBUG and Log.warning("problem with thread {{name|quote}}", cause=e, name=self.name) finally: self.stopped.go() DEBUG and Log.note("thread {{name|quote}} is done", name=self.name) def is_alive(self): return not self.stopped def join(self, till=None): """ RETURN THE RESULT {"response":r, "exception":e} OF THE THREAD EXECUTION (INCLUDING EXCEPTION, IF EXISTS) """ if self is Thread: Log.error("Thread.join() is not a valid call, use t.join()") with self.child_lock: children = copy(self.children) for c in children: c.join(till=till) DEBUG and Log.note("{{parent|quote}} waiting on thread {{child|quote}}", parent=Thread.current().name, child=self.name) (self.stopped | till).wait() if self.stopped: self.parent.remove_child(self) if not self.end_of_thread.exception: return self.end_of_thread.response else: Log.error("Thread {{name|quote}} did not end well", name=self.name, cause=self.end_of_thread.exception) else: raise Except(context=THREAD_TIMEOUT) @staticmethod def run(name, target, *args, **kwargs): # ENSURE target HAS please_stop ARGUMENT if get_function_name(target) == 'wrapper': pass # GIVE THE override DECORATOR A PASS elif "please_stop" not in target.__code__.co_varnames: Log.error("function must have please_stop argument for signalling emergency shutdown") Thread.num_threads += 1 output = Thread(name, target, *args, **kwargs) output.start() return output @staticmethod def current(): ident = get_ident() with ALL_LOCK: output = ALL.get(ident) if output is None: thread = BaseThread(ident) thread.cprofiler = CProfiler() thread.cprofiler.__enter__() with ALL_LOCK: ALL[ident] = thread Log.warning("this thread is not known. Register this thread at earliest known entry point.") return thread return output
def __init__(self): BaseThread.__init__(self, get_ident()) self.name = "Main Thread" self.please_stop = Signal() self.stop_logging = Log.stop self.timers = None
class Thread(BaseThread): """ join() ENHANCED TO ALLOW CAPTURE OF CTRL-C, AND RETURN POSSIBLE THREAD EXCEPTIONS run() ENHANCED TO CAPTURE EXCEPTIONS """ num_threads = 0 def __init__(self, name, target, *args, **kwargs): BaseThread.__init__(self, -1) self.name = name self.target = target self.end_of_thread = Data() self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.kwargs["please_stop"] = self.kwargs.get("please_stop", Signal("please_stop for " + self.name)) self.please_stop = self.kwargs["please_stop"] self.thread = None self.stopped = Signal("stopped signal for " + self.name) if "parent_thread" in kwargs: del self.kwargs["parent_thread"] self.parent = kwargs["parent_thread"] else: self.parent = Thread.current() self.parent.add_child(self) def __enter__(self): return self def __exit__(self, type, value, traceback): if isinstance(type, BaseException): self.please_stop.go() # TODO: AFTER A WHILE START KILLING THREAD self.join() self.args = None self.kwargs = None def start(self): try: self.thread = start_new_thread(Thread._run, (self,)) return self except Exception as e: Log.error("Can not start thread", e) def stop(self): """ SEND STOP SIGNAL, DO NOT BLOCK """ with self.child_lock: children = copy(self.children) for c in children: DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name) c.stop() self.please_stop.go() DEBUG and Log.note("Thread {{name|quote}} got request to stop", name=self.name) def _run(self): self.id = get_ident() with RegisterThread(self): try: if self.target is not None: a, k, self.args, self.kwargs = self.args, self.kwargs, None, None self.end_of_thread.response = self.target(*a, **k) self.parent.remove_child(self) # IF THREAD ENDS OK, THEN FORGET ABOUT IT except Exception as e: e = Except.wrap(e) with self.synch_lock: self.end_of_thread.exception = e with self.parent.child_lock: emit_problem = self not in self.parent.children if emit_problem: # THREAD FAILURES ARE A PROBLEM ONLY IF NO ONE WILL BE JOINING WITH IT try: Log.fatal("Problem in thread {{name|quote}}", name=self.name, cause=e) except Exception: sys.stderr.write(str("ERROR in thread: " + self.name + " " + text_type(e) + "\n")) finally: try: with self.child_lock: children = copy(self.children) for c in children: try: DEBUG and sys.stdout.write(str("Stopping thread " + c.name + "\n")) c.stop() except Exception as e: Log.warning("Problem stopping thread {{thread}}", thread=c.name, cause=e) for c in children: try: DEBUG and sys.stdout.write(str("Joining on thread " + c.name + "\n")) c.join() except Exception as e: Log.warning("Problem joining thread {{thread}}", thread=c.name, cause=e) finally: DEBUG and sys.stdout.write(str("Joined on thread " + c.name + "\n")) del self.target, self.args, self.kwargs DEBUG and Log.note("thread {{name|quote}} stopping", name=self.name) except Exception as e: DEBUG and Log.warning("problem with thread {{name|quote}}", cause=e, name=self.name) finally: self.stopped.go() DEBUG and Log.note("thread {{name|quote}} is done", name=self.name) def is_alive(self): return not self.stopped def join(self, till=None): """ RETURN THE RESULT {"response":r, "exception":e} OF THE THREAD EXECUTION (INCLUDING EXCEPTION, IF EXISTS) """ if self is Thread: Log.error("Thread.join() is not a valid call, use t.join()") with self.child_lock: children = copy(self.children) for c in children: c.join(till=till) DEBUG and Log.note("{{parent|quote}} waiting on thread {{child|quote}}", parent=Thread.current().name, child=self.name) (self.stopped | till).wait() if self.stopped: self.parent.remove_child(self) if not self.end_of_thread.exception: return self.end_of_thread.response else: Log.error("Thread {{name|quote}} did not end well", name=self.name, cause=self.end_of_thread.exception) else: raise Except(type=THREAD_TIMEOUT) @staticmethod def run(name, target, *args, **kwargs): # ENSURE target HAS please_stop ARGUMENT if get_function_name(target) == 'wrapper': pass # GIVE THE override DECORATOR A PASS elif "please_stop" not in target.__code__.co_varnames: Log.error("function must have please_stop argument for signalling emergency shutdown") Thread.num_threads += 1 output = Thread(name, target, *args, **kwargs) output.start() return output @staticmethod def current(): ident = get_ident() with ALL_LOCK: output = ALL.get(ident) if output is None: Log.warning("this thread is not known. Register this thread at earliest known entry point.") return BaseThread(get_ident()) return output
class Thread(object): """ join() ENHANCED TO ALLOW CAPTURE OF CTRL-C, AND RETURN POSSIBLE THREAD EXCEPTIONS run() ENHANCED TO CAPTURE EXCEPTIONS """ num_threads = 0 def __init__(self, name, target, *args, **kwargs): self.id = -1 self.name = name self.target = target self.end_of_thread = None self.synch_lock = Lock("response synch lock") self.args = args # ENSURE THERE IS A SHARED please_stop SIGNAL self.kwargs = copy(kwargs) self.kwargs["please_stop"] = self.kwargs.get( "please_stop", Signal("please_stop for " + self.name)) self.please_stop = self.kwargs["please_stop"] self.thread = None self.stopped = Signal("stopped signal for " + self.name) self.cprofiler = Null self.children = [] if "parent_thread" in kwargs: del self.kwargs["parent_thread"] self.parent = kwargs["parent_thread"] else: self.parent = Thread.current() self.parent.add_child(self) def __enter__(self): return self def __exit__(self, type, value, traceback): if isinstance(type, BaseException): self.please_stop.go() # TODO: AFTER A WHILE START KILLING THREAD self.join() self.args = None self.kwargs = None def start(self): try: self.thread = start_new_thread(Thread._run, (self, )) return self except Exception as e: Log.error("Can not start thread", e) def stop(self): for c in copy(self.children): DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name) c.stop() self.please_stop.go() DEBUG and Log.note("Thread {{name|quote}} got request to stop", name=self.name) def add_child(self, child): self.children.append(child) def remove_child(self, child): try: self.children.remove(child) except Exception as e: # happens when multiple joins on same thread pass def _run(self): self.id = get_ident() with ALL_LOCK: ALL[self.id] = self try: if self.target is not None: a, k, self.args, self.kwargs = self.args, self.kwargs, None, None self.cprofiler = CProfiler() with self.cprofiler: # PROFILE IN HERE SO THAT __exit__() IS RUN BEFORE THREAD MARKED AS stopped response = self.target(*a, **k) with self.synch_lock: self.end_of_thread = Data(response=response) else: with self.synch_lock: self.end_of_thread = Null except Exception as e: e = Except.wrap(e) with self.synch_lock: self.end_of_thread = Data(exception=e) if self not in self.parent.children: # THREAD FAILURES ARE A PROBLEM ONLY IF NO ONE WILL BE JOINING WITH IT try: Log.fatal("Problem in thread {{name|quote}}", name=self.name, cause=e) except Exception: sys.stderr.write( str("ERROR in thread: " + self.name + " " + text_type(e) + "\n")) finally: try: children = copy(self.children) for c in children: try: if DEBUG: sys.stdout.write( str("Stopping thread " + c.name + "\n")) c.stop() except Exception as e: Log.warning("Problem stopping thread {{thread}}", thread=c.name, cause=e) for c in children: try: if DEBUG: sys.stdout.write( str("Joining on thread " + c.name + "\n")) c.join() except Exception as e: Log.warning("Problem joining thread {{thread}}", thread=c.name, cause=e) finally: if DEBUG: sys.stdout.write( str("Joined on thread " + c.name + "\n")) self.stopped.go() DEBUG and Log.note("thread {{name|quote}} stopping", name=self.name) del self.target, self.args, self.kwargs with ALL_LOCK: del ALL[self.id] except Exception as e: DEBUG and Log.warning("problem with thread {{name|quote}}", cause=e, name=self.name) finally: DEBUG and Log.note("thread {{name|quote}} is done", name=self.name) self.stopped.go() def is_alive(self): return not self.stopped def join(self, till=None): """ RETURN THE RESULT {"response":r, "exception":e} OF THE THREAD EXECUTION (INCLUDING EXCEPTION, IF EXISTS) """ if self is Thread: Log.error("Thread.join() is not a valid call, use t.join()") children = copy(self.children) for c in children: c.join(till=till) DEBUG and Log.note( "{{parent|quote}} waiting on thread {{child|quote}}", parent=Thread.current().name, child=self.name) (self.stopped | till).wait() if self.stopped: self.parent.remove_child(self) if not self.end_of_thread.exception: return self.end_of_thread.response else: Log.error("Thread {{name|quote}} did not end well", name=self.name, cause=self.end_of_thread.exception) else: raise Except(type=THREAD_TIMEOUT) @staticmethod def run(name, target, *args, **kwargs): # ENSURE target HAS please_stop ARGUMENT if get_function_name(target) == 'wrapper': pass # GIVE THE override DECORATOR A PASS elif "please_stop" not in target.__code__.co_varnames: Log.error( "function must have please_stop argument for signalling emergency shutdown" ) Thread.num_threads += 1 output = Thread(name, target, *args, **kwargs) output.start() return output @staticmethod def current(): id = get_ident() with ALL_LOCK: try: return ALL[id] except KeyError: return MAIN_THREAD
class Queue(object): """ SIMPLE MESSAGE QUEUE, multiprocessing.Queue REQUIRES SERIALIZATION, WHICH IS DIFFICULT TO USE JUST BETWEEN THREADS (SERIALIZATION REQUIRED) """ def __init__(self, name, max=None, silent=False, unique=False, allow_add_after_close=False): """ max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK silent - COMPLAIN IF THE READERS ARE TOO SLOW unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME """ self.name = name self.max = coalesce(max, 2**10) self.silent = silent self.allow_add_after_close = allow_add_after_close self.unique = unique self.closed = Signal( "stop adding signal for " + name) # INDICATE THE PRODUCER IS DONE GENERATING ITEMS TO QUEUE self.lock = Lock("lock for queue " + name) self.queue = deque() self.next_warning = time() # FOR DEBUGGING def __iter__(self): try: while True: value = self.pop() if value is THREAD_STOP: break if value is not None: yield value except Exception as e: Log.warning("Tell me about what happened here", e) def add(self, value, timeout=None, force=False): """ :param value: ADDED THE THE QUEUE :param timeout: HOW LONG TO WAIT FOR QUEUE TO NOT BE FULL :param force: ADD TO QUEUE, EVEN IF FULL (USE ONLY WHEN CONSUMER IS RETURNING WORK TO THE QUEUE) :return: self """ with self.lock: if value is THREAD_STOP: # INSIDE THE lock SO THAT EXITING WILL RELEASE wait() self.queue.append(value) self.closed.go() return if not force: self._wait_for_queue_space(timeout=timeout) if self.closed and not self.allow_add_after_close: Log.error("Do not add to closed queue") else: if self.unique: if value not in self.queue: self.queue.append(value) else: self.queue.append(value) return self def push(self, value): """ SNEAK value TO FRONT OF THE QUEUE """ if self.closed and not self.allow_add_after_close: Log.error("Do not push to closed queue") with self.lock: self._wait_for_queue_space() if not self.closed: self.queue.appendleft(value) return self def pop_message(self, till=None): """ RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE DUMMY IMPLEMENTATION FOR DEBUGGING """ if till is not None and not isinstance(till, Signal): Log.error("Expecting a signal") return Null, self.pop(till=till) def extend(self, values): if self.closed and not self.allow_add_after_close: Log.error("Do not push to closed queue") with self.lock: # ONCE THE queue IS BELOW LIMIT, ALLOW ADDING MORE self._wait_for_queue_space() if not self.closed: if self.unique: for v in values: if v is THREAD_STOP: self.closed.go() continue if v not in self.queue: self.queue.append(v) else: for v in values: if v is THREAD_STOP: self.closed.go() continue self.queue.append(v) return self def _wait_for_queue_space(self, timeout=DEFAULT_WAIT_TIME): """ EXPECT THE self.lock TO BE HAD, WAITS FOR self.queue TO HAVE A LITTLE SPACE """ wait_time = 5 (DEBUG and len(self.queue) > 1 * 1000 * 1000 ) and Log.warning("Queue {{name}} has over a million items") now = time() if timeout != None: time_to_stop_waiting = now + timeout else: time_to_stop_waiting = now + DEFAULT_WAIT_TIME if self.next_warning < now: self.next_warning = now + wait_time while not self.closed and len(self.queue) >= self.max: if now > time_to_stop_waiting: Log.error(THREAD_TIMEOUT) if self.silent: self.lock.wait(Till(till=time_to_stop_waiting)) else: self.lock.wait(Till(seconds=wait_time)) if len(self.queue) >= self.max: now = time() if self.next_warning < now: self.next_warning = now + wait_time Log.alert( "Queue by name of {{name|quote}} is full with ({{num}} items), thread(s) have been waiting {{wait_time}} sec", name=self.name, num=len(self.queue), wait_time=wait_time) def __len__(self): with self.lock: return len(self.queue) def __nonzero__(self): with self.lock: return any(r != THREAD_STOP for r in self.queue) def pop(self, till=None): """ WAIT FOR NEXT ITEM ON THE QUEUE RETURN THREAD_STOP IF QUEUE IS CLOSED RETURN None IF till IS REACHED AND QUEUE IS STILL EMPTY :param till: A `Signal` to stop waiting and return None :return: A value, or a THREAD_STOP or None """ if till is not None and not isinstance(till, Signal): Log.error("expecting a signal") with self.lock: while True: if self.queue: value = self.queue.popleft() return value if self.closed: break if not self.lock.wait(till=self.closed | till): if self.closed: break return None (DEBUG or not self.silent) and Log.note(self.name + " queue closed") return THREAD_STOP def pop_all(self): """ NON-BLOCKING POP ALL IN QUEUE, IF ANY """ with self.lock: output = list(self.queue) self.queue.clear() return output def pop_one(self): """ NON-BLOCKING POP IN QUEUE, IF ANY """ with self.lock: if self.closed: return [THREAD_STOP] elif not self.queue: return None else: v = self.queue.pop() if v is THREAD_STOP: # SENDING A STOP INTO THE QUEUE IS ALSO AN OPTION self.closed.go() return v def close(self): self.closed.go() def commit(self): pass def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close()
class Process(object): def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1): self.name = name self.service_stopped = Signal("stopped signal for " + strings.quote(name)) self.stdin = Queue("stdin for process " + strings.quote(name), silent=True) self.stdout = Queue("stdout for process " + strings.quote(name), silent=True) self.stderr = Queue("stderr for process " + strings.quote(name), silent=True) try: self.debug = debug or DEBUG self.service = service = subprocess.Popen( params, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=bufsize, cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath, env=unwrap(set_default(env, os.environ)), shell=shell) self.please_stop = Signal() self.please_stop.on_go(self._kill) self.thread_locker = Lock() self.children = [ Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self), Thread.run(self.name + " waiter", self._monitor, parent_thread=self), ] except Exception as e: Log.error("Can not call", e) if self.debug: Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params))) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.join(raise_on_error=True) def stop(self): self.stdin.add("exit") # ONE MORE SEND self.please_stop.go() def join(self, raise_on_error=False): self.service_stopped.wait() with self.thread_locker: child_threads, self.children = self.children, [] for c in child_threads: c.join() if raise_on_error and self.returncode != 0: Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}", process=self.name, code=self.service.returncode, stderr=list(self.stderr)) return self def remove_child(self, child): with self.thread_locker: try: self.children.remove(child) except Exception: pass @property def pid(self): return self.service.pid @property def returncode(self): return self.service.returncode def _monitor(self, please_stop): self.service.wait() if self.debug: Log.note("{{process}} STOP: returncode={{returncode}}", process=self.name, returncode=self.service.returncode) self.service_stopped.go() please_stop.go() def _reader(self, name, pipe, recieve, please_stop): try: line = "dummy" while not please_stop and self.service.returncode is None and line: line = pipe.readline().rstrip() if line: recieve.add(line) if self.debug: Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line) # GRAB A FEW MORE LINES max = 100 while max: try: line = pipe.readline().rstrip() if line: max = 100 recieve.add(line) if self.debug: Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line) else: max -= 1 except Exception: break finally: pipe.close() recieve.add(THREAD_STOP) def _writer(self, pipe, send, please_stop): while not please_stop: line = send.pop(till=please_stop) if line == THREAD_STOP: please_stop.go() break if line: if self.debug: Log.note("{{process}} (stdin): {{line}}", process=self.name, line=line.rstrip()) pipe.write(line + b"\n") pipe.close() def _kill(self): try: self.service.kill() except Exception as e: ee = Except.wrap(e) if 'The operation completed successfully' in ee: return if 'No such process' in ee: return Log.warning("Failure to kill process {{process|quote}}", process=self.name, cause=ee)
class MainThread(BaseThread): def __init__(self): BaseThread.__init__(self, get_ident()) self.name = "Main Thread" self.please_stop = Signal() self.stop_logging = Log.stop self.timers = None def stop(self): """ BLOCKS UNTIL ALL THREADS HAVE STOPPED THEN RUNS sys.exit(0) """ global DEBUG self_thread = Thread.current() if self_thread != MAIN_THREAD or self_thread != self: Log.error("Only the main thread can call stop() on main thread") DEBUG = True self.please_stop.go() join_errors = [] with self.child_lock: children = copy(self.children) for c in reversed(children): DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name) try: c.stop() except Exception as e: join_errors.append(e) for c in children: DEBUG and c.name and Log.note("Joining on thread {{name|quote}}", name=c.name) try: c.join() except Exception as e: join_errors.append(e) DEBUG and c.name and Log.note("Done join on thread {{name|quote}}", name=c.name) if join_errors: Log.error("Problem while stopping {{name|quote}}", name=self.name, cause=unwraplist(join_errors)) self.stop_logging() self.timers.stop() self.timers.join() write_profiles(self.cprofiler) DEBUG and Log.note("Thread {{name|quote}} now stopped", name=self.name) sys.exit(0) def wait_for_shutdown_signal( self, please_stop=False, # ASSIGN SIGNAL TO STOP EARLY allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False => IF NO CHILD THREADS LEFT, THEN EXIT ): """ FOR USE BY PROCESSES THAT NEVER DIE UNLESS EXTERNAL SHUTDOWN IS REQUESTED CALLING THREAD WILL SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit" :param please_stop: :param allow_exit: :param wait_forever:: Assume all needed threads have been launched. When done :return: """ self_thread = Thread.current() if self_thread != MAIN_THREAD or self_thread != self: Log.error("Only the main thread can sleep forever (waiting for KeyboardInterrupt)") if isinstance(please_stop, Signal): # MUTUAL TRIGGERING, SO THEY ARE EFFECTIVELY THE SAME self.please_stop.on_go(please_stop.go) please_stop.on_go(self.please_stop.go) else: please_stop = self.please_stop if not wait_forever: # TRIGGER SIGNAL WHEN ALL CHILDREN THEADS ARE DONE with self_thread.child_lock: pending = copy(self_thread.children) all = AndSignals(please_stop, len(pending)) for p in pending: p.stopped.on_go(all.done) try: if allow_exit: _wait_for_exit(please_stop) else: _wait_for_interrupt(please_stop) except KeyboardInterrupt as _: Log.alert("SIGINT Detected! Stopping...") except SystemExit as _: Log.alert("SIGTERM Detected! Stopping...") finally: self.stop()