def wait_for_shutdown_signal(
please_stop=False, # ASSIGN SIGNAL TO STOP EARLY
allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP
wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False -> IF NO CHILD THREADS LEFT, THEN EXIT
):
"""
FOR USE BY PROCESSES NOT EXPECTED TO EVER COMPLETE UNTIL EXTERNAL
SHUTDOWN IS REQUESTED
SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit"
:param please_stop:
:param allow_exit:
:param wait_forever:: Assume all needed threads have been launched. When done
:return:
"""
if not isinstance(please_stop, Signal):
please_stop = Signal()
please_stop.on_go(
lambda: thread.start_new_thread(_stop_main_thread, ()))
self_thread = Thread.current()
if self_thread != MAIN_THREAD:
Log.error(
"Only the main thread can sleep forever (waiting for KeyboardInterrupt)"
)
if not wait_forever:
# TRIGGER SIGNAL WHEN ALL EXITING THREADS ARE DONE
pending = copy(self_thread.children)
all = AndSignals(please_stop, len(pending))
for p in pending:
p.stopped.on_go(all.done)
try:
if allow_exit:
_wait_for_exit(please_stop)
else:
_wait_for_interrupt(please_stop)
except (KeyboardInterrupt, SystemExit), _:
Log.alert("SIGINT Detected! Stopping...")
class Process(object):
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell
)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
if self.debug:
Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.join(raise_on_error=True)
def stop(self):
self.stdin.add("exit") # ONE MORE SEND
self.please_stop.go()
def join(self, raise_on_error=False):
self.service_stopped.wait()
with self.thread_locker:
child_threads, self.children = self.children, []
for c in child_threads:
c.join()
if raise_on_error and self.returncode != 0:
Log.error(
"{{process}} FAIL: returncode={{code}}\n{{stderr}}",
process=self.name,
code=self.service.returncode,
stderr=list(self.stderr)
)
return self
def remove_child(self, child):
with self.thread_locker:
try:
self.children.remove(child)
except Exception:
pass
@property
def pid(self):
return self.service.pid
@property
def returncode(self):
return self.service.returncode
def _monitor(self, please_stop):
self.service.wait()
if self.debug:
Log.note("{{process}} STOP: returncode={{returncode}}", process=self.name, returncode=self.service.returncode)
self.service_stopped.go()
please_stop.go()
def _reader(self, name, pipe, recieve, please_stop):
try:
line = "dummy"
while not please_stop and self.service.returncode is None and line:
line = pipe.readline().rstrip()
if line:
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
# GRAB A FEW MORE LINES
max = 100
while max:
try:
line = pipe.readline().rstrip()
if line:
max = 100
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
else:
max -= 1
except Exception:
break
finally:
pipe.close()
recieve.add(THREAD_STOP)
def _writer(self, pipe, send, please_stop):
while not please_stop:
line = send.pop(till=please_stop)
if line == THREAD_STOP:
please_stop.go()
break
if line:
if self.debug:
Log.note("{{process}} (stdin): {{line}}", process=self.name, line=line.rstrip())
pipe.write(line + b"\n")
pipe.close()
def _kill(self):
try:
self.service.kill()
except Exception as e:
ee = Except.wrap(e)
if 'The operation completed successfully' in ee:
return
if 'No such process' in ee:
return
Log.warning("Failure to kill process {{process|quote}}", process=self.name, cause=ee)
class MainThread(BaseThread):
def __init__(self):
BaseThread.__init__(self, get_ident())
self.name = "Main Thread"
self.please_stop = Signal()
self.stop_logging = Log.stop
self.timers = None
def stop(self):
"""
BLOCKS UNTIL ALL THREADS HAVE STOPPED
THEN RUNS sys.exit(0)
"""
global DEBUG
self_thread = Thread.current()
if self_thread != MAIN_THREAD or self_thread != self:
Log.error("Only the main thread can call stop() on main thread")
DEBUG = True
self.please_stop.go()
join_errors = []
with self.child_lock:
children = copy(self.children)
for c in reversed(children):
DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name)
try:
c.stop()
except Exception as e:
join_errors.append(e)
for c in children:
DEBUG and c.name and Log.note("Joining on thread {{name|quote}}", name=c.name)
try:
c.join()
except Exception as e:
join_errors.append(e)
DEBUG and c.name and Log.note("Done join on thread {{name|quote}}", name=c.name)
if join_errors:
Log.error("Problem while stopping {{name|quote}}", name=self.name, cause=unwraplist(join_errors))
self.stop_logging()
self.timers.stop()
self.timers.join()
write_profiles(self.cprofiler)
DEBUG and Log.note("Thread {{name|quote}} now stopped", name=self.name)
sys.exit()
def wait_for_shutdown_signal(
self,
please_stop=False, # ASSIGN SIGNAL TO STOP EARLY
allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP
wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False => IF NO CHILD THREADS LEFT, THEN EXIT
):
"""
FOR USE BY PROCESSES THAT NEVER DIE UNLESS EXTERNAL SHUTDOWN IS REQUESTED
CALLING THREAD WILL SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit"
:param please_stop:
:param allow_exit:
:param wait_forever:: Assume all needed threads have been launched. When done
:return:
"""
self_thread = Thread.current()
if self_thread != MAIN_THREAD or self_thread != self:
Log.error("Only the main thread can sleep forever (waiting for KeyboardInterrupt)")
if isinstance(please_stop, Signal):
# MUTUAL SIGNALING MAKES THESE TWO EFFECTIVELY THE SAME SIGNAL
self.please_stop.on_go(please_stop.go)
please_stop.on_go(self.please_stop.go)
else:
please_stop = self.please_stop
if not wait_forever:
# TRIGGER SIGNAL WHEN ALL CHILDREN THEADS ARE DONE
with self_thread.child_lock:
pending = copy(self_thread.children)
children_done = AndSignals(please_stop, len(pending))
children_done.signal.on_go(self.please_stop.go)
for p in pending:
p.stopped.on_go(children_done.done)
try:
if allow_exit:
_wait_for_exit(please_stop)
else:
_wait_for_interrupt(please_stop)
except KeyboardInterrupt as _:
Log.alert("SIGINT Detected! Stopping...")
except SystemExit as _:
Log.alert("SIGTERM Detected! Stopping...")
finally:
self.stop()
class Process(object):
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name),
silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name),
silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name),
silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType,
NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
if self.debug:
Log.note("{{process}} START: {{command}}",
process=self.name,
command=" ".join(map(strings.quote, params)))
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.join(raise_on_error=True)
def stop(self):
self.stdin.add("exit") # ONE MORE SEND
self.please_stop.go()
def join(self, raise_on_error=False):
self.service_stopped.wait()
with self.thread_locker:
child_threads, self.children = self.children, []
for c in child_threads:
c.join()
if raise_on_error and self.returncode != 0:
Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}",
process=self.name,
code=self.service.returncode,
stderr=list(self.stderr))
return self
def remove_child(self, child):
with self.thread_locker:
try:
self.children.remove(child)
except Exception:
pass
@property
def pid(self):
return self.service.pid
@property
def returncode(self):
return self.service.returncode
def _monitor(self, please_stop):
self.service.wait()
if self.debug:
Log.note("{{process}} STOP: returncode={{returncode}}",
process=self.name,
returncode=self.service.returncode)
self.service_stopped.go()
please_stop.go()
def _reader(self, name, pipe, recieve, please_stop):
try:
line = "dummy"
while not please_stop and self.service.returncode is None and line:
line = pipe.readline().rstrip()
if line:
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
# GRAB A FEW MORE LINES
max = 100
while max:
try:
line = pipe.readline().rstrip()
if line:
max = 100
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
else:
max -= 1
except Exception:
break
finally:
pipe.close()
recieve.add(THREAD_STOP)
def _writer(self, pipe, send, please_stop):
while not please_stop:
line = send.pop(till=please_stop)
if line == THREAD_STOP:
please_stop.go()
break
if line:
if self.debug:
Log.note("{{process}} (stdin): {{line}}",
process=self.name,
line=line.rstrip())
pipe.write(line + b"\n")
pipe.close()
def _kill(self):
try:
self.service.kill()
except Exception as e:
ee = Except.wrap(e)
if 'The operation completed successfully' in ee:
return
if 'No such process' in ee:
return
Log.warning("Failure to kill process {{process|quote}}",
process=self.name,
cause=ee)
class MainThread(BaseThread):
def __init__(self):
BaseThread.__init__(self, get_ident())
self.name = "Main Thread"
self.please_stop = Signal()
self.stop_logging = Log.stop
self.timers = None
def stop(self):
"""
BLOCKS UNTIL ALL THREADS HAVE STOPPED
THEN RUNS sys.exit(0)
"""
global DEBUG
self_thread = Thread.current()
if self_thread != MAIN_THREAD or self_thread != self:
Log.error("Only the main thread can call stop() on main thread")
DEBUG = True
self.please_stop.go()
join_errors = []
with self.child_lock:
children = copy(self.children)
for c in reversed(children):
DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name)
try:
c.stop()
except Exception as e:
join_errors.append(e)
for c in children:
DEBUG and c.name and Log.note("Joining on thread {{name|quote}}", name=c.name)
try:
c.join()
except Exception as e:
join_errors.append(e)
DEBUG and c.name and Log.note("Done join on thread {{name|quote}}", name=c.name)
if join_errors:
Log.error("Problem while stopping {{name|quote}}", name=self.name, cause=unwraplist(join_errors))
self.stop_logging()
self.timers.stop()
self.timers.join()
write_profiles(self.cprofiler)
DEBUG and Log.note("Thread {{name|quote}} now stopped", name=self.name)
sys.exit(0)
def wait_for_shutdown_signal(
self,
please_stop=False, # ASSIGN SIGNAL TO STOP EARLY
allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP
wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False => IF NO CHILD THREADS LEFT, THEN EXIT
):
"""
FOR USE BY PROCESSES THAT NEVER DIE UNLESS EXTERNAL SHUTDOWN IS REQUESTED
CALLING THREAD WILL SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit"
:param please_stop:
:param allow_exit:
:param wait_forever:: Assume all needed threads have been launched. When done
:return:
"""
self_thread = Thread.current()
if self_thread != MAIN_THREAD or self_thread != self:
Log.error("Only the main thread can sleep forever (waiting for KeyboardInterrupt)")
if isinstance(please_stop, Signal):
# MUTUAL TRIGGERING, SO THEY ARE EFFECTIVELY THE SAME
self.please_stop.on_go(please_stop.go)
please_stop.on_go(self.please_stop.go)
else:
please_stop = self.please_stop
if not wait_forever:
# TRIGGER SIGNAL WHEN ALL CHILDREN THEADS ARE DONE
with self_thread.child_lock:
pending = copy(self_thread.children)
all = AndSignals(please_stop, len(pending))
for p in pending:
p.stopped.on_go(all.done)
try:
if allow_exit:
_wait_for_exit(please_stop)
else:
_wait_for_interrupt(please_stop)
except KeyboardInterrupt as _:
Log.alert("SIGINT Detected! Stopping...")
except SystemExit as _:
Log.alert("SIGTERM Detected! Stopping...")
finally:
self.stop()
class Process(object):
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
string2quote(name))
self.stdin = Queue("stdin for process " + string2quote(name),
silent=True)
self.stdout = Queue("stdout for process " + string2quote(name),
silent=True)
self.stderr = Queue("stderr for process " + string2quote(name),
silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType,
NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception, e:
Log.error("Can not call", e)
if self.debug:
Log.note("{{process}} START: {{command}}",
process=self.name,
command=" ".join(map(strings.quote, params)))