class Process(object):
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell
)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
if self.debug:
Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.join(raise_on_error=True)
def stop(self):
self.stdin.add("exit") # ONE MORE SEND
self.please_stop.go()
def join(self, raise_on_error=False):
self.service_stopped.wait()
with self.thread_locker:
child_threads, self.children = self.children, []
for c in child_threads:
c.join()
if raise_on_error and self.returncode != 0:
Log.error(
"{{process}} FAIL: returncode={{code}}\n{{stderr}}",
process=self.name,
code=self.service.returncode,
stderr=list(self.stderr)
)
return self
def remove_child(self, child):
with self.thread_locker:
try:
self.children.remove(child)
except Exception:
pass
@property
def pid(self):
return self.service.pid
@property
def returncode(self):
return self.service.returncode
def _monitor(self, please_stop):
self.service.wait()
if self.debug:
Log.note("{{process}} STOP: returncode={{returncode}}", process=self.name, returncode=self.service.returncode)
self.service_stopped.go()
please_stop.go()
def _reader(self, name, pipe, recieve, please_stop):
try:
line = "dummy"
while not please_stop and self.service.returncode is None and line:
line = pipe.readline().rstrip()
if line:
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
# GRAB A FEW MORE LINES
max = 100
while max:
try:
line = pipe.readline().rstrip()
if line:
max = 100
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
else:
max -= 1
except Exception:
break
finally:
pipe.close()
recieve.add(THREAD_STOP)
def _writer(self, pipe, send, please_stop):
while not please_stop:
line = send.pop(till=please_stop)
if line == THREAD_STOP:
please_stop.go()
break
if line:
if self.debug:
Log.note("{{process}} (stdin): {{line}}", process=self.name, line=line.rstrip())
pipe.write(line + b"\n")
pipe.close()
def _kill(self):
try:
self.service.kill()
except Exception as e:
ee = Except.wrap(e)
if 'The operation completed successfully' in ee:
return
if 'No such process' in ee:
return
Log.warning("Failure to kill process {{process|quote}}", process=self.name, cause=ee)
class Thread(BaseThread):
"""
join() ENHANCED TO ALLOW CAPTURE OF CTRL-C, AND RETURN POSSIBLE THREAD EXCEPTIONS
run() ENHANCED TO CAPTURE EXCEPTIONS
"""
num_threads = 0
def __init__(self, name, target, *args, **kwargs):
BaseThread.__init__(self, -1)
self.name = coalesce(name, "thread_" + text_type(object.__hash__(self)))
self.target = target
self.end_of_thread = Data()
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get("please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if isinstance(type, BaseException):
self.please_stop.go()
# TODO: AFTER A WHILE START KILLING THREAD
self.join()
self.args = None
self.kwargs = None
def start(self):
try:
self.thread = start_new_thread(Thread._run, (self,))
return self
except Exception as e:
Log.error("Can not start thread", e)
def stop(self):
"""
SEND STOP SIGNAL, DO NOT BLOCK
"""
with self.child_lock:
children = copy(self.children)
for c in children:
DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name)
c.stop()
self.please_stop.go()
DEBUG and Log.note("Thread {{name|quote}} got request to stop", name=self.name)
def _run(self):
self.id = get_ident()
with RegisterThread(self):
try:
if self.target is not None:
a, k, self.args, self.kwargs = self.args, self.kwargs, None, None
self.end_of_thread.response = self.target(*a, **k)
self.parent.remove_child(self) # IF THREAD ENDS OK, THEN FORGET ABOUT IT
except Exception as e:
e = Except.wrap(e)
with self.synch_lock:
self.end_of_thread.exception = e
with self.parent.child_lock:
emit_problem = self not in self.parent.children
if emit_problem:
# THREAD FAILURES ARE A PROBLEM ONLY IF NO ONE WILL BE JOINING WITH IT
try:
Log.fatal("Problem in thread {{name|quote}}", name=self.name, cause=e)
except Exception:
sys.stderr.write(str("ERROR in thread: " + self.name + " " + text_type(e) + "\n"))
finally:
try:
with self.child_lock:
children = copy(self.children)
for c in children:
try:
DEBUG and sys.stdout.write(str("Stopping thread " + c.name + "\n"))
c.stop()
except Exception as e:
Log.warning("Problem stopping thread {{thread}}", thread=c.name, cause=e)
for c in children:
try:
DEBUG and sys.stdout.write(str("Joining on thread " + c.name + "\n"))
c.join()
except Exception as e:
Log.warning("Problem joining thread {{thread}}", thread=c.name, cause=e)
finally:
DEBUG and sys.stdout.write(str("Joined on thread " + c.name + "\n"))
del self.target, self.args, self.kwargs
DEBUG and Log.note("thread {{name|quote}} stopping", name=self.name)
except Exception as e:
DEBUG and Log.warning("problem with thread {{name|quote}}", cause=e, name=self.name)
finally:
self.stopped.go()
DEBUG and Log.note("thread {{name|quote}} is done", name=self.name)
def is_alive(self):
return not self.stopped
def join(self, till=None):
"""
RETURN THE RESULT {"response":r, "exception":e} OF THE THREAD EXECUTION (INCLUDING EXCEPTION, IF EXISTS)
"""
if self is Thread:
Log.error("Thread.join() is not a valid call, use t.join()")
with self.child_lock:
children = copy(self.children)
for c in children:
c.join(till=till)
DEBUG and Log.note("{{parent|quote}} waiting on thread {{child|quote}}", parent=Thread.current().name, child=self.name)
(self.stopped | till).wait()
if self.stopped:
self.parent.remove_child(self)
if not self.end_of_thread.exception:
return self.end_of_thread.response
else:
Log.error("Thread {{name|quote}} did not end well", name=self.name, cause=self.end_of_thread.exception)
else:
raise Except(context=THREAD_TIMEOUT)
@staticmethod
def run(name, target, *args, **kwargs):
# ENSURE target HAS please_stop ARGUMENT
if get_function_name(target) == 'wrapper':
pass # GIVE THE override DECORATOR A PASS
elif "please_stop" not in target.__code__.co_varnames:
Log.error("function must have please_stop argument for signalling emergency shutdown")
Thread.num_threads += 1
output = Thread(name, target, *args, **kwargs)
output.start()
return output
@staticmethod
def current():
ident = get_ident()
with ALL_LOCK:
output = ALL.get(ident)
if output is None:
thread = BaseThread(ident)
thread.cprofiler = CProfiler()
thread.cprofiler.__enter__()
with ALL_LOCK:
ALL[ident] = thread
Log.warning("this thread is not known. Register this thread at earliest known entry point.")
return thread
return output
class MainThread(BaseThread):
def __init__(self):
BaseThread.__init__(self, get_ident())
self.name = "Main Thread"
self.please_stop = Signal()
self.stop_logging = Log.stop
self.timers = None
def stop(self):
"""
BLOCKS UNTIL ALL THREADS HAVE STOPPED
THEN RUNS sys.exit(0)
"""
global DEBUG
self_thread = Thread.current()
if self_thread != MAIN_THREAD or self_thread != self:
Log.error("Only the main thread can call stop() on main thread")
DEBUG = True
self.please_stop.go()
join_errors = []
with self.child_lock:
children = copy(self.children)
for c in reversed(children):
DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name)
try:
c.stop()
except Exception as e:
join_errors.append(e)
for c in children:
DEBUG and c.name and Log.note("Joining on thread {{name|quote}}", name=c.name)
try:
c.join()
except Exception as e:
join_errors.append(e)
DEBUG and c.name and Log.note("Done join on thread {{name|quote}}", name=c.name)
if join_errors:
Log.error("Problem while stopping {{name|quote}}", name=self.name, cause=unwraplist(join_errors))
self.stop_logging()
self.timers.stop()
self.timers.join()
write_profiles(self.cprofiler)
DEBUG and Log.note("Thread {{name|quote}} now stopped", name=self.name)
sys.exit()
def wait_for_shutdown_signal(
self,
please_stop=False, # ASSIGN SIGNAL TO STOP EARLY
allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP
wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False => IF NO CHILD THREADS LEFT, THEN EXIT
):
"""
FOR USE BY PROCESSES THAT NEVER DIE UNLESS EXTERNAL SHUTDOWN IS REQUESTED
CALLING THREAD WILL SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit"
:param please_stop:
:param allow_exit:
:param wait_forever:: Assume all needed threads have been launched. When done
:return:
"""
self_thread = Thread.current()
if self_thread != MAIN_THREAD or self_thread != self:
Log.error("Only the main thread can sleep forever (waiting for KeyboardInterrupt)")
if isinstance(please_stop, Signal):
# MUTUAL SIGNALING MAKES THESE TWO EFFECTIVELY THE SAME SIGNAL
self.please_stop.on_go(please_stop.go)
please_stop.on_go(self.please_stop.go)
else:
please_stop = self.please_stop
if not wait_forever:
# TRIGGER SIGNAL WHEN ALL CHILDREN THEADS ARE DONE
with self_thread.child_lock:
pending = copy(self_thread.children)
children_done = AndSignals(please_stop, len(pending))
children_done.signal.on_go(self.please_stop.go)
for p in pending:
p.stopped.on_go(children_done.done)
try:
if allow_exit:
_wait_for_exit(please_stop)
else:
_wait_for_interrupt(please_stop)
except KeyboardInterrupt as _:
Log.alert("SIGINT Detected! Stopping...")
except SystemExit as _:
Log.alert("SIGTERM Detected! Stopping...")
finally:
self.stop()
class Queue(object):
"""
SIMPLE MESSAGE QUEUE, multiprocessing.Queue REQUIRES SERIALIZATION, WHICH
IS DIFFICULT TO USE JUST BETWEEN THREADS (SERIALIZATION REQUIRED)
"""
def __init__(self,
name,
max=None,
silent=False,
unique=False,
allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
self.name = name
self.max = coalesce(max, 2**10)
self.silent = silent
self.allow_add_after_close = allow_add_after_close
self.unique = unique
self.closed = Signal(
"stop adding signal for " +
name) # INDICATE THE PRODUCER IS DONE GENERATING ITEMS TO QUEUE
self.lock = Lock("lock for queue " + name)
self.queue = deque()
self.next_warning = time() # FOR DEBUGGING
def __iter__(self):
try:
while True:
value = self.pop()
if value is THREAD_STOP:
break
if value is not None:
yield value
except Exception as e:
Log.warning("Tell me about what happened here", e)
def add(self, value, timeout=None, force=False):
"""
:param value: ADDED THE THE QUEUE
:param timeout: HOW LONG TO WAIT FOR QUEUE TO NOT BE FULL
:param force: ADD TO QUEUE, EVEN IF FULL (USE ONLY WHEN CONSUMER IS RETURNING WORK TO THE QUEUE)
:return: self
"""
with self.lock:
if value is THREAD_STOP:
# INSIDE THE lock SO THAT EXITING WILL RELEASE wait()
self.queue.append(value)
self.closed.go()
return
if not force:
self._wait_for_queue_space(timeout=timeout)
if self.closed and not self.allow_add_after_close:
Log.error("Do not add to closed queue")
else:
if self.unique:
if value not in self.queue:
self.queue.append(value)
else:
self.queue.append(value)
return self
def push(self, value):
"""
SNEAK value TO FRONT OF THE QUEUE
"""
if self.closed and not self.allow_add_after_close:
Log.error("Do not push to closed queue")
with self.lock:
self._wait_for_queue_space()
if not self.closed:
self.queue.appendleft(value)
return self
def pop_message(self, till=None):
"""
RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE
DUMMY IMPLEMENTATION FOR DEBUGGING
"""
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
return Null, self.pop(till=till)
def extend(self, values):
if self.closed and not self.allow_add_after_close:
Log.error("Do not push to closed queue")
with self.lock:
# ONCE THE queue IS BELOW LIMIT, ALLOW ADDING MORE
self._wait_for_queue_space()
if not self.closed:
if self.unique:
for v in values:
if v is THREAD_STOP:
self.closed.go()
continue
if v not in self.queue:
self.queue.append(v)
else:
for v in values:
if v is THREAD_STOP:
self.closed.go()
continue
self.queue.append(v)
return self
def _wait_for_queue_space(self, timeout=DEFAULT_WAIT_TIME):
"""
EXPECT THE self.lock TO BE HAD, WAITS FOR self.queue TO HAVE A LITTLE SPACE
"""
wait_time = 5
(DEBUG and len(self.queue) > 1 * 1000 * 1000
) and Log.warning("Queue {{name}} has over a million items")
now = time()
if timeout != None:
time_to_stop_waiting = now + timeout
else:
time_to_stop_waiting = now + DEFAULT_WAIT_TIME
if self.next_warning < now:
self.next_warning = now + wait_time
while not self.closed and len(self.queue) >= self.max:
if now > time_to_stop_waiting:
Log.error(THREAD_TIMEOUT)
if self.silent:
self.lock.wait(Till(till=time_to_stop_waiting))
else:
self.lock.wait(Till(seconds=wait_time))
if len(self.queue) >= self.max:
now = time()
if self.next_warning < now:
self.next_warning = now + wait_time
Log.alert(
"Queue by name of {{name|quote}} is full with ({{num}} items), thread(s) have been waiting {{wait_time}} sec",
name=self.name,
num=len(self.queue),
wait_time=wait_time)
def __len__(self):
with self.lock:
return len(self.queue)
def __nonzero__(self):
with self.lock:
return any(r != THREAD_STOP for r in self.queue)
def pop(self, till=None):
"""
WAIT FOR NEXT ITEM ON THE QUEUE
RETURN THREAD_STOP IF QUEUE IS CLOSED
RETURN None IF till IS REACHED AND QUEUE IS STILL EMPTY
:param till: A `Signal` to stop waiting and return None
:return: A value, or a THREAD_STOP or None
"""
if till is not None and not isinstance(till, Signal):
Log.error("expecting a signal")
with self.lock:
while True:
if self.queue:
value = self.queue.popleft()
return value
if self.closed:
break
if not self.lock.wait(till=self.closed | till):
if self.closed:
break
return None
(DEBUG or not self.silent) and Log.note(self.name + " queue closed")
return THREAD_STOP
def pop_all(self):
"""
NON-BLOCKING POP ALL IN QUEUE, IF ANY
"""
with self.lock:
output = list(self.queue)
self.queue.clear()
return output
def pop_one(self):
"""
NON-BLOCKING POP IN QUEUE, IF ANY
"""
with self.lock:
if self.closed:
return [THREAD_STOP]
elif not self.queue:
return None
else:
v = self.queue.pop()
if v is THREAD_STOP: # SENDING A STOP INTO THE QUEUE IS ALSO AN OPTION
self.closed.go()
return v
def close(self):
self.closed.go()
def commit(self):
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
class Thread(object):
"""
join() ENHANCED TO ALLOW CAPTURE OF CTRL-C, AND RETURN POSSIBLE THREAD EXCEPTIONS
run() ENHANCED TO CAPTURE EXCEPTIONS
"""
num_threads = 0
def __init__(self, name, target, *args, **kwargs):
self.id = -1
self.name = name
self.target = target
self.end_of_thread = None
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get(
"please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
self.cprofiler = Null
self.children = []
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if isinstance(type, BaseException):
self.please_stop.go()
# TODO: AFTER A WHILE START KILLING THREAD
self.join()
self.args = None
self.kwargs = None
def start(self):
try:
self.thread = start_new_thread(Thread._run, (self, ))
return self
except Exception as e:
Log.error("Can not start thread", e)
def stop(self):
for c in copy(self.children):
DEBUG and c.name and Log.note("Stopping thread {{name|quote}}",
name=c.name)
c.stop()
self.please_stop.go()
DEBUG and Log.note("Thread {{name|quote}} got request to stop",
name=self.name)
def add_child(self, child):
self.children.append(child)
def remove_child(self, child):
try:
self.children.remove(child)
except Exception as e:
# happens when multiple joins on same thread
pass
def _run(self):
self.id = get_ident()
with ALL_LOCK:
ALL[self.id] = self
try:
if self.target is not None:
a, k, self.args, self.kwargs = self.args, self.kwargs, None, None
self.cprofiler = CProfiler()
with self.cprofiler: # PROFILE IN HERE SO THAT __exit__() IS RUN BEFORE THREAD MARKED AS stopped
response = self.target(*a, **k)
with self.synch_lock:
self.end_of_thread = Data(response=response)
else:
with self.synch_lock:
self.end_of_thread = Null
except Exception as e:
e = Except.wrap(e)
with self.synch_lock:
self.end_of_thread = Data(exception=e)
if self not in self.parent.children:
# THREAD FAILURES ARE A PROBLEM ONLY IF NO ONE WILL BE JOINING WITH IT
try:
Log.fatal("Problem in thread {{name|quote}}",
name=self.name,
cause=e)
except Exception:
sys.stderr.write(
str("ERROR in thread: " + self.name + " " +
text_type(e) + "\n"))
finally:
try:
children = copy(self.children)
for c in children:
try:
if DEBUG:
sys.stdout.write(
str("Stopping thread " + c.name + "\n"))
c.stop()
except Exception as e:
Log.warning("Problem stopping thread {{thread}}",
thread=c.name,
cause=e)
for c in children:
try:
if DEBUG:
sys.stdout.write(
str("Joining on thread " + c.name + "\n"))
c.join()
except Exception as e:
Log.warning("Problem joining thread {{thread}}",
thread=c.name,
cause=e)
finally:
if DEBUG:
sys.stdout.write(
str("Joined on thread " + c.name + "\n"))
self.stopped.go()
DEBUG and Log.note("thread {{name|quote}} stopping",
name=self.name)
del self.target, self.args, self.kwargs
with ALL_LOCK:
del ALL[self.id]
except Exception as e:
DEBUG and Log.warning("problem with thread {{name|quote}}",
cause=e,
name=self.name)
finally:
DEBUG and Log.note("thread {{name|quote}} is done",
name=self.name)
self.stopped.go()
def is_alive(self):
return not self.stopped
def join(self, till=None):
"""
RETURN THE RESULT {"response":r, "exception":e} OF THE THREAD EXECUTION (INCLUDING EXCEPTION, IF EXISTS)
"""
if self is Thread:
Log.error("Thread.join() is not a valid call, use t.join()")
children = copy(self.children)
for c in children:
c.join(till=till)
DEBUG and Log.note(
"{{parent|quote}} waiting on thread {{child|quote}}",
parent=Thread.current().name,
child=self.name)
(self.stopped | till).wait()
if self.stopped:
self.parent.remove_child(self)
if not self.end_of_thread.exception:
return self.end_of_thread.response
else:
Log.error("Thread {{name|quote}} did not end well",
name=self.name,
cause=self.end_of_thread.exception)
else:
raise Except(type=THREAD_TIMEOUT)
@staticmethod
def run(name, target, *args, **kwargs):
# ENSURE target HAS please_stop ARGUMENT
if get_function_name(target) == 'wrapper':
pass # GIVE THE override DECORATOR A PASS
elif "please_stop" not in target.__code__.co_varnames:
Log.error(
"function must have please_stop argument for signalling emergency shutdown"
)
Thread.num_threads += 1
output = Thread(name, target, *args, **kwargs)
output.start()
return output
@staticmethod
def current():
id = get_ident()
with ALL_LOCK:
try:
return ALL[id]
except KeyError:
return MAIN_THREAD
class Process(object):
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name),
silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name),
silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name),
silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType,
NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
if self.debug:
Log.note("{{process}} START: {{command}}",
process=self.name,
command=" ".join(map(strings.quote, params)))
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.join(raise_on_error=True)
def stop(self):
self.stdin.add("exit") # ONE MORE SEND
self.please_stop.go()
def join(self, raise_on_error=False):
self.service_stopped.wait()
with self.thread_locker:
child_threads, self.children = self.children, []
for c in child_threads:
c.join()
if raise_on_error and self.returncode != 0:
Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}",
process=self.name,
code=self.service.returncode,
stderr=list(self.stderr))
return self
def remove_child(self, child):
with self.thread_locker:
try:
self.children.remove(child)
except Exception:
pass
@property
def pid(self):
return self.service.pid
@property
def returncode(self):
return self.service.returncode
def _monitor(self, please_stop):
self.service.wait()
if self.debug:
Log.note("{{process}} STOP: returncode={{returncode}}",
process=self.name,
returncode=self.service.returncode)
self.service_stopped.go()
please_stop.go()
def _reader(self, name, pipe, recieve, please_stop):
try:
line = "dummy"
while not please_stop and self.service.returncode is None and line:
line = pipe.readline().rstrip()
if line:
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
# GRAB A FEW MORE LINES
max = 100
while max:
try:
line = pipe.readline().rstrip()
if line:
max = 100
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
else:
max -= 1
except Exception:
break
finally:
pipe.close()
recieve.add(THREAD_STOP)
def _writer(self, pipe, send, please_stop):
while not please_stop:
line = send.pop(till=please_stop)
if line == THREAD_STOP:
please_stop.go()
break
if line:
if self.debug:
Log.note("{{process}} (stdin): {{line}}",
process=self.name,
line=line.rstrip())
pipe.write(line + b"\n")
pipe.close()
def _kill(self):
try:
self.service.kill()
except Exception as e:
ee = Except.wrap(e)
if 'The operation completed successfully' in ee:
return
if 'No such process' in ee:
return
Log.warning("Failure to kill process {{process|quote}}",
process=self.name,
cause=ee)
class MainThread(BaseThread):
def __init__(self):
BaseThread.__init__(self, get_ident())
self.name = "Main Thread"
self.please_stop = Signal()
self.stop_logging = Log.stop
self.timers = None
def stop(self):
"""
BLOCKS UNTIL ALL THREADS HAVE STOPPED
THEN RUNS sys.exit(0)
"""
global DEBUG
self_thread = Thread.current()
if self_thread != MAIN_THREAD or self_thread != self:
Log.error("Only the main thread can call stop() on main thread")
DEBUG = True
self.please_stop.go()
join_errors = []
with self.child_lock:
children = copy(self.children)
for c in reversed(children):
DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name)
try:
c.stop()
except Exception as e:
join_errors.append(e)
for c in children:
DEBUG and c.name and Log.note("Joining on thread {{name|quote}}", name=c.name)
try:
c.join()
except Exception as e:
join_errors.append(e)
DEBUG and c.name and Log.note("Done join on thread {{name|quote}}", name=c.name)
if join_errors:
Log.error("Problem while stopping {{name|quote}}", name=self.name, cause=unwraplist(join_errors))
self.stop_logging()
self.timers.stop()
self.timers.join()
write_profiles(self.cprofiler)
DEBUG and Log.note("Thread {{name|quote}} now stopped", name=self.name)
sys.exit(0)
def wait_for_shutdown_signal(
self,
please_stop=False, # ASSIGN SIGNAL TO STOP EARLY
allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP
wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False => IF NO CHILD THREADS LEFT, THEN EXIT
):
"""
FOR USE BY PROCESSES THAT NEVER DIE UNLESS EXTERNAL SHUTDOWN IS REQUESTED
CALLING THREAD WILL SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit"
:param please_stop:
:param allow_exit:
:param wait_forever:: Assume all needed threads have been launched. When done
:return:
"""
self_thread = Thread.current()
if self_thread != MAIN_THREAD or self_thread != self:
Log.error("Only the main thread can sleep forever (waiting for KeyboardInterrupt)")
if isinstance(please_stop, Signal):
# MUTUAL TRIGGERING, SO THEY ARE EFFECTIVELY THE SAME
self.please_stop.on_go(please_stop.go)
please_stop.on_go(self.please_stop.go)
else:
please_stop = self.please_stop
if not wait_forever:
# TRIGGER SIGNAL WHEN ALL CHILDREN THEADS ARE DONE
with self_thread.child_lock:
pending = copy(self_thread.children)
all = AndSignals(please_stop, len(pending))
for p in pending:
p.stopped.on_go(all.done)
try:
if allow_exit:
_wait_for_exit(please_stop)
else:
_wait_for_interrupt(please_stop)
except KeyboardInterrupt as _:
Log.alert("SIGINT Detected! Stopping...")
except SystemExit as _:
Log.alert("SIGTERM Detected! Stopping...")
finally:
self.stop()
class Thread(BaseThread):
"""
join() ENHANCED TO ALLOW CAPTURE OF CTRL-C, AND RETURN POSSIBLE THREAD EXCEPTIONS
run() ENHANCED TO CAPTURE EXCEPTIONS
"""
num_threads = 0
def __init__(self, name, target, *args, **kwargs):
BaseThread.__init__(self, -1)
self.name = name
self.target = target
self.end_of_thread = Data()
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get("please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if isinstance(type, BaseException):
self.please_stop.go()
# TODO: AFTER A WHILE START KILLING THREAD
self.join()
self.args = None
self.kwargs = None
def start(self):
try:
self.thread = start_new_thread(Thread._run, (self,))
return self
except Exception as e:
Log.error("Can not start thread", e)
def stop(self):
"""
SEND STOP SIGNAL, DO NOT BLOCK
"""
with self.child_lock:
children = copy(self.children)
for c in children:
DEBUG and c.name and Log.note("Stopping thread {{name|quote}}", name=c.name)
c.stop()
self.please_stop.go()
DEBUG and Log.note("Thread {{name|quote}} got request to stop", name=self.name)
def _run(self):
self.id = get_ident()
with RegisterThread(self):
try:
if self.target is not None:
a, k, self.args, self.kwargs = self.args, self.kwargs, None, None
self.end_of_thread.response = self.target(*a, **k)
self.parent.remove_child(self) # IF THREAD ENDS OK, THEN FORGET ABOUT IT
except Exception as e:
e = Except.wrap(e)
with self.synch_lock:
self.end_of_thread.exception = e
with self.parent.child_lock:
emit_problem = self not in self.parent.children
if emit_problem:
# THREAD FAILURES ARE A PROBLEM ONLY IF NO ONE WILL BE JOINING WITH IT
try:
Log.fatal("Problem in thread {{name|quote}}", name=self.name, cause=e)
except Exception:
sys.stderr.write(str("ERROR in thread: " + self.name + " " + text_type(e) + "\n"))
finally:
try:
with self.child_lock:
children = copy(self.children)
for c in children:
try:
DEBUG and sys.stdout.write(str("Stopping thread " + c.name + "\n"))
c.stop()
except Exception as e:
Log.warning("Problem stopping thread {{thread}}", thread=c.name, cause=e)
for c in children:
try:
DEBUG and sys.stdout.write(str("Joining on thread " + c.name + "\n"))
c.join()
except Exception as e:
Log.warning("Problem joining thread {{thread}}", thread=c.name, cause=e)
finally:
DEBUG and sys.stdout.write(str("Joined on thread " + c.name + "\n"))
del self.target, self.args, self.kwargs
DEBUG and Log.note("thread {{name|quote}} stopping", name=self.name)
except Exception as e:
DEBUG and Log.warning("problem with thread {{name|quote}}", cause=e, name=self.name)
finally:
self.stopped.go()
DEBUG and Log.note("thread {{name|quote}} is done", name=self.name)
def is_alive(self):
return not self.stopped
def join(self, till=None):
"""
RETURN THE RESULT {"response":r, "exception":e} OF THE THREAD EXECUTION (INCLUDING EXCEPTION, IF EXISTS)
"""
if self is Thread:
Log.error("Thread.join() is not a valid call, use t.join()")
with self.child_lock:
children = copy(self.children)
for c in children:
c.join(till=till)
DEBUG and Log.note("{{parent|quote}} waiting on thread {{child|quote}}", parent=Thread.current().name, child=self.name)
(self.stopped | till).wait()
if self.stopped:
self.parent.remove_child(self)
if not self.end_of_thread.exception:
return self.end_of_thread.response
else:
Log.error("Thread {{name|quote}} did not end well", name=self.name, cause=self.end_of_thread.exception)
else:
raise Except(type=THREAD_TIMEOUT)
@staticmethod
def run(name, target, *args, **kwargs):
# ENSURE target HAS please_stop ARGUMENT
if get_function_name(target) == 'wrapper':
pass # GIVE THE override DECORATOR A PASS
elif "please_stop" not in target.__code__.co_varnames:
Log.error("function must have please_stop argument for signalling emergency shutdown")
Thread.num_threads += 1
output = Thread(name, target, *args, **kwargs)
output.start()
return output
@staticmethod
def current():
ident = get_ident()
with ALL_LOCK:
output = ALL.get(ident)
if output is None:
Log.warning("this thread is not known. Register this thread at earliest known entry point.")
return BaseThread(get_ident())
return output
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from thread import allocate_lock as _allocate_lock
from time import sleep, time
from mo_threads.signal import Signal
DEBUG = False
INTERVAL = 0.1
_till_locker = _allocate_lock()
next_ping = time()
done = Signal("Timers shutdown")
done.go()
class Till(Signal):
"""
TIMEOUT AS A SIGNAL
"""
enabled = False
new_timers = []
def __new__(cls, till=None, timeout=None, seconds=None):
if not Till.enabled:
return done
elif till is None and timeout is None and seconds is None:
return None
else:
class Thread(object):
"""
join() ENHANCED TO ALLOW CAPTURE OF CTRL-C, AND RETURN POSSIBLE THREAD EXCEPTIONS
run() ENHANCED TO CAPTURE EXCEPTIONS
"""
num_threads = 0
def __init__(self, name, target, *args, **kwargs):
self.id = -1
self.name = name
self.target = target
self.end_of_thread = None
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get(
"please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
self.cprofiler = None
self.children = []
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if isinstance(type, BaseException):
self.please_stop.go()
# TODO: AFTER A WHILE START KILLING THREAD
self.join()
self.args = None
self.kwargs = None
def start(self):
try:
self.thread = thread.start_new_thread(Thread._run, (self, ))
return self
except Exception as e:
Log.error("Can not start thread", e)
def stop(self):
for c in copy(self.children):
if DEBUG and c.name:
Log.note("Stopping thread {{name|quote}}", name=c.name)
c.stop()
self.please_stop.go()
if DEBUG:
Log.note("Thread {{name|quote}} got request to stop",
name=self.name)
def add_child(self, child):
self.children.append(child)
def remove_child(self, child):
try:
self.children.remove(child)
except Exception as e:
# happens when multiple joins on same thread
pass
def _run(self):
with CProfiler():
self.id = thread.get_ident()
with ALL_LOCK:
ALL[self.id] = self
try:
if self.target is not None:
a, k, self.args, self.kwargs = self.args, self.kwargs, None, None
response = self.target(*a, **k)
with self.synch_lock:
self.end_of_thread = Data(response=response)
else:
with self.synch_lock:
self.end_of_thread = Null
except Exception as e:
e = Except.wrap(e)
with self.synch_lock:
self.end_of_thread = Data(exception=e)
if self not in self.parent.children:
# THREAD FAILURES ARE A PROBLEM ONLY IF NO ONE WILL BE JOINING WITH IT
try:
Log.fatal("Problem in thread {{name|quote}}",
name=self.name,
cause=e)
except Exception:
sys.stderr.write(b"ERROR in thread: " +
str(self.name) + b" " + str(e) +
b"\n")
finally:
try:
children = copy(self.children)
for c in children:
try:
if DEBUG:
sys.stdout.write(b"Stopping thread " +
str(c.name) + b"\n")
c.stop()
except Exception as e:
Log.warning("Problem stopping thread {{thread}}",
thread=c.name,
cause=e)
for c in children:
try:
if DEBUG:
sys.stdout.write(b"Joining on thread " +
str(c.name) + b"\n")
c.join()
except Exception as e:
Log.warning("Problem joining thread {{thread}}",
thread=c.name,
cause=e)
finally:
if DEBUG:
sys.stdout.write(b"Joined on thread " +
str(c.name) + b"\n")
self.stopped.go()
if DEBUG:
Log.note("thread {{name|quote}} stopping",
name=self.name)
del self.target, self.args, self.kwargs
with ALL_LOCK:
del ALL[self.id]
except Exception as e:
if DEBUG:
Log.warning("problem with thread {{name|quote}}",
cause=e,
name=self.name)
finally:
self.stopped.go()
if DEBUG:
Log.note("thread {{name|quote}} is done",
name=self.name)
def is_alive(self):
return not self.stopped
def join(self, till=None):
"""
RETURN THE RESULT {"response":r, "exception":e} OF THE THREAD EXECUTION (INCLUDING EXCEPTION, IF EXISTS)
"""
if self is Thread:
Log.error("Thread.join() is not a valid call, use t.join()")
children = copy(self.children)
for c in children:
c.join(till=till)
if DEBUG:
Log.note("{{parent|quote}} waiting on thread {{child|quote}}",
parent=Thread.current().name,
child=self.name)
(self.stopped | till).wait()
if self.stopped:
self.parent.remove_child(self)
if not self.end_of_thread.exception:
return self.end_of_thread.response
else:
Log.error("Thread {{name|quote}} did not end well",
name=self.name,
cause=self.end_of_thread.exception)
else:
raise Except(type=THREAD_TIMEOUT)
@staticmethod
def run(name, target, *args, **kwargs):
# ENSURE target HAS please_stop ARGUMENT
if "please_stop" not in target.__code__.co_varnames:
Log.error(
"function must have please_stop argument for signalling emergency shutdown"
)
Thread.num_threads += 1
output = Thread(name, target, *args, **kwargs)
output.start()
return output
@staticmethod
def wait_for_shutdown_signal(
please_stop=False, # ASSIGN SIGNAL TO STOP EARLY
allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP
wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False -> IF NO CHILD THREADS LEFT, THEN EXIT
):
"""
FOR USE BY PROCESSES NOT EXPECTED TO EVER COMPLETE UNTIL EXTERNAL
SHUTDOWN IS REQUESTED
SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit"
:param please_stop:
:param allow_exit:
:param wait_forever:: Assume all needed threads have been launched. When done
:return:
"""
if not isinstance(please_stop, Signal):
please_stop = Signal()
please_stop.on_go(
lambda: thread.start_new_thread(_stop_main_thread, ()))
self_thread = Thread.current()
if self_thread != MAIN_THREAD:
Log.error(
"Only the main thread can sleep forever (waiting for KeyboardInterrupt)"
)
if not wait_forever:
# TRIGGER SIGNAL WHEN ALL EXITING THREADS ARE DONE
pending = copy(self_thread.children)
all = AndSignals(please_stop, len(pending))
for p in pending:
p.stopped.on_go(all.done)
try:
if allow_exit:
_wait_for_exit(please_stop)
else:
_wait_for_interrupt(please_stop)
except (KeyboardInterrupt, SystemExit), _:
Log.alert("SIGINT Detected! Stopping...")
finally: