def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (str, NullType, none_type)) else cwd.abspath,
env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
shell=shell
)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, target, *args, **kwargs):
self.id = -1
self.name = name
self.target = target
self.end_of_thread = None
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get(
"please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
self.cprofiler = Null
self.children = []
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def wait(self, till=None):
"""
THE ASSUMPTION IS wait() WILL ALWAYS RETURN WITH THE LOCK ACQUIRED
:param till: WHEN TO GIVE UP WAITING FOR ANOTHER THREAD TO SIGNAL
:return: True IF SIGNALED TO GO, False IF TIMEOUT HAPPENED
"""
waiter = Signal()
if self.waiting:
if DEBUG:
_Log.note("{{name}} waiting with others", name=self.name)
self.waiting.insert(0, waiter)
else:
self.waiting = [waiter]
try:
self.lock.release()
(waiter | till).wait()
if DEBUG:
trace = _extract_stack(0)[2]
_Log.note(
"{{name|quote}} out of lock waiting till {{till|quote}}\n{{trace}} ",
till=till.name if till else "",
name=self.name,
trace=trace)
except Exception, e:
if not _Log:
_late_import()
_Log.warning("problem", cause=e)
def __init__(self):
self.name = "Main Thread"
self.id = get_ident()
self.please_stop = Signal()
self.children = []
self.stop_logging = Log.stop
self.timers = None
self.cprofiler = Null
def __init__(self, name, target, *args, **kwargs):
BaseThread.__init__(self, -1)
self.name = coalesce(name, "thread_" + text_type(object.__hash__(self)))
self.target = target
self.end_of_thread = Data()
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get("please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def __init__(self, name, max=None, silent=False, unique=False, allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
self.name = name
self.max = coalesce(max, 2 ** 10)
self.silent = silent
self.allow_add_after_close=allow_add_after_close
self.unique = unique
self.please_stop = Signal("stop signal for " + name)
self.lock = Lock("lock for queue " + name)
self.queue = deque()
self.next_warning = time() # FOR DEBUGGING
class Till(Signal):
"""
TIMEOUT AS A SIGNAL
"""
__slots__ = []
locker = _allocate_lock()
next_ping = time()
done = Signal("Timers shutdown")
enabled = False
new_timers = []
def __new__(cls, till=None, timeout=None, seconds=None):
if not Till.enabled:
return Till.done
elif till == None and timeout == None and seconds == None:
return None
else:
return object.__new__(cls)
def __init__(self, till=None, timeout=None, seconds=None):
now = time()
if till != None:
if not isinstance(till, (float, int)):
from mo_logs import Log
Log.error("Date objects for Till are no longer allowed")
timeout = till
elif seconds != None:
timeout = now + seconds
elif timeout != None:
if not isinstance(timeout, (float, int)):
from mo_logs import Log
Log.error("Duration objects for Till are no longer allowed")
timeout = now + timeout
else:
from mo_logs import Log
Log.error("Should not happen")
Signal.__init__(self, name=text_type(timeout))
with Till.locker:
if timeout != None:
Till.next_ping = min(Till.next_ping, timeout)
Till.new_timers.append((timeout, ref(self)))
def wait(self, till=None):
"""
THE ASSUMPTION IS wait() WILL ALWAYS RETURN WITH THE LOCK ACQUIRED
:param till: WHEN TO GIVE UP WAITING FOR ANOTHER THREAD TO SIGNAL
:return: True IF SIGNALED TO GO, False IF TIMEOUT HAPPENED
"""
waiter = Signal()
if self.waiting:
if DEBUG:
_Log.note("waiting with {{num}} others on {{name|quote}}",
num=len(self.waiting),
name=self.name)
self.waiting.insert(0, waiter)
else:
if DEBUG:
_Log.note("waiting by self on {{name|quote}}", name=self.name)
self.waiting = [waiter]
try:
self.lock.release()
if DEBUG:
_Log.note("out of lock {{name|quote}}", name=self.name)
(waiter | till).wait()
if DEBUG:
_Log.note("done minimum wait (for signal {{till|quote}})",
till=till.name if till else "",
name=self.name)
except Exception as e:
if not _Log:
_late_import()
_Log.warning("problem", cause=e)
finally:
self.lock.acquire()
if DEBUG:
_Log.note("re-acquired lock {{name|quote}}", name=self.name)
try:
self.waiting.remove(waiter)
if DEBUG:
_Log.note("removed own signal from {{name|quote}}",
name=self.name)
except Exception:
pass
return bool(waiter)
def wait_for_shutdown_signal(
please_stop=False, # ASSIGN SIGNAL TO STOP EARLY
allow_exit=False, # ALLOW "exit" COMMAND ON CONSOLE TO ALSO STOP THE APP
wait_forever=True # IGNORE CHILD THREADS, NEVER EXIT. False -> IF NO CHILD THREADS LEFT, THEN EXIT
):
"""
FOR USE BY PROCESSES NOT EXPECTED TO EVER COMPLETE UNTIL EXTERNAL
SHUTDOWN IS REQUESTED
SLEEP UNTIL keyboard interrupt, OR please_stop, OR "exit"
:param please_stop:
:param allow_exit:
:param wait_forever:: Assume all needed threads have been launched. When done
:return:
"""
if not isinstance(please_stop, Signal):
please_stop = Signal()
please_stop.on_go(
lambda: thread.start_new_thread(_stop_main_thread, ()))
self_thread = Thread.current()
if self_thread != MAIN_THREAD:
Log.error(
"Only the main thread can sleep forever (waiting for KeyboardInterrupt)"
)
if not wait_forever:
# TRIGGER SIGNAL WHEN ALL EXITING THREADS ARE DONE
pending = copy(self_thread.children)
all = AndSignals(please_stop, len(pending))
for p in pending:
p.stopped.on_go(all.done)
try:
if allow_exit:
_wait_for_exit(please_stop)
else:
_wait_for_interrupt(please_stop)
except (KeyboardInterrupt, SystemExit), _:
Log.alert("SIGINT Detected! Stopping...")
def __init__(self):
BaseThread.__init__(self, get_ident())
self.name = "Main Thread"
self.please_stop = Signal()
self.stop_logging = Log.stop
self.timers = None
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
string2quote(name))
self.stdin = Queue("stdin for process " + string2quote(name),
silent=True)
self.stdout = Queue("stdout for process " + string2quote(name),
silent=True)
self.stderr = Queue("stderr for process " + string2quote(name),
silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType,
NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception, e:
Log.error("Can not call", e)
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from thread import allocate_lock as _allocate_lock
from time import sleep, time
from mo_threads.signal import Signal
DEBUG = False
INTERVAL = 0.1
_till_locker = _allocate_lock()
next_ping = time()
done = Signal("Timers shutdown")
done.go()
class Till(Signal):
"""
TIMEOUT AS A SIGNAL
"""
enabled = False
new_timers = []
def __new__(cls, till=None, timeout=None, seconds=None):
if not Till.enabled:
return done
elif till is None and timeout is None and seconds is None:
return None