def __init__(self, name, max=None, silent=False, unique=False, allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
self.name = name
self.max = coalesce(max, 2 ** 10)
self.silent = silent
self.allow_add_after_close=allow_add_after_close
self.unique = unique
self.please_stop = Signal("stop signal for " + name)
self.lock = Lock("lock for queue " + name)
self.queue = deque()
self.next_warning = time() # FOR DEBUGGING
def __init__(self, name, target, *args, **kwargs):
self.id = -1
self.name = name
self.target = target
self.end_of_thread = None
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get(
"please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
self.cprofiler = Null
self.children = []
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (str, NullType, none_type)) else cwd.abspath,
env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
shell=shell
)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, target, *args, **kwargs):
BaseThread.__init__(self, -1)
self.name = coalesce(name, "thread_" + text(object.__hash__(self)))
self.target = target
self.end_of_thread = Data()
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.please_stop = self.kwargs.get(PLEASE_STOP)
if self.please_stop is None:
self.please_stop = self.kwargs[PLEASE_STOP] = Signal(
"please_stop for " + self.name
)
self.thread = None
self.join_attempt = Signal("joining with " + self.name)
self.stopped = Signal("stopped signal for " + self.name)
if PARENT_THREAD in kwargs:
del self.kwargs[PARENT_THREAD]
self.parent = kwargs[PARENT_THREAD]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
"""
Spawns multiple threads to manage the stdin/stdout/stderr of the child process; communication is done
via proper thread-safe queues of the same name.
Since the process is managed and monitored by threads, the main thread is not blocked when the child process
encounters problems
:param name: name given to this process
:param params: list of strings for program name and parameters
:param cwd: current working directory
:param env: enviroment variables
:param debug: true to be verbose about stdin/stdout
:param shell: true to run as command line
:param bufsize: if you want to screw stuff up
"""
self.debug = debug or DEBUG
self.process_id = Process.next_process_id
Process.next_process_id += 1
self.name = name + " (" + text(self.process_id) + ")"
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=not self.debug)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=not self.debug)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=not self.debug)
try:
if cwd == None:
cwd = os.getcwd()
else:
cwd = str(cwd)
command = [str(p) for p in params]
self.debug and Log.note("command: {{command}}", command=command)
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd,
env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
shell=shell
)
self.please_stop = Signal()
self.please_stop.then(self._kill)
self.child_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self,
name,
max=None,
silent=False,
unique=False,
allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
self.name = name
self.max = coalesce(max, 2**10)
self.silent = silent
self.allow_add_after_close = allow_add_after_close
self.unique = unique
self.closed = Signal(
"stop adding signal for " +
name) # INDICATE THE PRODUCER IS DONE GENERATING ITEMS TO QUEUE
self.lock = Lock("lock for queue " + name)
self.queue = deque()
def __init__(self, name, target, *args, **kwargs):
BaseThread.__init__(self, -1)
self.name = coalesce(name, "thread_" + text_type(object.__hash__(self)))
self.target = target
self.end_of_thread = Data()
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get("please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
class Queue(object):
"""
SIMPLE MULTI-THREADED QUEUE
(multiprocessing.Queue REQUIRES SERIALIZATION, WHICH
IS DIFFICULT TO USE JUST BETWEEN THREADS)
"""
def __init__(self,
name,
max=None,
silent=False,
unique=False,
allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
self.name = name
self.max = coalesce(max, 2**10)
self.silent = silent
self.allow_add_after_close = allow_add_after_close
self.unique = unique
self.closed = Signal(
"stop adding signal for " +
name) # INDICATE THE PRODUCER IS DONE GENERATING ITEMS TO QUEUE
self.lock = Lock("lock for queue " + name)
self.queue = deque()
def __iter__(self):
try:
while True:
value = self.pop()
if value is THREAD_STOP:
break
if value is not None:
yield value
except Exception as e:
Log.warning("Tell me about what happened here", e)
def add(self, value, timeout=None, force=False):
"""
:param value: ADDED THE THE QUEUE
:param timeout: HOW LONG TO WAIT FOR QUEUE TO NOT BE FULL
:param force: ADD TO QUEUE, EVEN IF FULL (USE ONLY WHEN CONSUMER IS RETURNING WORK TO THE QUEUE)
:return: self
"""
with self.lock:
if value is THREAD_STOP:
# INSIDE THE lock SO THAT EXITING WILL RELEASE wait()
self.queue.append(value)
self.closed.go()
return
if not force:
self._wait_for_queue_space(timeout=timeout)
if self.closed and not self.allow_add_after_close:
Log.error("Do not add to closed queue")
if self.unique:
if value not in self.queue:
self.queue.append(value)
else:
self.queue.append(value)
return self
def push(self, value):
"""
SNEAK value TO FRONT OF THE QUEUE
"""
if self.closed and not self.allow_add_after_close:
Log.error("Do not push to closed queue")
with self.lock:
self._wait_for_queue_space()
if not self.closed:
self.queue.appendleft(value)
return self
def push_all(self, values):
"""
SNEAK values TO FRONT OF THE QUEUE
"""
if self.closed and not self.allow_add_after_close:
Log.error("Do not push to closed queue")
with self.lock:
self._wait_for_queue_space()
if not self.closed:
self.queue.extendleft(values)
return self
def pop_message(self, till=None):
"""
RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE
DUMMY IMPLEMENTATION FOR DEBUGGING
"""
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
return Null, self.pop(till=till)
def extend(self, values):
if self.closed and not self.allow_add_after_close:
Log.error("Do not push to closed queue")
with self.lock:
# ONCE THE queue IS BELOW LIMIT, ALLOW ADDING MORE
self._wait_for_queue_space()
if not self.closed:
if self.unique:
for v in values:
if v is THREAD_STOP:
self.closed.go()
continue
if v not in self.queue:
self.queue.append(v)
else:
for v in values:
if v is THREAD_STOP:
self.closed.go()
continue
self.queue.append(v)
return self
def _wait_for_queue_space(self, timeout=None):
"""
EXPECT THE self.lock TO BE HAD, WAITS FOR self.queue TO HAVE A LITTLE SPACE
:param timeout: IN SECONDS
"""
wait_time = 5
(DEBUG and len(self.queue) > 1 * 1000 * 1000
) and Log.warning("Queue {{name}} has over a million items")
start = time()
stop_waiting = Till(till=start + coalesce(timeout, DEFAULT_WAIT_TIME))
while not self.closed and len(self.queue) >= self.max:
if stop_waiting:
Log.error(THREAD_TIMEOUT)
if self.silent:
self.lock.wait(stop_waiting)
else:
self.lock.wait(Till(seconds=wait_time))
if not stop_waiting and len(self.queue) >= self.max:
now = time()
Log.alert(
"Queue with name {{name|quote}} is full with ({{num}} items), thread(s) have been waiting {{wait_time}} sec",
name=self.name,
num=len(self.queue),
wait_time=now - start)
def __len__(self):
with self.lock:
return len(self.queue)
def __nonzero__(self):
with self.lock:
return any(r != THREAD_STOP for r in self.queue)
def pop(self, till=None):
"""
WAIT FOR NEXT ITEM ON THE QUEUE
RETURN THREAD_STOP IF QUEUE IS CLOSED
RETURN None IF till IS REACHED AND QUEUE IS STILL EMPTY
:param till: A `Signal` to stop waiting and return None
:return: A value, or a THREAD_STOP or None
"""
if till is not None and not isinstance(till, Signal):
Log.error("expecting a signal")
with self.lock:
while True:
if self.queue:
return self.queue.popleft()
if self.closed:
break
if not self.lock.wait(till=self.closed | till):
if self.closed:
break
return None
(DEBUG or not self.silent) and Log.note(self.name + " queue closed")
return THREAD_STOP
def pop_all(self):
"""
NON-BLOCKING POP ALL IN QUEUE, IF ANY
"""
with self.lock:
output = list(self.queue)
self.queue.clear()
return output
def pop_one(self):
"""
NON-BLOCKING POP IN QUEUE, IF ANY
"""
with self.lock:
if self.closed:
return THREAD_STOP
elif not self.queue:
return None
else:
v = self.queue.popleft()
if v is THREAD_STOP: # SENDING A STOP INTO THE QUEUE IS ALSO AN OPTION
self.closed.go()
return v
def close(self):
self.closed.go()
def commit(self):
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
class Command(object):
"""
FASTER Process CLASS - OPENS A COMMAND_LINE APP (CMD on windows) AND KEEPS IT OPEN FOR MULTIPLE COMMANDS
EACH WORKING DIRECTORY WILL HAVE ITS OWN PROCESS, MULTIPLE PROCESSES WILL OPEN FOR THE SAME DIR IF MULTIPLE
THREADS ARE REQUESTING Commands
"""
available_locker = Lock("cmd lock")
available_process = {}
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
shell = True
self.name = name
self.key = (cwd, wrap(env), debug, shell)
self.stdout = Queue("stdout for " + name)
self.stderr = Queue("stderr for " + name)
with Command.available_locker:
avail = Command.available_process.setdefault(self.key, [])
if not avail:
self.process = Process("command shell", [cmd()], cwd, env,
debug, shell, bufsize)
self.process.stdin.add(set_prompt())
self.process.stdin.add("echo %errorlevel%")
_wait_for_start(self.process.stdout, Null)
else:
self.process = avail.pop()
self.process.stdin.add(" ".join(cmd_escape(p) for p in params))
self.process.stdin.add("echo %errorlevel%")
self.stdout_thread = Thread.run("", self._stream_relay,
self.process.stdout, self.stdout)
self.stderr_thread = Thread.run("", self._stream_relay,
self.process.stderr, self.stderr)
self.returncode = None
def join(self, raise_on_error=False, till=None):
try:
try:
# WAIT FOR COMMAND LINE RESPONSE ON stdout
self.stdout_thread.join()
except Exception as e:
Log.error("unexpected problem processing stdout", cause=e)
try:
self.stderr_thread.please_stop.go()
self.stderr_thread.join()
except Exception as e:
Log.error("unexpected problem processing stderr", cause=e)
if raise_on_error and self.returncode != 0:
Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}",
process=self.name,
code=self.returncode,
stderr=list(self.stderr))
return self
finally:
with Command.available_locker:
Command.available_process[self.key].append(self.process)
def _stream_relay(self, source, destination, please_stop=None):
"""
:param source:
:param destination:
:param error: Throw error if line shows up
:param please_stop:
:return:
"""
prompt_count = 0
prompt = PROMPT + ">"
line_count = 0
while not please_stop:
value = source.pop(till=please_stop)
if value is None:
destination.add(THREAD_STOP)
return
elif value is THREAD_STOP:
destination.add(THREAD_STOP)
return
elif line_count == 0 and "is not recognized as an internal or external command" in value:
Log.error("Problem with command: {{desc}}", desc=value)
elif value.startswith(prompt):
if prompt_count:
# GET THE ERROR LEVEL
self.returncode = int(source.pop(till=please_stop))
destination.add(THREAD_STOP)
return
else:
prompt_count += 1
else:
line_count += 1
destination.add(value)
else:
cr_count = -1000000 # NOT /dev/null
if line.strip() == "exit":
Log.alert("'exit' Detected! Stopping...")
return
def _wait_for_interrupt(please_stop):
DEBUG and Log.note("inside wait-for-shutdown loop")
while not please_stop:
try:
sleep(1)
except Exception:
pass
def _interrupt_main_safely():
try:
interrupt_main()
except KeyboardInterrupt:
# WE COULD BE INTERRUPTING SELF
pass
MAIN_THREAD = MainThread()
ALL_LOCK = Lock("threads ALL_LOCK")
ALL = dict()
ALL[get_ident()] = MAIN_THREAD
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
string2quote(name))
self.stdin = Queue("stdin for process " + string2quote(name),
silent=True)
self.stdout = Queue("stdout for process " + string2quote(name),
silent=True)
self.stderr = Queue("stderr for process " + string2quote(name),
silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType,
NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception, e:
Log.error("Can not call", e)
class Queue(object):
"""
SIMPLE MESSAGE QUEUE, multiprocessing.Queue REQUIRES SERIALIZATION, WHICH
IS DIFFICULT TO USE JUST BETWEEN THREADS (SERIALIZATION REQUIRED)
"""
def __init__(self,
name,
max=None,
silent=False,
unique=False,
allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
if not _Log:
_late_import()
self.name = name
self.max = coalesce(max, 2**10)
self.silent = silent
self.allow_add_after_close = allow_add_after_close
self.unique = unique
self.please_stop = Signal("stop signal for " + name)
self.lock = Lock("lock for queue " + name)
self.queue = deque()
self.next_warning = time() # FOR DEBUGGING
def __iter__(self):
try:
while True:
value = self.pop(self.please_stop)
if value is THREAD_STOP:
break
if value is not None:
yield value
except Exception as e:
_Log.warning("Tell me about what happened here", e)
if not self.silent:
_Log.note("queue iterator is done")
def add(self, value, timeout=None):
with self.lock:
if value is THREAD_STOP:
# INSIDE THE lock SO THAT EXITING WILL RELEASE wait()
self.queue.append(value)
self.please_stop.go()
return
self._wait_for_queue_space(timeout=timeout)
if self.please_stop and not self.allow_add_after_close:
_Log.error("Do not add to closed queue")
else:
if self.unique:
if value not in self.queue:
self.queue.append(value)
else:
self.queue.append(value)
return self
def push(self, value):
"""
SNEAK value TO FRONT OF THE QUEUE
"""
if self.please_stop and not self.allow_add_after_close:
_Log.error("Do not push to closed queue")
with self.lock:
self._wait_for_queue_space()
if not self.please_stop:
self.queue.appendleft(value)
return self
def pop_message(self, till=None):
"""
RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE
DUMMY IMPLEMENTATION FOR DEBUGGING
"""
if till is not None and not isinstance(till, Signal):
_Log.error("Expecting a signal")
return Null, self.pop(till=till)
def extend(self, values):
if self.please_stop and not self.allow_add_after_close:
_Log.error("Do not push to closed queue")
with self.lock:
# ONCE THE queue IS BELOW LIMIT, ALLOW ADDING MORE
self._wait_for_queue_space()
if not self.please_stop:
if self.unique:
for v in values:
if v is THREAD_STOP:
self.please_stop.go()
continue
if v not in self.queue:
self.queue.append(v)
else:
for v in values:
if v is THREAD_STOP:
self.please_stop.go()
continue
self.queue.append(v)
return self
def _wait_for_queue_space(self, timeout=DEFAULT_WAIT_TIME):
"""
EXPECT THE self.lock TO BE HAD, WAITS FOR self.queue TO HAVE A LITTLE SPACE
"""
wait_time = 5
now = time()
if timeout != None:
time_to_stop_waiting = now + timeout
else:
time_to_stop_waiting = Null
if self.next_warning < now:
self.next_warning = now + wait_time
while not self.please_stop and len(self.queue) >= self.max:
if now > time_to_stop_waiting:
if not _Log:
_late_import()
_Log.error(THREAD_TIMEOUT)
if self.silent:
self.lock.wait(Till(till=time_to_stop_waiting))
else:
self.lock.wait(Till(timeout=wait_time))
if len(self.queue) > self.max:
now = time()
if self.next_warning < now:
self.next_warning = now + wait_time
_Log.alert(
"Queue by name of {{name|quote}} is full with ({{num}} items), thread(s) have been waiting {{wait_time}} sec",
name=self.name,
num=len(self.queue),
wait_time=wait_time)
def __len__(self):
with self.lock:
return len(self.queue)
def __nonzero__(self):
with self.lock:
return any(r != THREAD_STOP for r in self.queue)
def pop(self, till=None):
"""
WAIT FOR NEXT ITEM ON THE QUEUE
RETURN THREAD_STOP IF QUEUE IS CLOSED
RETURN None IF till IS REACHED AND QUEUE IS STILL EMPTY
:param till: A `Signal` to stop waiting and return None
:return: A value, or a THREAD_STOP or None
"""
if till is not None and not isinstance(till, Signal):
_Log.error("expecting a signal")
with self.lock:
while True:
if self.queue:
value = self.queue.popleft()
return value
if self.please_stop:
break
if not self.lock.wait(till=till | self.please_stop):
if self.please_stop:
break
return None
if DEBUG or not self.silent:
_Log.note(self.name + " queue stopped")
return THREAD_STOP
def pop_all(self):
"""
NON-BLOCKING POP ALL IN QUEUE, IF ANY
"""
with self.lock:
output = list(self.queue)
self.queue.clear()
return output
def pop_one(self):
"""
NON-BLOCKING POP IN QUEUE, IF ANY
"""
with self.lock:
if self.please_stop:
return [THREAD_STOP]
elif not self.queue:
return None
else:
v = self.queue.pop()
if v is THREAD_STOP: # SENDING A STOP INTO THE QUEUE IS ALSO AN OPTION
self.please_stop.go()
return v
def close(self):
with self.lock:
self.please_stop.go()
def commit(self):
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()