def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (str, NullType, none_type)) else cwd.abspath,
env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
shell=shell
)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
shell = True
self.name = name
self.key = (cwd, wrap(env), debug, shell)
self.stdout = Queue("stdout for " + name)
self.stderr = Queue("stderr for " + name)
with Command.available_locker:
avail = Command.available_process.setdefault(self.key, [])
if not avail:
self.process = Process("command shell", [cmd()], cwd, env,
debug, shell, bufsize)
self.process.stdin.add(set_prompt())
self.process.stdin.add("echo %errorlevel%")
_wait_for_start(self.process.stdout, Null)
else:
self.process = avail.pop()
self.process.stdin.add(" ".join(cmd_escape(p) for p in params))
self.process.stdin.add("echo %errorlevel%")
self.stdout_thread = Thread.run("", self._stream_relay,
self.process.stdout, self.stdout)
self.stderr_thread = Thread.run("", self._stream_relay,
self.process.stderr, self.stderr)
self.returncode = None
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
"""
Spawns multiple threads to manage the stdin/stdout/stderr of the child process; communication is done
via proper thread-safe queues of the same name.
Since the process is managed and monitored by threads, the main thread is not blocked when the child process
encounters problems
:param name: name given to this process
:param params: list of strings for program name and parameters
:param cwd: current working directory
:param env: enviroment variables
:param debug: true to be verbose about stdin/stdout
:param shell: true to run as command line
:param bufsize: if you want to screw stuff up
"""
self.debug = debug or DEBUG
self.process_id = Process.next_process_id
Process.next_process_id += 1
self.name = name + " (" + text(self.process_id) + ")"
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=not self.debug)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=not self.debug)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=not self.debug)
try:
if cwd == None:
cwd = os.getcwd()
else:
cwd = str(cwd)
command = [str(p) for p in params]
self.debug and Log.note("command: {{command}}", command=command)
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd,
env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
shell=shell
)
self.please_stop = Signal()
self.please_stop.then(self._kill)
self.child_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell
)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
if self.debug:
Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def enable_profilers(filename):
global FILENAME
global cprofiler_stats
if cprofiler_stats is not None:
return
if filename:
FILENAME = filename
from mo_threads.threads import ALL_LOCK, ALL, Thread
from mo_threads.queues import Queue
cprofiler_stats = Queue("cprofiler stats")
current_thread = Thread.current()
with ALL_LOCK:
threads = list(ALL.values())
for t in threads:
t.cprofiler = CProfiler()
if t is current_thread:
Log.note("starting cprofile for thread {{name}}", name=t.name)
t.cprofiler.__enter__()
else:
Log.note(
"cprofiler not started for thread {{name}} (already running)",
name=t.name)
class Process(object):
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell
)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
if self.debug:
Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.join(raise_on_error=True)
def stop(self):
self.stdin.add("exit") # ONE MORE SEND
self.please_stop.go()
def join(self, raise_on_error=False):
self.service_stopped.wait()
with self.thread_locker:
child_threads, self.children = self.children, []
for c in child_threads:
c.join()
if raise_on_error and self.returncode != 0:
Log.error(
"{{process}} FAIL: returncode={{code}}\n{{stderr}}",
process=self.name,
code=self.service.returncode,
stderr=list(self.stderr)
)
return self
def remove_child(self, child):
with self.thread_locker:
try:
self.children.remove(child)
except Exception:
pass
@property
def pid(self):
return self.service.pid
@property
def returncode(self):
return self.service.returncode
def _monitor(self, please_stop):
self.service.wait()
if self.debug:
Log.note("{{process}} STOP: returncode={{returncode}}", process=self.name, returncode=self.service.returncode)
self.service_stopped.go()
please_stop.go()
def _reader(self, name, pipe, recieve, please_stop):
try:
line = "dummy"
while not please_stop and self.service.returncode is None and line:
line = pipe.readline().rstrip()
if line:
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
# GRAB A FEW MORE LINES
max = 100
while max:
try:
line = pipe.readline().rstrip()
if line:
max = 100
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
else:
max -= 1
except Exception:
break
finally:
pipe.close()
recieve.add(THREAD_STOP)
def _writer(self, pipe, send, please_stop):
while not please_stop:
line = send.pop(till=please_stop)
if line == THREAD_STOP:
please_stop.go()
break
if line:
if self.debug:
Log.note("{{process}} (stdin): {{line}}", process=self.name, line=line.rstrip())
pipe.write(line + b"\n")
pipe.close()
def _kill(self):
try:
self.service.kill()
except Exception as e:
ee = Except.wrap(e)
if 'The operation completed successfully' in ee:
return
if 'No such process' in ee:
return
Log.warning("Failure to kill process {{process|quote}}", process=self.name, cause=ee)
class Process(object):
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name),
silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name),
silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name),
silent=True)
try:
if cwd == None:
cwd = os.getcwd()
else:
cwd = str(cwd)
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd,
env={
str(k): str(v)
for k, v in set_default(env, os.environ).items()
},
shell=shell)
self.please_stop = Signal()
self.please_stop.then(self._kill)
self.child_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}",
process=self.name,
command=" ".join(map(strings.quote, params)))
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.join(raise_on_error=True)
def stop(self):
self.stdin.add(THREAD_STOP) # ONE MORE SEND
self.please_stop.go()
def join(self, raise_on_error=False):
self.service_stopped.wait()
with self.child_locker:
child_threads, self.children = self.children, []
for c in child_threads:
c.join()
if raise_on_error and self.returncode != 0:
Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}",
process=self.name,
code=self.service.returncode,
stderr=list(self.stderr))
return self
def remove_child(self, child):
with self.child_locker:
try:
self.children.remove(child)
except Exception:
pass
@property
def pid(self):
return self.service.pid
@property
def returncode(self):
return self.service.returncode
def _monitor(self, please_stop):
with Timer(self.name):
self.service.wait()
self.debug and Log.note(
"{{process}} STOP: returncode={{returncode}}",
process=self.name,
returncode=self.service.returncode)
self.service_stopped.go()
please_stop.go()
def _reader(self, name, pipe, receive, please_stop):
try:
while not please_stop and self.service.returncode is None:
line = to_text(pipe.readline().rstrip())
if line:
receive.add(line)
self.debug and Log.note("{{process}} ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
continue
# GRAB A FEW MORE LINES
for _ in range(100):
try:
line = to_text(pipe.readline().rstrip())
if line:
receive.add(line)
self.debug and Log.note(
"{{process}} ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
break
except Exception:
break
else:
Till(seconds=5).wait()
# GRAB A FEW MORE LINES
max = 100
while max:
try:
line = to_text(pipe.readline().rstrip())
if line:
max = 100
receive.add(line)
self.debug and Log.note(
"{{process}} ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
else:
max -= 1
except Exception:
break
finally:
pipe.close()
receive.add(THREAD_STOP)
self.debug and Log.note(
"{{process}} ({{name}} is closed)", name=name, process=self.name)
receive.add(THREAD_STOP)
def _writer(self, pipe, send, please_stop):
while not please_stop:
line = send.pop(till=please_stop)
if line is THREAD_STOP:
please_stop.go()
break
elif line is None:
continue
self.debug and Log.note("{{process}} (stdin): {{line}}",
process=self.name,
line=line.rstrip())
pipe.write(line.encode('utf8') + b"\n")
pipe.flush()
def _kill(self):
try:
self.service.kill()
Log.note("Service was successfully terminated.")
except Exception as e:
ee = Except.wrap(e)
if 'The operation completed successfully' in ee:
return
if 'No such process' in ee:
return
Log.warning("Failure to kill process {{process|quote}}",
process=self.name,
cause=ee)
# THIS SIGNAL IS IMPORTANT FOR PROPER SIGNALLING WHICH ALLOWS
# FOR FAST AND PREDICTABLE SHUTDOWN AND CLEANUP OF THREADS
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from mo_logs import Log
from mo_threads.lock import Lock
from mo_threads.multiprocess import Process
from mo_threads.queues import Queue, ThreadedQueue
from mo_threads.signal import Signal
from mo_threads.threads import Thread, THREAD_STOP, THREAD_TIMEOUT, MainThread, stop_main_thread, MAIN_THREAD
from mo_threads.till import Till
Log.cprofiler_stats = Queue(
"cprofiler stats") # ACCUMULATION OF STATS FROM ALL THREADS
MAIN_THREAD.timers = Thread.run("timers daemon", till.daemon)
MAIN_THREAD.children.remove(threads.MAIN_THREAD.timers)
# from threading import Thread as _threading_Thread
# _temp = _threading_Thread.setDaemon
#
# fixes = []
# # WE NOW ADD A FIX FOR EACH KNOWN BAD ACTOR
# try:
# from paramiko import Transport
#
# def fix(self):
# if isinstance(self, Transport):
# self.stop = self.close # WE KNOW Transport DOES NOT HAVE A stop() METHOD, SO ADDING SHOULD BE FINE
class Process(object):
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name),
silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name),
silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name),
silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType,
NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
if self.debug:
Log.note("{{process}} START: {{command}}",
process=self.name,
command=" ".join(map(strings.quote, params)))
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.join(raise_on_error=True)
def stop(self):
self.stdin.add("exit") # ONE MORE SEND
self.please_stop.go()
def join(self, raise_on_error=False):
self.service_stopped.wait()
with self.thread_locker:
child_threads, self.children = self.children, []
for c in child_threads:
c.join()
if raise_on_error and self.returncode != 0:
Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}",
process=self.name,
code=self.service.returncode,
stderr=list(self.stderr))
return self
def remove_child(self, child):
with self.thread_locker:
try:
self.children.remove(child)
except Exception:
pass
@property
def pid(self):
return self.service.pid
@property
def returncode(self):
return self.service.returncode
def _monitor(self, please_stop):
self.service.wait()
if self.debug:
Log.note("{{process}} STOP: returncode={{returncode}}",
process=self.name,
returncode=self.service.returncode)
self.service_stopped.go()
please_stop.go()
def _reader(self, name, pipe, recieve, please_stop):
try:
line = "dummy"
while not please_stop and self.service.returncode is None and line:
line = pipe.readline().rstrip()
if line:
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
# GRAB A FEW MORE LINES
max = 100
while max:
try:
line = pipe.readline().rstrip()
if line:
max = 100
recieve.add(line)
if self.debug:
Log.note("{{process}} ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
else:
max -= 1
except Exception:
break
finally:
pipe.close()
recieve.add(THREAD_STOP)
def _writer(self, pipe, send, please_stop):
while not please_stop:
line = send.pop(till=please_stop)
if line == THREAD_STOP:
please_stop.go()
break
if line:
if self.debug:
Log.note("{{process}} (stdin): {{line}}",
process=self.name,
line=line.rstrip())
pipe.write(line + b"\n")
pipe.close()
def _kill(self):
try:
self.service.kill()
except Exception as e:
ee = Except.wrap(e)
if 'The operation completed successfully' in ee:
return
if 'No such process' in ee:
return
Log.warning("Failure to kill process {{process|quote}}",
process=self.name,
cause=ee)
class Process(object):
next_process_id = 0
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
"""
Spawns multiple threads to manage the stdin/stdout/stderr of the child process; communication is done
via proper thread-safe queues of the same name.
Since the process is managed and monitored by threads, the main thread is not blocked when the child process
encounters problems
:param name: name given to this process
:param params: list of strings for program name and parameters
:param cwd: current working directory
:param env: enviroment variables
:param debug: true to be verbose about stdin/stdout
:param shell: true to run as command line
:param bufsize: if you want to screw stuff up
"""
self.debug = debug or DEBUG
self.process_id = Process.next_process_id
Process.next_process_id += 1
self.name = name + " (" + text(self.process_id) + ")"
self.service_stopped = Signal("stopped signal for " +
strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name),
silent=not self.debug)
self.stdout = Queue("stdout for process " + strings.quote(name),
silent=not self.debug)
self.stderr = Queue("stderr for process " + strings.quote(name),
silent=not self.debug)
try:
if cwd == None:
cwd = os.getcwd()
else:
cwd = str(cwd)
command = [str(p) for p in params]
self.debug and Log.note("command: {{command}}", command=command)
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd,
env={
str(k): str(v)
for k, v in set_default(env, os.environ).items()
},
shell=shell)
self.please_stop = Signal()
self.please_stop.then(self._kill)
self.child_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}",
process=self.name,
command=" ".join(map(strings.quote, params)))
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.join(raise_on_error=True)
def stop(self):
self.stdin.add(THREAD_STOP) # ONE MORE SEND
self.please_stop.go()
def join(self, raise_on_error=False):
self.service_stopped.wait()
with self.child_locker:
child_threads, self.children = self.children, []
for c in child_threads:
c.join()
if raise_on_error and self.returncode != 0:
Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}",
process=self.name,
code=self.service.returncode,
stderr=list(self.stderr))
return self
def remove_child(self, child):
with self.child_locker:
try:
self.children.remove(child)
except Exception:
pass
@property
def pid(self):
return self.service.pid
@property
def returncode(self):
return self.service.returncode
def _monitor(self, please_stop):
with Timer(self.name, verbose=self.debug):
self.service.wait()
self.debug and Log.note(
"{{process}} STOP: returncode={{returncode}}",
process=self.name,
returncode=self.service.returncode)
self.service_stopped.go()
please_stop.go()
def _reader(self, name, pipe, receive, please_stop):
try:
while not please_stop and self.service.returncode is None:
line = to_text(pipe.readline().rstrip())
if line:
receive.add(line)
self.debug and Log.note("{{process}} ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
else:
(Till(seconds=1) | please_stop).wait()
# GRAB A FEW MORE LINES
max = 100
while max:
try:
line = to_text(pipe.readline().rstrip())
if line:
max = 100
receive.add(line)
self.debug and Log.note(
"{{process}} RESIDUE: ({{name}}): {{line}}",
name=name,
process=self.name,
line=line)
else:
max -= 1
except Exception:
break
finally:
pipe.close()
receive.add(THREAD_STOP)
self.debug and Log.note(
"{{process}} ({{name}} is closed)", name=name, process=self.name)
receive.add(THREAD_STOP)
def _writer(self, pipe, send, please_stop):
while not please_stop:
line = send.pop(till=please_stop)
if line is THREAD_STOP:
please_stop.go()
break
elif line is None:
continue
self.debug and Log.note("{{process}} (stdin): {{line}}",
process=self.name,
line=line.rstrip())
pipe.write(line.encode('utf8') + b"\n")
pipe.flush()
def _kill(self):
try:
self.service.kill()
Log.note("Service was successfully terminated.")
except Exception as e:
ee = Except.wrap(e)
if 'The operation completed successfully' in ee:
return
if 'No such process' in ee:
return
Log.warning("Failure to kill process {{process|quote}}",
process=self.name,
cause=ee)
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
string2quote(name))
self.stdin = Queue("stdin for process " + string2quote(name),
silent=True)
self.stdout = Queue("stdout for process " + string2quote(name),
silent=True)
self.stderr = Queue("stderr for process " + string2quote(name),
silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType,
NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception, e:
Log.error("Can not call", e)