def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (str, NullType, none_type)) else cwd.abspath,
env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
shell=shell
)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, target, *args, **kwargs):
self.id = -1
self.name = name
self.target = target
self.end_of_thread = None
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get(
"please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
self.cprofiler = Null
self.children = []
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def __init__(self, name, target, *args, **kwargs):
BaseThread.__init__(self, -1)
self.name = coalesce(name, "thread_" + text(object.__hash__(self)))
self.target = target
self.end_of_thread = Data()
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.please_stop = self.kwargs.get(PLEASE_STOP)
if self.please_stop is None:
self.please_stop = self.kwargs[PLEASE_STOP] = Signal(
"please_stop for " + self.name
)
self.thread = None
self.join_attempt = Signal("joining with " + self.name)
self.stopped = Signal("stopped signal for " + self.name)
if PARENT_THREAD in kwargs:
del self.kwargs[PARENT_THREAD]
self.parent = kwargs[PARENT_THREAD]
else:
self.parent = Thread.current()
self.parent.add_child(self)
def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
"""
Spawns multiple threads to manage the stdin/stdout/stderr of the child process; communication is done
via proper thread-safe queues of the same name.
Since the process is managed and monitored by threads, the main thread is not blocked when the child process
encounters problems
:param name: name given to this process
:param params: list of strings for program name and parameters
:param cwd: current working directory
:param env: enviroment variables
:param debug: true to be verbose about stdin/stdout
:param shell: true to run as command line
:param bufsize: if you want to screw stuff up
"""
self.debug = debug or DEBUG
self.process_id = Process.next_process_id
Process.next_process_id += 1
self.name = name + " (" + text(self.process_id) + ")"
self.service_stopped = Signal("stopped signal for " + strings.quote(name))
self.stdin = Queue("stdin for process " + strings.quote(name), silent=not self.debug)
self.stdout = Queue("stdout for process " + strings.quote(name), silent=not self.debug)
self.stderr = Queue("stderr for process " + strings.quote(name), silent=not self.debug)
try:
if cwd == None:
cwd = os.getcwd()
else:
cwd = str(cwd)
command = [str(p) for p in params]
self.debug and Log.note("command: {{command}}", command=command)
self.service = service = subprocess.Popen(
[str(p) for p in params],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd,
env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
shell=shell
)
self.please_stop = Signal()
self.please_stop.then(self._kill)
self.child_locker = Lock()
self.children = [
Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
]
except Exception as e:
Log.error("Can not call", e)
self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
def __init__(self, name, max=None, silent=False, unique=False, allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
self.name = name
self.max = coalesce(max, 2 ** 10)
self.silent = silent
self.allow_add_after_close=allow_add_after_close
self.unique = unique
self.please_stop = Signal("stop signal for " + name)
self.lock = Lock("lock for queue " + name)
self.queue = deque()
self.next_warning = time() # FOR DEBUGGING
def __init__(self,
name,
max=None,
silent=False,
unique=False,
allow_add_after_close=False):
"""
max - LIMIT THE NUMBER IN THE QUEUE, IF TOO MANY add() AND extend() WILL BLOCK
silent - COMPLAIN IF THE READERS ARE TOO SLOW
unique - SET True IF YOU WANT ONLY ONE INSTANCE IN THE QUEUE AT A TIME
"""
self.name = name
self.max = coalesce(max, 2**10)
self.silent = silent
self.allow_add_after_close = allow_add_after_close
self.unique = unique
self.closed = Signal(
"stop adding signal for " +
name) # INDICATE THE PRODUCER IS DONE GENERATING ITEMS TO QUEUE
self.lock = Lock("lock for queue " + name)
self.queue = deque()
def __init__(self, name, target, *args, **kwargs):
BaseThread.__init__(self, -1)
self.name = coalesce(name, "thread_" + text_type(object.__hash__(self)))
self.target = target
self.end_of_thread = Data()
self.synch_lock = Lock("response synch lock")
self.args = args
# ENSURE THERE IS A SHARED please_stop SIGNAL
self.kwargs = copy(kwargs)
self.kwargs["please_stop"] = self.kwargs.get("please_stop", Signal("please_stop for " + self.name))
self.please_stop = self.kwargs["please_stop"]
self.thread = None
self.stopped = Signal("stopped signal for " + self.name)
if "parent_thread" in kwargs:
del self.kwargs["parent_thread"]
self.parent = kwargs["parent_thread"]
else:
self.parent = Thread.current()
self.parent.add_child(self)
class Command(object):
"""
FASTER Process CLASS - OPENS A COMMAND_LINE APP (CMD on windows) AND KEEPS IT OPEN FOR MULTIPLE COMMANDS
EACH WORKING DIRECTORY WILL HAVE ITS OWN PROCESS, MULTIPLE PROCESSES WILL OPEN FOR THE SAME DIR IF MULTIPLE
THREADS ARE REQUESTING Commands
"""
available_locker = Lock("cmd lock")
available_process = {}
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
shell = True
self.name = name
self.key = (cwd, wrap(env), debug, shell)
self.stdout = Queue("stdout for " + name)
self.stderr = Queue("stderr for " + name)
with Command.available_locker:
avail = Command.available_process.setdefault(self.key, [])
if not avail:
self.process = Process("command shell", [cmd()], cwd, env,
debug, shell, bufsize)
self.process.stdin.add(set_prompt())
self.process.stdin.add("echo %errorlevel%")
_wait_for_start(self.process.stdout, Null)
else:
self.process = avail.pop()
self.process.stdin.add(" ".join(cmd_escape(p) for p in params))
self.process.stdin.add("echo %errorlevel%")
self.stdout_thread = Thread.run("", self._stream_relay,
self.process.stdout, self.stdout)
self.stderr_thread = Thread.run("", self._stream_relay,
self.process.stderr, self.stderr)
self.returncode = None
def join(self, raise_on_error=False, till=None):
try:
try:
# WAIT FOR COMMAND LINE RESPONSE ON stdout
self.stdout_thread.join()
except Exception as e:
Log.error("unexpected problem processing stdout", cause=e)
try:
self.stderr_thread.please_stop.go()
self.stderr_thread.join()
except Exception as e:
Log.error("unexpected problem processing stderr", cause=e)
if raise_on_error and self.returncode != 0:
Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}",
process=self.name,
code=self.returncode,
stderr=list(self.stderr))
return self
finally:
with Command.available_locker:
Command.available_process[self.key].append(self.process)
def _stream_relay(self, source, destination, please_stop=None):
"""
:param source:
:param destination:
:param error: Throw error if line shows up
:param please_stop:
:return:
"""
prompt_count = 0
prompt = PROMPT + ">"
line_count = 0
while not please_stop:
value = source.pop(till=please_stop)
if value is None:
destination.add(THREAD_STOP)
return
elif value is THREAD_STOP:
destination.add(THREAD_STOP)
return
elif line_count == 0 and "is not recognized as an internal or external command" in value:
Log.error("Problem with command: {{desc}}", desc=value)
elif value.startswith(prompt):
if prompt_count:
# GET THE ERROR LEVEL
self.returncode = int(source.pop(till=please_stop))
destination.add(THREAD_STOP)
return
else:
prompt_count += 1
else:
line_count += 1
destination.add(value)
else:
cr_count = -1000000 # NOT /dev/null
if line.strip() == "exit":
Log.alert("'exit' Detected! Stopping...")
return
def _wait_for_interrupt(please_stop):
DEBUG and Log.note("inside wait-for-shutdown loop")
while not please_stop:
try:
sleep(1)
except Exception:
pass
def _interrupt_main_safely():
try:
interrupt_main()
except KeyboardInterrupt:
# WE COULD BE INTERRUPTING SELF
pass
MAIN_THREAD = MainThread()
ALL_LOCK = Lock("threads ALL_LOCK")
ALL = dict()
ALL[get_ident()] = MAIN_THREAD
def __init__(self,
name,
params,
cwd=None,
env=None,
debug=False,
shell=False,
bufsize=-1):
self.name = name
self.service_stopped = Signal("stopped signal for " +
string2quote(name))
self.stdin = Queue("stdin for process " + string2quote(name),
silent=True)
self.stdout = Queue("stdout for process " + string2quote(name),
silent=True)
self.stderr = Queue("stderr for process " + string2quote(name),
silent=True)
try:
self.debug = debug or DEBUG
self.service = service = subprocess.Popen(
params,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=bufsize,
cwd=cwd if isinstance(cwd, (basestring, NullType,
NoneType)) else cwd.abspath,
env=unwrap(set_default(env, os.environ)),
shell=shell)
self.please_stop = Signal()
self.please_stop.on_go(self._kill)
self.thread_locker = Lock()
self.children = [
Thread.run(self.name + " stdin",
self._writer,
service.stdin,
self.stdin,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stdout",
self._reader,
"stdout",
service.stdout,
self.stdout,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " stderr",
self._reader,
"stderr",
service.stderr,
self.stderr,
please_stop=self.service_stopped,
parent_thread=self),
Thread.run(self.name + " waiter",
self._monitor,
parent_thread=self),
]
except Exception, e:
Log.error("Can not call", e)