Ejemplo n.º 1
0
    def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
        self.name = name
        self.service_stopped = Signal("stopped signal for " + strings.quote(name))
        self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
        self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
        self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)

        try:
            self.debug = debug or DEBUG
            self.service = service = subprocess.Popen(
                [str(p) for p in params],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=bufsize,
                cwd=cwd if isinstance(cwd, (str, NullType, none_type)) else cwd.abspath,
                env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
                shell=shell
            )

            self.please_stop = Signal()
            self.please_stop.on_go(self._kill)
            self.thread_locker = Lock()
            self.children = [
                Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
            ]
        except Exception as e:
            Log.error("Can not call", e)

        self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
Ejemplo n.º 2
0
    def __init__(self,
                 name,
                 params,
                 cwd=None,
                 env=None,
                 debug=False,
                 shell=False,
                 bufsize=-1):
        shell = True
        self.name = name
        self.key = (cwd, wrap(env), debug, shell)
        self.stdout = Queue("stdout for " + name)
        self.stderr = Queue("stderr for " + name)

        with Command.available_locker:
            avail = Command.available_process.setdefault(self.key, [])
            if not avail:
                self.process = Process("command shell", [cmd()], cwd, env,
                                       debug, shell, bufsize)
                self.process.stdin.add(set_prompt())
                self.process.stdin.add("echo %errorlevel%")
                _wait_for_start(self.process.stdout, Null)
            else:
                self.process = avail.pop()

        self.process.stdin.add(" ".join(cmd_escape(p) for p in params))
        self.process.stdin.add("echo %errorlevel%")
        self.stdout_thread = Thread.run("", self._stream_relay,
                                        self.process.stdout, self.stdout)
        self.stderr_thread = Thread.run("", self._stream_relay,
                                        self.process.stderr, self.stderr)
        self.returncode = None
    def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
        """
        Spawns multiple threads to manage the stdin/stdout/stderr of the child process; communication is done
        via proper thread-safe queues of the same name.

        Since the process is managed and monitored by threads, the main thread is not blocked when the child process
        encounters problems

        :param name: name given to this process
        :param params: list of strings for program name and parameters
        :param cwd: current working directory
        :param env: enviroment variables
        :param debug: true to be verbose about stdin/stdout
        :param shell: true to run as command line
        :param bufsize: if you want to screw stuff up
        """
        self.debug = debug or DEBUG
        self.process_id = Process.next_process_id
        Process.next_process_id += 1
        self.name = name + " (" + text(self.process_id) + ")"
        self.service_stopped = Signal("stopped signal for " + strings.quote(name))
        self.stdin = Queue("stdin for process " + strings.quote(name), silent=not self.debug)
        self.stdout = Queue("stdout for process " + strings.quote(name), silent=not self.debug)
        self.stderr = Queue("stderr for process " + strings.quote(name), silent=not self.debug)

        try:
            if cwd == None:
                cwd = os.getcwd()
            else:
                cwd = str(cwd)

            command = [str(p) for p in params]
            self.debug and Log.note("command: {{command}}", command=command)
            self.service = service = subprocess.Popen(
                [str(p) for p in params],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=bufsize,
                cwd=cwd,
                env={str(k): str(v) for k, v in set_default(env, os.environ).items()},
                shell=shell
            )

            self.please_stop = Signal()
            self.please_stop.then(self._kill)
            self.child_locker = Lock()
            self.children = [
                Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
            ]
        except Exception as e:
            Log.error("Can not call", e)

        self.debug and Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
Ejemplo n.º 4
0
    def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
        self.name = name
        self.service_stopped = Signal("stopped signal for " + strings.quote(name))
        self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
        self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
        self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)

        try:
            self.debug = debug or DEBUG
            self.service = service = subprocess.Popen(
                params,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=bufsize,
                cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath,
                env=unwrap(set_default(env, os.environ)),
                shell=shell
            )

            self.please_stop = Signal()
            self.please_stop.on_go(self._kill)
            self.thread_locker = Lock()
            self.children = [
                Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
            ]
        except Exception as e:
            Log.error("Can not call", e)

        if self.debug:
            Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))
Ejemplo n.º 5
0
def enable_profilers(filename):
    global FILENAME
    global cprofiler_stats

    if cprofiler_stats is not None:
        return
    if filename:
        FILENAME = filename

    from mo_threads.threads import ALL_LOCK, ALL, Thread
    from mo_threads.queues import Queue
    cprofiler_stats = Queue("cprofiler stats")

    current_thread = Thread.current()
    with ALL_LOCK:
        threads = list(ALL.values())
    for t in threads:
        t.cprofiler = CProfiler()
        if t is current_thread:
            Log.note("starting cprofile for thread {{name}}", name=t.name)
            t.cprofiler.__enter__()
        else:
            Log.note(
                "cprofiler not started for thread {{name}} (already running)",
                name=t.name)
Ejemplo n.º 6
0
class Process(object):
    def __init__(self, name, params, cwd=None, env=None, debug=False, shell=False, bufsize=-1):
        self.name = name
        self.service_stopped = Signal("stopped signal for " + strings.quote(name))
        self.stdin = Queue("stdin for process " + strings.quote(name), silent=True)
        self.stdout = Queue("stdout for process " + strings.quote(name), silent=True)
        self.stderr = Queue("stderr for process " + strings.quote(name), silent=True)

        try:
            self.debug = debug or DEBUG
            self.service = service = subprocess.Popen(
                params,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=bufsize,
                cwd=cwd if isinstance(cwd, (basestring, NullType, NoneType)) else cwd.abspath,
                env=unwrap(set_default(env, os.environ)),
                shell=shell
            )

            self.please_stop = Signal()
            self.please_stop.on_go(self._kill)
            self.thread_locker = Lock()
            self.children = [
                Thread.run(self.name + " stdin", self._writer, service.stdin, self.stdin, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stdout", self._reader, "stdout", service.stdout, self.stdout, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " stderr", self._reader, "stderr", service.stderr, self.stderr, please_stop=self.service_stopped, parent_thread=self),
                Thread.run(self.name + " waiter", self._monitor, parent_thread=self),
            ]
        except Exception as e:
            Log.error("Can not call", e)

        if self.debug:
            Log.note("{{process}} START: {{command}}", process=self.name, command=" ".join(map(strings.quote, params)))

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.join(raise_on_error=True)

    def stop(self):
        self.stdin.add("exit")  # ONE MORE SEND
        self.please_stop.go()

    def join(self, raise_on_error=False):
        self.service_stopped.wait()
        with self.thread_locker:
            child_threads, self.children = self.children, []
        for c in child_threads:
            c.join()
        if raise_on_error and self.returncode != 0:
            Log.error(
                "{{process}} FAIL: returncode={{code}}\n{{stderr}}",
                process=self.name,
                code=self.service.returncode,
                stderr=list(self.stderr)
            )
        return self

    def remove_child(self, child):
        with self.thread_locker:
            try:
                self.children.remove(child)
            except Exception:
                pass

    @property
    def pid(self):
        return self.service.pid

    @property
    def returncode(self):
        return self.service.returncode

    def _monitor(self, please_stop):
        self.service.wait()
        if self.debug:
            Log.note("{{process}} STOP: returncode={{returncode}}", process=self.name, returncode=self.service.returncode)
        self.service_stopped.go()
        please_stop.go()

    def _reader(self, name, pipe, recieve, please_stop):
        try:
            line = "dummy"
            while not please_stop and self.service.returncode is None and line:
                line = pipe.readline().rstrip()
                if line:
                    recieve.add(line)
                if self.debug:
                    Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
            # GRAB A FEW MORE LINES
            max = 100
            while max:
                try:
                    line = pipe.readline().rstrip()
                    if line:
                        max = 100
                        recieve.add(line)
                        if self.debug:
                            Log.note("{{process}} ({{name}}): {{line}}", name=name, process=self.name, line=line)
                    else:
                        max -= 1
                except Exception:
                    break
        finally:
            pipe.close()

        recieve.add(THREAD_STOP)

    def _writer(self, pipe, send, please_stop):
        while not please_stop:
            line = send.pop(till=please_stop)
            if line == THREAD_STOP:
                please_stop.go()
                break

            if line:
                if self.debug:
                    Log.note("{{process}} (stdin): {{line}}", process=self.name, line=line.rstrip())
                pipe.write(line + b"\n")
        pipe.close()

    def _kill(self):
        try:
            self.service.kill()
        except Exception as e:
            ee = Except.wrap(e)
            if 'The operation completed successfully' in ee:
                return
            if 'No such process' in ee:
                return

            Log.warning("Failure to kill process {{process|quote}}", process=self.name, cause=ee)
Ejemplo n.º 7
0
class Process(object):
    def __init__(self,
                 name,
                 params,
                 cwd=None,
                 env=None,
                 debug=False,
                 shell=False,
                 bufsize=-1):
        self.name = name
        self.service_stopped = Signal("stopped signal for " +
                                      strings.quote(name))
        self.stdin = Queue("stdin for process " + strings.quote(name),
                           silent=True)
        self.stdout = Queue("stdout for process " + strings.quote(name),
                            silent=True)
        self.stderr = Queue("stderr for process " + strings.quote(name),
                            silent=True)

        try:
            if cwd == None:
                cwd = os.getcwd()
            else:
                cwd = str(cwd)

            self.debug = debug or DEBUG
            self.service = service = subprocess.Popen(
                [str(p) for p in params],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=bufsize,
                cwd=cwd,
                env={
                    str(k): str(v)
                    for k, v in set_default(env, os.environ).items()
                },
                shell=shell)

            self.please_stop = Signal()
            self.please_stop.then(self._kill)
            self.child_locker = Lock()
            self.children = [
                Thread.run(self.name + " stdin",
                           self._writer,
                           service.stdin,
                           self.stdin,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " stdout",
                           self._reader,
                           "stdout",
                           service.stdout,
                           self.stdout,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " stderr",
                           self._reader,
                           "stderr",
                           service.stderr,
                           self.stderr,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " waiter",
                           self._monitor,
                           parent_thread=self),
            ]
        except Exception as e:
            Log.error("Can not call", e)

        self.debug and Log.note("{{process}} START: {{command}}",
                                process=self.name,
                                command=" ".join(map(strings.quote, params)))

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.join(raise_on_error=True)

    def stop(self):
        self.stdin.add(THREAD_STOP)  # ONE MORE SEND
        self.please_stop.go()

    def join(self, raise_on_error=False):
        self.service_stopped.wait()
        with self.child_locker:
            child_threads, self.children = self.children, []
        for c in child_threads:
            c.join()
        if raise_on_error and self.returncode != 0:
            Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}",
                      process=self.name,
                      code=self.service.returncode,
                      stderr=list(self.stderr))
        return self

    def remove_child(self, child):
        with self.child_locker:
            try:
                self.children.remove(child)
            except Exception:
                pass

    @property
    def pid(self):
        return self.service.pid

    @property
    def returncode(self):
        return self.service.returncode

    def _monitor(self, please_stop):
        with Timer(self.name):
            self.service.wait()
            self.debug and Log.note(
                "{{process}} STOP: returncode={{returncode}}",
                process=self.name,
                returncode=self.service.returncode)
            self.service_stopped.go()
            please_stop.go()

    def _reader(self, name, pipe, receive, please_stop):
        try:
            while not please_stop and self.service.returncode is None:
                line = to_text(pipe.readline().rstrip())
                if line:
                    receive.add(line)
                    self.debug and Log.note("{{process}} ({{name}}): {{line}}",
                                            name=name,
                                            process=self.name,
                                            line=line)
                    continue

                # GRAB A FEW MORE LINES
                for _ in range(100):
                    try:
                        line = to_text(pipe.readline().rstrip())
                        if line:
                            receive.add(line)
                            self.debug and Log.note(
                                "{{process}} ({{name}}): {{line}}",
                                name=name,
                                process=self.name,
                                line=line)
                            break
                    except Exception:
                        break
                else:
                    Till(seconds=5).wait()

            # GRAB A FEW MORE LINES
            max = 100
            while max:
                try:
                    line = to_text(pipe.readline().rstrip())
                    if line:
                        max = 100
                        receive.add(line)
                        self.debug and Log.note(
                            "{{process}} ({{name}}): {{line}}",
                            name=name,
                            process=self.name,
                            line=line)
                    else:
                        max -= 1
                except Exception:
                    break
        finally:
            pipe.close()
            receive.add(THREAD_STOP)
        self.debug and Log.note(
            "{{process}} ({{name}} is closed)", name=name, process=self.name)

        receive.add(THREAD_STOP)

    def _writer(self, pipe, send, please_stop):
        while not please_stop:
            line = send.pop(till=please_stop)
            if line is THREAD_STOP:
                please_stop.go()
                break
            elif line is None:
                continue

            self.debug and Log.note("{{process}} (stdin): {{line}}",
                                    process=self.name,
                                    line=line.rstrip())
            pipe.write(line.encode('utf8') + b"\n")
            pipe.flush()

    def _kill(self):
        try:
            self.service.kill()
            Log.note("Service was successfully terminated.")
        except Exception as e:
            ee = Except.wrap(e)
            if 'The operation completed successfully' in ee:
                return
            if 'No such process' in ee:
                return

            Log.warning("Failure to kill process {{process|quote}}",
                        process=self.name,
                        cause=ee)
Ejemplo n.º 8
0
# THIS SIGNAL IS IMPORTANT FOR PROPER SIGNALLING WHICH ALLOWS
# FOR FAST AND PREDICTABLE SHUTDOWN AND CLEANUP OF THREADS

from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from mo_logs import Log
from mo_threads.lock import Lock
from mo_threads.multiprocess import Process
from mo_threads.queues import Queue, ThreadedQueue
from mo_threads.signal import Signal
from mo_threads.threads import Thread, THREAD_STOP, THREAD_TIMEOUT, MainThread, stop_main_thread, MAIN_THREAD
from mo_threads.till import Till

Log.cprofiler_stats = Queue(
    "cprofiler stats")  # ACCUMULATION OF STATS FROM ALL THREADS

MAIN_THREAD.timers = Thread.run("timers daemon", till.daemon)
MAIN_THREAD.children.remove(threads.MAIN_THREAD.timers)

# from threading import Thread as _threading_Thread
# _temp = _threading_Thread.setDaemon
#
# fixes = []
# # WE NOW ADD A FIX FOR EACH KNOWN BAD ACTOR
# try:
#     from paramiko import Transport
#
#     def fix(self):
#         if isinstance(self, Transport):
#             self.stop = self.close   # WE KNOW Transport DOES NOT HAVE A stop() METHOD, SO ADDING SHOULD BE FINE
Ejemplo n.º 9
0
class Process(object):
    def __init__(self,
                 name,
                 params,
                 cwd=None,
                 env=None,
                 debug=False,
                 shell=False,
                 bufsize=-1):
        self.name = name
        self.service_stopped = Signal("stopped signal for " +
                                      strings.quote(name))
        self.stdin = Queue("stdin for process " + strings.quote(name),
                           silent=True)
        self.stdout = Queue("stdout for process " + strings.quote(name),
                            silent=True)
        self.stderr = Queue("stderr for process " + strings.quote(name),
                            silent=True)

        try:
            self.debug = debug or DEBUG
            self.service = service = subprocess.Popen(
                params,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=bufsize,
                cwd=cwd if isinstance(cwd, (basestring, NullType,
                                            NoneType)) else cwd.abspath,
                env=unwrap(set_default(env, os.environ)),
                shell=shell)

            self.please_stop = Signal()
            self.please_stop.on_go(self._kill)
            self.thread_locker = Lock()
            self.children = [
                Thread.run(self.name + " stdin",
                           self._writer,
                           service.stdin,
                           self.stdin,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " stdout",
                           self._reader,
                           "stdout",
                           service.stdout,
                           self.stdout,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " stderr",
                           self._reader,
                           "stderr",
                           service.stderr,
                           self.stderr,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " waiter",
                           self._monitor,
                           parent_thread=self),
            ]
        except Exception as e:
            Log.error("Can not call", e)

        if self.debug:
            Log.note("{{process}} START: {{command}}",
                     process=self.name,
                     command=" ".join(map(strings.quote, params)))

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.join(raise_on_error=True)

    def stop(self):
        self.stdin.add("exit")  # ONE MORE SEND
        self.please_stop.go()

    def join(self, raise_on_error=False):
        self.service_stopped.wait()
        with self.thread_locker:
            child_threads, self.children = self.children, []
        for c in child_threads:
            c.join()
        if raise_on_error and self.returncode != 0:
            Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}",
                      process=self.name,
                      code=self.service.returncode,
                      stderr=list(self.stderr))
        return self

    def remove_child(self, child):
        with self.thread_locker:
            try:
                self.children.remove(child)
            except Exception:
                pass

    @property
    def pid(self):
        return self.service.pid

    @property
    def returncode(self):
        return self.service.returncode

    def _monitor(self, please_stop):
        self.service.wait()
        if self.debug:
            Log.note("{{process}} STOP: returncode={{returncode}}",
                     process=self.name,
                     returncode=self.service.returncode)
        self.service_stopped.go()
        please_stop.go()

    def _reader(self, name, pipe, recieve, please_stop):
        try:
            line = "dummy"
            while not please_stop and self.service.returncode is None and line:
                line = pipe.readline().rstrip()
                if line:
                    recieve.add(line)
                if self.debug:
                    Log.note("{{process}} ({{name}}): {{line}}",
                             name=name,
                             process=self.name,
                             line=line)
            # GRAB A FEW MORE LINES
            max = 100
            while max:
                try:
                    line = pipe.readline().rstrip()
                    if line:
                        max = 100
                        recieve.add(line)
                        if self.debug:
                            Log.note("{{process}} ({{name}}): {{line}}",
                                     name=name,
                                     process=self.name,
                                     line=line)
                    else:
                        max -= 1
                except Exception:
                    break
        finally:
            pipe.close()

        recieve.add(THREAD_STOP)

    def _writer(self, pipe, send, please_stop):
        while not please_stop:
            line = send.pop(till=please_stop)
            if line == THREAD_STOP:
                please_stop.go()
                break

            if line:
                if self.debug:
                    Log.note("{{process}} (stdin): {{line}}",
                             process=self.name,
                             line=line.rstrip())
                pipe.write(line + b"\n")
        pipe.close()

    def _kill(self):
        try:
            self.service.kill()
        except Exception as e:
            ee = Except.wrap(e)
            if 'The operation completed successfully' in ee:
                return
            if 'No such process' in ee:
                return

            Log.warning("Failure to kill process {{process|quote}}",
                        process=self.name,
                        cause=ee)
Ejemplo n.º 10
0
class Process(object):
    next_process_id = 0

    def __init__(self,
                 name,
                 params,
                 cwd=None,
                 env=None,
                 debug=False,
                 shell=False,
                 bufsize=-1):
        """
        Spawns multiple threads to manage the stdin/stdout/stderr of the child process; communication is done
        via proper thread-safe queues of the same name.

        Since the process is managed and monitored by threads, the main thread is not blocked when the child process
        encounters problems

        :param name: name given to this process
        :param params: list of strings for program name and parameters
        :param cwd: current working directory
        :param env: enviroment variables
        :param debug: true to be verbose about stdin/stdout
        :param shell: true to run as command line
        :param bufsize: if you want to screw stuff up
        """
        self.debug = debug or DEBUG
        self.process_id = Process.next_process_id
        Process.next_process_id += 1
        self.name = name + " (" + text(self.process_id) + ")"
        self.service_stopped = Signal("stopped signal for " +
                                      strings.quote(name))
        self.stdin = Queue("stdin for process " + strings.quote(name),
                           silent=not self.debug)
        self.stdout = Queue("stdout for process " + strings.quote(name),
                            silent=not self.debug)
        self.stderr = Queue("stderr for process " + strings.quote(name),
                            silent=not self.debug)

        try:
            if cwd == None:
                cwd = os.getcwd()
            else:
                cwd = str(cwd)

            command = [str(p) for p in params]
            self.debug and Log.note("command: {{command}}", command=command)
            self.service = service = subprocess.Popen(
                [str(p) for p in params],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=bufsize,
                cwd=cwd,
                env={
                    str(k): str(v)
                    for k, v in set_default(env, os.environ).items()
                },
                shell=shell)

            self.please_stop = Signal()
            self.please_stop.then(self._kill)
            self.child_locker = Lock()
            self.children = [
                Thread.run(self.name + " stdin",
                           self._writer,
                           service.stdin,
                           self.stdin,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " stdout",
                           self._reader,
                           "stdout",
                           service.stdout,
                           self.stdout,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " stderr",
                           self._reader,
                           "stderr",
                           service.stderr,
                           self.stderr,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " waiter",
                           self._monitor,
                           parent_thread=self),
            ]
        except Exception as e:
            Log.error("Can not call", e)

        self.debug and Log.note("{{process}} START: {{command}}",
                                process=self.name,
                                command=" ".join(map(strings.quote, params)))

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.join(raise_on_error=True)

    def stop(self):
        self.stdin.add(THREAD_STOP)  # ONE MORE SEND
        self.please_stop.go()

    def join(self, raise_on_error=False):
        self.service_stopped.wait()
        with self.child_locker:
            child_threads, self.children = self.children, []
        for c in child_threads:
            c.join()
        if raise_on_error and self.returncode != 0:
            Log.error("{{process}} FAIL: returncode={{code}}\n{{stderr}}",
                      process=self.name,
                      code=self.service.returncode,
                      stderr=list(self.stderr))
        return self

    def remove_child(self, child):
        with self.child_locker:
            try:
                self.children.remove(child)
            except Exception:
                pass

    @property
    def pid(self):
        return self.service.pid

    @property
    def returncode(self):
        return self.service.returncode

    def _monitor(self, please_stop):
        with Timer(self.name, verbose=self.debug):
            self.service.wait()
            self.debug and Log.note(
                "{{process}} STOP: returncode={{returncode}}",
                process=self.name,
                returncode=self.service.returncode)
            self.service_stopped.go()
            please_stop.go()

    def _reader(self, name, pipe, receive, please_stop):
        try:
            while not please_stop and self.service.returncode is None:
                line = to_text(pipe.readline().rstrip())
                if line:
                    receive.add(line)
                    self.debug and Log.note("{{process}} ({{name}}): {{line}}",
                                            name=name,
                                            process=self.name,
                                            line=line)
                else:
                    (Till(seconds=1) | please_stop).wait()

            # GRAB A FEW MORE LINES
            max = 100
            while max:
                try:
                    line = to_text(pipe.readline().rstrip())
                    if line:
                        max = 100
                        receive.add(line)
                        self.debug and Log.note(
                            "{{process}} RESIDUE: ({{name}}): {{line}}",
                            name=name,
                            process=self.name,
                            line=line)
                    else:
                        max -= 1
                except Exception:
                    break
        finally:
            pipe.close()
            receive.add(THREAD_STOP)
        self.debug and Log.note(
            "{{process}} ({{name}} is closed)", name=name, process=self.name)

        receive.add(THREAD_STOP)

    def _writer(self, pipe, send, please_stop):
        while not please_stop:
            line = send.pop(till=please_stop)
            if line is THREAD_STOP:
                please_stop.go()
                break
            elif line is None:
                continue

            self.debug and Log.note("{{process}} (stdin): {{line}}",
                                    process=self.name,
                                    line=line.rstrip())
            pipe.write(line.encode('utf8') + b"\n")
            pipe.flush()

    def _kill(self):
        try:
            self.service.kill()
            Log.note("Service was successfully terminated.")
        except Exception as e:
            ee = Except.wrap(e)
            if 'The operation completed successfully' in ee:
                return
            if 'No such process' in ee:
                return

            Log.warning("Failure to kill process {{process|quote}}",
                        process=self.name,
                        cause=ee)
Ejemplo n.º 11
0
    def __init__(self,
                 name,
                 params,
                 cwd=None,
                 env=None,
                 debug=False,
                 shell=False,
                 bufsize=-1):
        self.name = name
        self.service_stopped = Signal("stopped signal for " +
                                      string2quote(name))
        self.stdin = Queue("stdin for process " + string2quote(name),
                           silent=True)
        self.stdout = Queue("stdout for process " + string2quote(name),
                            silent=True)
        self.stderr = Queue("stderr for process " + string2quote(name),
                            silent=True)

        try:
            self.debug = debug or DEBUG
            self.service = service = subprocess.Popen(
                params,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                bufsize=bufsize,
                cwd=cwd if isinstance(cwd, (basestring, NullType,
                                            NoneType)) else cwd.abspath,
                env=unwrap(set_default(env, os.environ)),
                shell=shell)

            self.please_stop = Signal()
            self.please_stop.on_go(self._kill)
            self.thread_locker = Lock()
            self.children = [
                Thread.run(self.name + " stdin",
                           self._writer,
                           service.stdin,
                           self.stdin,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " stdout",
                           self._reader,
                           "stdout",
                           service.stdout,
                           self.stdout,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " stderr",
                           self._reader,
                           "stderr",
                           service.stderr,
                           self.stderr,
                           please_stop=self.service_stopped,
                           parent_thread=self),
                Thread.run(self.name + " waiter",
                           self._monitor,
                           parent_thread=self),
            ]
        except Exception, e:
            Log.error("Can not call", e)